mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

IOMMU Updates for Linux v6.7 

Including:
 
 	- Core changes:
 	  - Make default-domains mandatory for all IOMMU drivers
 	  - Remove group refcounting
 	  - Add generic_single_device_group() helper and consolidate
 	    drivers
 	  - Cleanup map/unmap ops
 	  - Scaling improvements for the IOVA rcache depot
 	  - Convert dart & iommufd to the new domain_alloc_paging()
 
 	- ARM-SMMU:
 	  - Device-tree binding update:
 	    - Add qcom,sm7150-smmu-v2 for Adreno on SM7150 SoC
 	  - SMMUv2:
 	    - Support for Qualcomm SDM670 (MDSS) and SM7150 SoCs
 	  - SMMUv3:
 	    - Large refactoring of the context descriptor code to
 	      move the CD table into the master, paving the way
 	      for '->set_dev_pasid()' support on non-SVA domains
 	  - Minor cleanups to the SVA code
 
 	- Intel VT-d:
 	  - Enable debugfs to dump domain attached to a pasid
 	  - Remove an unnecessary inline function.
 
 	- AMD IOMMU:
 	  - Initial patches for SVA support (not complete yet)
 
 	- S390 IOMMU:
 	  - DMA-API conversion and optimized IOTLB flushing
 
 	- Some smaller fixes and improvements
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEr9jSbILcajRFYWYyK/BELZcBGuMFAmVJFcEACgkQK/BELZcB
 GuMgDxAAsnYVQjQ7wRkwR0rHARuEaJ+Lz2vkLNH+uYXjBzhFe2bT+ykMcZysAkdK
 A5PMLOFT5Etf+PAqOM0CoIGQFOefAId6uGl7S61Fp9ZWDKhMrOBFWhxGOaufA1Du
 tNvt3i66hwPSDZa82kY3wRCluYtj0aBBzmM6ZTwBwFZdQ7LABMtE8OxisqncVvq0
 H6vhV213fqvhCFSQJ6PnTAEiv70WvWBWygA+Z/gwYf9hypZQae91PNXdK9313a9z
 OvCzGBkL/R5/3KkJd88UhFwyYzyNGxq/DmH1etawYR5gYZ8UT/Z/sYpcx9hlO7qr
 eENPqeQc+YHZXpKqkaq66HBA1FSnXUqRZLl4cVaZahRRMe/yArsBM6R0W1AfkMAR
 rZxwHKoHUWeuHQLMVvmSDNL57h/GJJpTXjRc8HMxLZkVp+ScvnT5XCYHWWzRdCdx
 TcC/pJ1tet0FQ8rw09ovlwpGVA6eojWvcpVbLVLfGN8ZWViSVfvNFoPNb7HsGK6M
 iRi+L41Y7s63cyogC/Gsae2RAvYv29ZpvE91lmon2u+VBlTpMdOFX9EhWS6RqOBF
 cV30bhsw0dyCB7v5jDPtABYEOaR6l1mPLhn1gX3u0Ue/tmPhLX69k4bVWBY6wP3p
 gmmJD9ub8FuPQtFCGPE7/8ZINjGGrfiKO24DNI2Ty3XEeq21hU4=
 =UyWC
 -----END PGP SIGNATURE-----

Merge tag 'iommu-updates-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel:
 "Core changes:
   - Make default-domains mandatory for all IOMMU drivers
   - Remove group refcounting
   - Add generic_single_device_group() helper and consolidate drivers
   - Cleanup map/unmap ops
   - Scaling improvements for the IOVA rcache depot
   - Convert dart & iommufd to the new domain_alloc_paging()

  ARM-SMMU:
   - Device-tree binding update:
       - Add qcom,sm7150-smmu-v2 for Adreno on SM7150 SoC
   - SMMUv2:
       - Support for Qualcomm SDM670 (MDSS) and SM7150 SoCs
   - SMMUv3:
       - Large refactoring of the context descriptor code to move the CD
         table into the master, paving the way for '->set_dev_pasid()'
         support on non-SVA domains
   - Minor cleanups to the SVA code

  Intel VT-d:
   - Enable debugfs to dump domain attached to a pasid
   - Remove an unnecessary inline function

  AMD IOMMU:
   - Initial patches for SVA support (not complete yet)

  S390 IOMMU:
   - DMA-API conversion and optimized IOTLB flushing

  And some smaller fixes and improvements"

* tag 'iommu-updates-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (102 commits)
  iommu/dart: Remove the force_bypass variable
  iommu/dart: Call apple_dart_finalize_domain() as part of alloc_paging()
  iommu/dart: Convert to domain_alloc_paging()
  iommu/dart: Move the blocked domain support to a global static
  iommu/dart: Use static global identity domains
  iommufd: Convert to alloc_domain_paging()
  iommu/vt-d: Use ops->blocked_domain
  iommu/vt-d: Update the definition of the blocking domain
  iommu: Move IOMMU_DOMAIN_BLOCKED global statics to ops->blocked_domain
  Revert "iommu/vt-d: Remove unused function"
  iommu/amd: Remove DMA_FQ type from domain allocation path
  iommu: change iommu_map_sgtable to return signed values
  iommu/virtio: Add __counted_by for struct viommu_request and use struct_size()
  iommu/vt-d: debugfs: Support dumping a specified page table
  iommu/vt-d: debugfs: Create/remove debugfs file per {device, pasid}
  iommu/vt-d: debugfs: Dump entry pointing to huge page
  iommu/vt-d: Remove unused function
  iommu/arm-smmu-v3-sva: Remove bond refcount
  iommu/arm-smmu-v3-sva: Remove unused iommu_sva handle
  iommu/arm-smmu-v3: Rename cdcfg to cd_table
  ...
This commit is contained in:
Linus Torvalds 2023-11-09 13:37:28 -08:00
parent 6bc986ab83 e8cca466a8
commit 4bbdb725a3
58 changed files with 2148 additions and 3918 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Documentation/admin-guide/kernel-parameters.txt
View File

@ -2227,7 +2227,7 @@
forcing Dual Address Cycle for PCI cards supporting
greater than 32-bit addressing.
iommu.strict= [ARM64, X86] Configure TLB invalidation behaviour
iommu.strict= [ARM64, X86, S390] Configure TLB invalidation behaviour
Format: { "0" | "1" }
0 - Lazy mode.
Request that DMA unmap operations use deferred
@ -5694,9 +5694,10 @@
s390_iommu= [HW,S390]
Set s390 IOTLB flushing mode
strict
With strict flushing every unmap operation will result in
an IOTLB flush. Default is lazy flushing before reuse,
which is faster.
With strict flushing every unmap operation will result
in an IOTLB flush. Default is lazy flushing before
reuse, which is faster. Deprecated, equivalent to
iommu.strict=1.
s390_iommu_aperture= [KNL,S390]
Specifies the size of the per device DMA address space

2
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
View File

@ -110,6 +110,7 @@ properties:
- qcom,sdm630-smmu-v2
- qcom,sdm845-smmu-v2
- qcom,sm6350-smmu-v2
- qcom,sm7150-smmu-v2
- const: qcom,adreno-smmu
- const: qcom,smmu-v2
- description: Qcom Adreno GPUs on Google Cheza platform
@ -409,6 +410,7 @@ allOf:
contains:
enum:
- qcom,sm6350-smmu-v2
- qcom,sm7150-smmu-v2
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
then:

1
arch/arm/configs/multi_v7_defconfig
View File

@ -1076,7 +1076,6 @@ CONFIG_QCOM_IPCC=y
CONFIG_OMAP_IOMMU=y
CONFIG_OMAP_IOMMU_DEBUG=y
CONFIG_ROCKCHIP_IOMMU=y
CONFIG_TEGRA_IOMMU_GART=y
CONFIG_TEGRA_IOMMU_SMMU=y
CONFIG_EXYNOS_IOMMU=y
CONFIG_QCOM_IOMMU=y

1
arch/arm/configs/tegra_defconfig
View File

@ -292,7 +292,6 @@ CONFIG_CHROME_PLATFORMS=y
CONFIG_CROS_EC=y
CONFIG_CROS_EC_I2C=m
CONFIG_CROS_EC_SPI=m
CONFIG_TEGRA_IOMMU_GART=y
CONFIG_TEGRA_IOMMU_SMMU=y
CONFIG_ARCH_TEGRA_2x_SOC=y
CONFIG_ARCH_TEGRA_3x_SOC=y

53
arch/powerpc/kernel/iommu.c
View File

@ -1280,13 +1280,19 @@ struct iommu_table_group_ops spapr_tce_table_group_ops = {
/*
* A simple iommu_ops to allow less cruft in generic VFIO code.
*/
static int spapr_tce_blocking_iommu_attach_dev(struct iommu_domain *dom,
struct device *dev)
static int
spapr_tce_platform_iommu_attach_dev(struct iommu_domain *platform_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct iommu_group *grp = iommu_group_get(dev);
struct iommu_table_group *table_group;
int ret = -EINVAL;
/* At first attach the ownership is already set */
if (!domain)
return 0;
if (!grp)
return -ENODEV;
@ -1297,17 +1303,22 @@ static int spapr_tce_blocking_iommu_attach_dev(struct iommu_domain *dom,
return ret;
}
static void spapr_tce_blocking_iommu_set_platform_dma(struct device *dev)
{
struct iommu_group *grp = iommu_group_get(dev);
struct iommu_table_group *table_group;
static const struct iommu_domain_ops spapr_tce_platform_domain_ops = {
.attach_dev = spapr_tce_platform_iommu_attach_dev,
};
table_group = iommu_group_get_iommudata(grp);
table_group->ops->release_ownership(table_group);
}
static struct iommu_domain spapr_tce_platform_domain = {
.type = IOMMU_DOMAIN_PLATFORM,
.ops = &spapr_tce_platform_domain_ops,
};
static const struct iommu_domain_ops spapr_tce_blocking_domain_ops = {
.attach_dev = spapr_tce_blocking_iommu_attach_dev,
static struct iommu_domain spapr_tce_blocked_domain = {
.type = IOMMU_DOMAIN_BLOCKED,
/*
* FIXME: SPAPR mixes blocked and platform behaviors, the blocked domain
* also sets the dma_api ops
*/
.ops = &spapr_tce_platform_domain_ops,
};
static bool spapr_tce_iommu_capable(struct device *dev, enum iommu_cap cap)
@ -1322,22 +1333,6 @@ static bool spapr_tce_iommu_capable(struct device *dev, enum iommu_cap cap)
return false;
}
static struct iommu_domain *spapr_tce_iommu_domain_alloc(unsigned int type)
{
struct iommu_domain *dom;
if (type != IOMMU_DOMAIN_BLOCKED)
return NULL;
dom = kzalloc(sizeof(*dom), GFP_KERNEL);
if (!dom)
return NULL;
dom->ops = &spapr_tce_blocking_domain_ops;
return dom;
}
static struct iommu_device *spapr_tce_iommu_probe_device(struct device *dev)
{
struct pci_dev *pdev;
@ -1371,12 +1366,12 @@ static struct iommu_group *spapr_tce_iommu_device_group(struct device *dev)
}
static const struct iommu_ops spapr_tce_iommu_ops = {
.default_domain = &spapr_tce_platform_domain,
.blocked_domain = &spapr_tce_blocked_domain,
.capable = spapr_tce_iommu_capable,
.domain_alloc = spapr_tce_iommu_domain_alloc,
.probe_device = spapr_tce_iommu_probe_device,
.release_device = spapr_tce_iommu_release_device,
.device_group = spapr_tce_iommu_device_group,
.set_platform_dma_ops = spapr_tce_blocking_iommu_set_platform_dma,
};
static struct attribute *spapr_tce_iommu_attrs[] = {

11
arch/s390/include/asm/pci.h
View File

@ -159,13 +159,6 @@ struct zpci_dev {
unsigned long *dma_table;
int tlb_refresh;
spinlock_t iommu_bitmap_lock;
unsigned long *iommu_bitmap;
unsigned long *lazy_bitmap;
unsigned long iommu_size;
unsigned long iommu_pages;
unsigned int next_bit;
struct iommu_device iommu_dev; /* IOMMU core handle */
char res_name[16];
@ -180,10 +173,6 @@ struct zpci_dev {
struct zpci_fmb *fmb;
u16 fmb_update; /* update interval */
u16 fmb_length;
/* software counters */
atomic64_t allocated_pages;
atomic64_t mapped_pages;
atomic64_t unmapped_pages;
u8 version;
enum pci_bus_speed max_bus_speed;

3
arch/s390/include/asm/pci_clp.h
View File

@ -50,6 +50,9 @@ struct clp_fh_list_entry {
#define CLP_UTIL_STR_LEN 64
#define CLP_PFIP_NR_SEGMENTS 4
/* PCI function type numbers */
#define PCI_FUNC_TYPE_ISM 0x5 /* ISM device */
extern bool zpci_unique_uid;
struct clp_rsp_slpc_pci {

119
arch/s390/include/asm/pci_dma.h
View File

@ -82,117 +82,16 @@ enum zpci_ioat_dtype {
#define ZPCI_TABLE_VALID_MASK 0x20
#define ZPCI_TABLE_PROT_MASK 0x200
static inline unsigned int calc_rtx(dma_addr_t ptr)
{
return ((unsigned long) ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
}
struct zpci_iommu_ctrs {
atomic64_t mapped_pages;
atomic64_t unmapped_pages;
atomic64_t global_rpcits;
atomic64_t sync_map_rpcits;
atomic64_t sync_rpcits;
};
static inline unsigned int calc_sx(dma_addr_t ptr)
{
return ((unsigned long) ptr >> ZPCI_ST_SHIFT) & ZPCI_INDEX_MASK;
}
static inline unsigned int calc_px(dma_addr_t ptr)
{
return ((unsigned long) ptr >> PAGE_SHIFT) & ZPCI_PT_MASK;
}
static inline void set_pt_pfaa(unsigned long *entry, phys_addr_t pfaa)
{
*entry &= ZPCI_PTE_FLAG_MASK;
*entry |= (pfaa & ZPCI_PTE_ADDR_MASK);
}
static inline void set_rt_sto(unsigned long *entry, phys_addr_t sto)
{
*entry &= ZPCI_RTE_FLAG_MASK;
*entry |= (sto & ZPCI_RTE_ADDR_MASK);
*entry |= ZPCI_TABLE_TYPE_RTX;
}
static inline void set_st_pto(unsigned long *entry, phys_addr_t pto)
{
*entry &= ZPCI_STE_FLAG_MASK;
*entry |= (pto & ZPCI_STE_ADDR_MASK);
*entry |= ZPCI_TABLE_TYPE_SX;
}
static inline void validate_rt_entry(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_VALID_MASK;
*entry &= ~ZPCI_TABLE_OFFSET_MASK;
*entry |= ZPCI_TABLE_VALID;
*entry |= ZPCI_TABLE_LEN_RTX;
}
static inline void validate_st_entry(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_VALID_MASK;
*entry |= ZPCI_TABLE_VALID;
}
static inline void invalidate_pt_entry(unsigned long *entry)
{
WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_INVALID);
*entry &= ~ZPCI_PTE_VALID_MASK;
*entry |= ZPCI_PTE_INVALID;
}
static inline void validate_pt_entry(unsigned long *entry)
{
WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID);
*entry &= ~ZPCI_PTE_VALID_MASK;
*entry |= ZPCI_PTE_VALID;
}
static inline void entry_set_protected(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_PROT_MASK;
*entry |= ZPCI_TABLE_PROTECTED;
}
static inline void entry_clr_protected(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_PROT_MASK;
*entry |= ZPCI_TABLE_UNPROTECTED;
}
static inline int reg_entry_isvalid(unsigned long entry)
{
return (entry & ZPCI_TABLE_VALID_MASK) == ZPCI_TABLE_VALID;
}
static inline int pt_entry_isvalid(unsigned long entry)
{
return (entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID;
}
static inline unsigned long *get_rt_sto(unsigned long entry)
{
if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RTX)
return phys_to_virt(entry & ZPCI_RTE_ADDR_MASK);
else
return NULL;
}
static inline unsigned long *get_st_pto(unsigned long entry)
{
if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_SX)
return phys_to_virt(entry & ZPCI_STE_ADDR_MASK);
else
return NULL;
}
/* Prototypes */
void dma_free_seg_table(unsigned long);
unsigned long *dma_alloc_cpu_table(gfp_t gfp);
void dma_cleanup_tables(unsigned long *);
unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr,
gfp_t gfp);
void dma_update_cpu_trans(unsigned long *entry, phys_addr_t page_addr, int flags);
extern const struct dma_map_ops s390_pci_dma_ops;
struct zpci_dev;
struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev);
#endif

2
arch/s390/pci/Makefile
View File

@ -3,7 +3,7 @@
# Makefile for the s390 PCI subsystem.
#
obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \
obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_clp.o pci_sysfs.o \
pci_event.o pci_debug.o pci_insn.o pci_mmio.o \
pci_bus.o pci_kvm_hook.o
obj-$(CONFIG_PCI_IOV) += pci_iov.o

35
arch/s390/pci/pci.c
View File

@ -124,7 +124,11 @@ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
WARN_ON_ONCE(iota & 0x3fff);
fib.pba = base;
fib.pal = limit;
/* Work around off by one in ISM virt device */
if (zdev->pft == PCI_FUNC_TYPE_ISM && limit > base)
fib.pal = limit + (1 << 12);
else
fib.pal = limit;
fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
fib.gd = zdev->gisa;
cc = zpci_mod_fc(req, &fib, status);
@ -153,6 +157,7 @@ int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
int zpci_fmb_enable_device(struct zpci_dev *zdev)
{
u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
struct zpci_iommu_ctrs *ctrs;
struct zpci_fib fib = {0};
u8 cc, status;
@ -165,9 +170,15 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
WARN_ON((u64) zdev->fmb & 0xf);
/* reset software counters */
atomic64_set(&zdev->allocated_pages, 0);
atomic64_set(&zdev->mapped_pages, 0);
atomic64_set(&zdev->unmapped_pages, 0);
ctrs = zpci_get_iommu_ctrs(zdev);
if (ctrs) {
atomic64_set(&ctrs->mapped_pages, 0);
atomic64_set(&ctrs->unmapped_pages, 0);
atomic64_set(&ctrs->global_rpcits, 0);
atomic64_set(&ctrs->sync_map_rpcits, 0);
atomic64_set(&ctrs->sync_rpcits, 0);
}
fib.fmb_addr = virt_to_phys(zdev->fmb);
fib.gd = zdev->gisa;
@ -582,7 +593,6 @@ int pcibios_device_add(struct pci_dev *pdev)
pdev->no_vf_scan = 1;
pdev->dev.groups = zpci_attr_groups;
pdev->dev.dma_ops = &s390_pci_dma_ops;
zpci_map_resources(pdev);
for (i = 0; i < PCI_STD_NUM_BARS; i++) {
@ -756,8 +766,6 @@ int zpci_hot_reset_device(struct zpci_dev *zdev)
if (zdev->dma_table)
rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table), &status);
else
rc = zpci_dma_init_device(zdev);
if (rc) {
zpci_disable_device(zdev);
return rc;
@ -865,11 +873,6 @@ int zpci_deconfigure_device(struct zpci_dev *zdev)
if (zdev->zbus->bus)
zpci_bus_remove_device(zdev, false);
if (zdev->dma_table) {
rc = zpci_dma_exit_device(zdev);
if (rc)
return rc;
}
if (zdev_enabled(zdev)) {
rc = zpci_disable_device(zdev);
if (rc)
@ -918,8 +921,6 @@ void zpci_release_device(struct kref *kref)
if (zdev->zbus->bus)
zpci_bus_remove_device(zdev, false);
if (zdev->dma_table)
zpci_dma_exit_device(zdev);
if (zdev_enabled(zdev))
zpci_disable_device(zdev);
@ -1109,10 +1110,6 @@ static int __init pci_base_init(void)
if (rc)
goto out_irq;
rc = zpci_dma_init();
if (rc)
goto out_dma;
rc = clp_scan_pci_devices();
if (rc)
goto out_find;
@ -1122,8 +1119,6 @@ static int __init pci_base_init(void)
return 0;
out_find:
zpci_dma_exit();
out_dma:
zpci_irq_exit();
out_irq:
zpci_mem_exit();

5
arch/s390/pci/pci_bus.c
View File

@ -47,11 +47,6 @@ static int zpci_bus_prepare_device(struct zpci_dev *zdev)
rc = zpci_enable_device(zdev);
if (rc)
return rc;
rc = zpci_dma_init_device(zdev);
if (rc) {
zpci_disable_device(zdev);
return rc;
}
}
if (!zdev->has_resources) {

12
arch/s390/pci/pci_debug.c
View File

@ -53,9 +53,11 @@ static char *pci_fmt3_names[] = {
};
static char *pci_sw_names[] = {
"Allocated pages",
"Mapped pages",
"Unmapped pages",
"Global RPCITs",
"Sync Map RPCITs",
"Sync RPCITs",
};
static void pci_fmb_show(struct seq_file *m, char *name[], int length,
@ -69,10 +71,14 @@ static void pci_fmb_show(struct seq_file *m, char *name[], int length,
static void pci_sw_counter_show(struct seq_file *m)
{
struct zpci_dev *zdev = m->private;
atomic64_t *counter = &zdev->allocated_pages;
struct zpci_iommu_ctrs *ctrs = zpci_get_iommu_ctrs(m->private);
atomic64_t *counter;
int i;
if (!ctrs)
return;
counter = &ctrs->mapped_pages;
for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
atomic64_read(counter));

746
arch/s390/pci/pci_dma.c
View File

@ -1,746 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright IBM Corp. 2012
*
* Author(s):
* Jan Glauber <jang@linux.vnet.ibm.com>
*/
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/iommu-helper.h>
#include <linux/dma-map-ops.h>
#include <linux/vmalloc.h>
#include <linux/pci.h>
#include <asm/pci_dma.h>
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
static int s390_iommu_strict;
static u64 s390_iommu_aperture;
static u32 s390_iommu_aperture_factor = 1;
static int zpci_refresh_global(struct zpci_dev *zdev)
{
return zpci_refresh_trans((u64) zdev->fh << 32, zdev->start_dma,
zdev->iommu_pages * PAGE_SIZE);
}
unsigned long *dma_alloc_cpu_table(gfp_t gfp)
{
unsigned long *table, *entry;
table = kmem_cache_alloc(dma_region_table_cache, gfp);
if (!table)
return NULL;
for (entry = table; entry < table + ZPCI_TABLE_ENTRIES; entry++)
*entry = ZPCI_TABLE_INVALID;
return table;
}
static void dma_free_cpu_table(void *table)
{
kmem_cache_free(dma_region_table_cache, table);
}
static unsigned long *dma_alloc_page_table(gfp_t gfp)
{
unsigned long *table, *entry;
table = kmem_cache_alloc(dma_page_table_cache, gfp);
if (!table)
return NULL;
for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++)
*entry = ZPCI_PTE_INVALID;
return table;
}
static void dma_free_page_table(void *table)
{
kmem_cache_free(dma_page_table_cache, table);
}
static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp)
{
unsigned long old_rte, rte;
unsigned long *sto;
rte = READ_ONCE(*rtep);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
} else {
sto = dma_alloc_cpu_table(gfp);
if (!sto)
return NULL;
set_rt_sto(&rte, virt_to_phys(sto));
validate_rt_entry(&rte);
entry_clr_protected(&rte);
old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte);
if (old_rte != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_cpu_table(sto);
sto = get_rt_sto(old_rte);
}
}
return sto;
}
static unsigned long *dma_get_page_table_origin(unsigned long *step, gfp_t gfp)
{
unsigned long old_ste, ste;
unsigned long *pto;
ste = READ_ONCE(*step);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
} else {
pto = dma_alloc_page_table(gfp);
if (!pto)
return NULL;
set_st_pto(&ste, virt_to_phys(pto));
validate_st_entry(&ste);
entry_clr_protected(&ste);
old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste);
if (old_ste != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_page_table(pto);
pto = get_st_pto(old_ste);
}
}
return pto;
}
unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr,
gfp_t gfp)
{
unsigned long *sto, *pto;
unsigned int rtx, sx, px;
rtx = calc_rtx(dma_addr);
sto = dma_get_seg_table_origin(&rto[rtx], gfp);
if (!sto)
return NULL;
sx = calc_sx(dma_addr);
pto = dma_get_page_table_origin(&sto[sx], gfp);
if (!pto)
return NULL;
px = calc_px(dma_addr);
return &pto[px];
}
void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags)
{
unsigned long pte;
pte = READ_ONCE(*ptep);
if (flags & ZPCI_PTE_INVALID) {
invalidate_pt_entry(&pte);
} else {
set_pt_pfaa(&pte, page_addr);
validate_pt_entry(&pte);
}
if (flags & ZPCI_TABLE_PROTECTED)
entry_set_protected(&pte);
else
entry_clr_protected(&pte);
xchg(ptep, pte);
}
static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
dma_addr_t dma_addr, size_t size, int flags)
{
unsigned int nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
phys_addr_t page_addr = (pa & PAGE_MASK);
unsigned long *entry;
int i, rc = 0;
if (!nr_pages)
return -EINVAL;
if (!zdev->dma_table)
return -EINVAL;
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr,
GFP_ATOMIC);
if (!entry) {
rc = -ENOMEM;
goto undo_cpu_trans;
}
dma_update_cpu_trans(entry, page_addr, flags);
page_addr += PAGE_SIZE;
dma_addr += PAGE_SIZE;
}
undo_cpu_trans:
if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
flags = ZPCI_PTE_INVALID;
while (i-- > 0) {
page_addr -= PAGE_SIZE;
dma_addr -= PAGE_SIZE;
entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr,
GFP_ATOMIC);
if (!entry)
break;
dma_update_cpu_trans(entry, page_addr, flags);
}
}
return rc;
}
static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr,
size_t size, int flags)
{
unsigned long irqflags;
int ret;
/*
* With zdev->tlb_refresh == 0, rpcit is not required to establish new
* translations when previously invalid translation-table entries are
* validated. With lazy unmap, rpcit is skipped for previously valid
* entries, but a global rpcit is then required before any address can
* be re-used, i.e. after each iommu bitmap wrap-around.
*/
if ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID) {
if (!zdev->tlb_refresh)
return 0;
} else {
if (!s390_iommu_strict)
return 0;
}
ret = zpci_refresh_trans((u64) zdev->fh << 32, dma_addr,
PAGE_ALIGN(size));
if (ret == -ENOMEM && !s390_iommu_strict) {
/* enable the hypervisor to free some resources */
if (zpci_refresh_global(zdev))
goto out;
spin_lock_irqsave(&zdev->iommu_bitmap_lock, irqflags);
bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap,
zdev->lazy_bitmap, zdev->iommu_pages);
bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages);
spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, irqflags);
ret = 0;
}
out:
return ret;
}
static int dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
dma_addr_t dma_addr, size_t size, int flags)
{
int rc;
rc = __dma_update_trans(zdev, pa, dma_addr, size, flags);
if (rc)
return rc;
rc = __dma_purge_tlb(zdev, dma_addr, size, flags);
if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID))
__dma_update_trans(zdev, pa, dma_addr, size, ZPCI_PTE_INVALID);
return rc;
}
void dma_free_seg_table(unsigned long entry)
{
unsigned long *sto = get_rt_sto(entry);
int sx;
for (sx = 0; sx < ZPCI_TABLE_ENTRIES; sx++)
if (reg_entry_isvalid(sto[sx]))
dma_free_page_table(get_st_pto(sto[sx]));
dma_free_cpu_table(sto);
}
void dma_cleanup_tables(unsigned long *table)
{
int rtx;
if (!table)
return;
for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++)
if (reg_entry_isvalid(table[rtx]))
dma_free_seg_table(table[rtx]);
dma_free_cpu_table(table);
}
static unsigned long __dma_alloc_iommu(struct device *dev,
unsigned long start, int size)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages,
start, size, zdev->start_dma >> PAGE_SHIFT,
dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT),
0);
}
static dma_addr_t dma_alloc_address(struct device *dev, int size)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
unsigned long offset, flags;
spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
offset = __dma_alloc_iommu(dev, zdev->next_bit, size);
if (offset == -1) {
if (!s390_iommu_strict) {
/* global flush before DMA addresses are reused */
if (zpci_refresh_global(zdev))
goto out_error;
bitmap_andnot(zdev->iommu_bitmap, zdev->iommu_bitmap,
zdev->lazy_bitmap, zdev->iommu_pages);
bitmap_zero(zdev->lazy_bitmap, zdev->iommu_pages);
}
/* wrap-around */
offset = __dma_alloc_iommu(dev, 0, size);
if (offset == -1)
goto out_error;
}
zdev->next_bit = offset + size;
spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
return zdev->start_dma + offset * PAGE_SIZE;
out_error:
spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
return DMA_MAPPING_ERROR;
}
static void dma_free_address(struct device *dev, dma_addr_t dma_addr, int size)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
unsigned long flags, offset;
offset = (dma_addr - zdev->start_dma) >> PAGE_SHIFT;
spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
if (!zdev->iommu_bitmap)
goto out;
if (s390_iommu_strict)
bitmap_clear(zdev->iommu_bitmap, offset, size);
else
bitmap_set(zdev->lazy_bitmap, offset, size);
out:
spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
}
static inline void zpci_err_dma(unsigned long rc, unsigned long addr)
{
struct {
unsigned long rc;
unsigned long addr;
} __packed data = {rc, addr};
zpci_err_hex(&data, sizeof(data));
}
static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction direction,
unsigned long attrs)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
unsigned long pa = page_to_phys(page) + offset;
int flags = ZPCI_PTE_VALID;
unsigned long nr_pages;
dma_addr_t dma_addr;
int ret;
/* This rounds up number of pages based on size and offset */
nr_pages = iommu_num_pages(pa, size, PAGE_SIZE);
dma_addr = dma_alloc_address(dev, nr_pages);
if (dma_addr == DMA_MAPPING_ERROR) {
ret = -ENOSPC;
goto out_err;
}
/* Use rounded up size */
size = nr_pages * PAGE_SIZE;
if (direction == DMA_NONE || direction == DMA_TO_DEVICE)
flags |= ZPCI_TABLE_PROTECTED;
ret = dma_update_trans(zdev, pa, dma_addr, size, flags);
if (ret)
goto out_free;
atomic64_add(nr_pages, &zdev->mapped_pages);
return dma_addr + (offset & ~PAGE_MASK);
out_free:
dma_free_address(dev, dma_addr, nr_pages);
out_err:
zpci_err("map error:\n");
zpci_err_dma(ret, pa);
return DMA_MAPPING_ERROR;
}
static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr,
size_t size, enum dma_data_direction direction,
unsigned long attrs)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
int npages, ret;
npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
dma_addr = dma_addr & PAGE_MASK;
ret = dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE,
ZPCI_PTE_INVALID);
if (ret) {
zpci_err("unmap error:\n");
zpci_err_dma(ret, dma_addr);
return;
}
atomic64_add(npages, &zdev->unmapped_pages);
dma_free_address(dev, dma_addr, npages);
}
static void *s390_dma_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
struct page *page;
phys_addr_t pa;
dma_addr_t map;
size = PAGE_ALIGN(size);
page = alloc_pages(flag | __GFP_ZERO, get_order(size));
if (!page)
return NULL;
pa = page_to_phys(page);
map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, 0);
if (dma_mapping_error(dev, map)) {
__free_pages(page, get_order(size));
return NULL;
}
atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages);
if (dma_handle)
*dma_handle = map;
return phys_to_virt(pa);
}
static void s390_dma_free(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
unsigned long attrs)
{
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
size = PAGE_ALIGN(size);
atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages);
s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, 0);
free_pages((unsigned long)vaddr, get_order(size));
}
/* Map a segment into a contiguous dma address area */
static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
size_t size, dma_addr_t *handle,
enum dma_data_direction dir)
{
unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
dma_addr_t dma_addr_base, dma_addr;
int flags = ZPCI_PTE_VALID;
struct scatterlist *s;
phys_addr_t pa = 0;
int ret;
dma_addr_base = dma_alloc_address(dev, nr_pages);
if (dma_addr_base == DMA_MAPPING_ERROR)
return -ENOMEM;
dma_addr = dma_addr_base;
if (dir == DMA_NONE || dir == DMA_TO_DEVICE)
flags |= ZPCI_TABLE_PROTECTED;
for (s = sg; dma_addr < dma_addr_base + size; s = sg_next(s)) {
pa = page_to_phys(sg_page(s));
ret = __dma_update_trans(zdev, pa, dma_addr,
s->offset + s->length, flags);
if (ret)
goto unmap;
dma_addr += s->offset + s->length;
}
ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags);
if (ret)
goto unmap;
*handle = dma_addr_base;
atomic64_add(nr_pages, &zdev->mapped_pages);
return ret;
unmap:
dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base,
ZPCI_PTE_INVALID);
dma_free_address(dev, dma_addr_base, nr_pages);
zpci_err("map error:\n");
zpci_err_dma(ret, pa);
return ret;
}
static int s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
int nr_elements, enum dma_data_direction dir,
unsigned long attrs)
{
struct scatterlist *s = sg, *start = sg, *dma = sg;
unsigned int max = dma_get_max_seg_size(dev);
unsigned int size = s->offset + s->length;
unsigned int offset = s->offset;
int count = 0, i, ret;
for (i = 1; i < nr_elements; i++) {
s = sg_next(s);
s->dma_length = 0;
if (s->offset || (size & ~PAGE_MASK) ||
size + s->length > max) {
ret = __s390_dma_map_sg(dev, start, size,
&dma->dma_address, dir);
if (ret)
goto unmap;
dma->dma_address += offset;
dma->dma_length = size - offset;
size = offset = s->offset;
start = s;
dma = sg_next(dma);
count++;
}
size += s->length;
}
ret = __s390_dma_map_sg(dev, start, size, &dma->dma_address, dir);
if (ret)
goto unmap;
dma->dma_address += offset;
dma->dma_length = size - offset;
return count + 1;
unmap:
for_each_sg(sg, s, count, i)
s390_dma_unmap_pages(dev, sg_dma_address(s), sg_dma_len(s),
dir, attrs);
return ret;
}
static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int nr_elements, enum dma_data_direction dir,
unsigned long attrs)
{
struct scatterlist *s;
int i;
for_each_sg(sg, s, nr_elements, i) {
if (s->dma_length)
s390_dma_unmap_pages(dev, s->dma_address, s->dma_length,
dir, attrs);
s->dma_address = 0;
s->dma_length = 0;
}
}
static unsigned long *bitmap_vzalloc(size_t bits, gfp_t flags)
{
size_t n = BITS_TO_LONGS(bits);
size_t bytes;
if (unlikely(check_mul_overflow(n, sizeof(unsigned long), &bytes)))
return NULL;
return vzalloc(bytes);
}
int zpci_dma_init_device(struct zpci_dev *zdev)
{
u8 status;
int rc;
/*
* At this point, if the device is part of an IOMMU domain, this would
* be a strong hint towards a bug in the IOMMU API (common) code and/or
* simultaneous access via IOMMU and DMA API. So let's issue a warning.
*/
WARN_ON(zdev->s390_domain);
spin_lock_init(&zdev->iommu_bitmap_lock);
zdev->dma_table = dma_alloc_cpu_table(GFP_KERNEL);
if (!zdev->dma_table) {
rc = -ENOMEM;
goto out;
}
/*
* Restrict the iommu bitmap size to the minimum of the following:
* - s390_iommu_aperture which defaults to high_memory
* - 3-level pagetable address limit minus start_dma offset
* - DMA address range allowed by the hardware (clp query pci fn)
*
* Also set zdev->end_dma to the actual end address of the usable
* range, instead of the theoretical maximum as reported by hardware.
*
* This limits the number of concurrently usable DMA mappings since
* for each DMA mapped memory address we need a DMA address including
* extra DMA addresses for multiple mappings of the same memory address.
*/
zdev->start_dma = PAGE_ALIGN(zdev->start_dma);
zdev->iommu_size = min3(s390_iommu_aperture,
ZPCI_TABLE_SIZE_RT - zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
zdev->end_dma = zdev->start_dma + zdev->iommu_size - 1;
zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT;
zdev->iommu_bitmap = bitmap_vzalloc(zdev->iommu_pages, GFP_KERNEL);
if (!zdev->iommu_bitmap) {
rc = -ENOMEM;
goto free_dma_table;
}
if (!s390_iommu_strict) {
zdev->lazy_bitmap = bitmap_vzalloc(zdev->iommu_pages, GFP_KERNEL);
if (!zdev->lazy_bitmap) {
rc = -ENOMEM;
goto free_bitmap;
}
}
if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table), &status)) {
rc = -EIO;
goto free_bitmap;
}
return 0;
free_bitmap:
vfree(zdev->iommu_bitmap);
zdev->iommu_bitmap = NULL;
vfree(zdev->lazy_bitmap);
zdev->lazy_bitmap = NULL;
free_dma_table:
dma_free_cpu_table(zdev->dma_table);
zdev->dma_table = NULL;
out:
return rc;
}
int zpci_dma_exit_device(struct zpci_dev *zdev)
{
int cc = 0;
/*
* At this point, if the device is part of an IOMMU domain, this would
* be a strong hint towards a bug in the IOMMU API (common) code and/or
* simultaneous access via IOMMU and DMA API. So let's issue a warning.
*/
WARN_ON(zdev->s390_domain);
if (zdev_enabled(zdev))
cc = zpci_unregister_ioat(zdev, 0);
/*
* cc == 3 indicates the function is gone already. This can happen
* if the function was deconfigured/disabled suddenly and we have not
* received a new handle yet.
*/
if (cc && cc != 3)
return -EIO;
dma_cleanup_tables(zdev->dma_table);
zdev->dma_table = NULL;
vfree(zdev->iommu_bitmap);
zdev->iommu_bitmap = NULL;
vfree(zdev->lazy_bitmap);
zdev->lazy_bitmap = NULL;
zdev->next_bit = 0;
return 0;
}
static int __init dma_alloc_cpu_table_caches(void)
{
dma_region_table_cache = kmem_cache_create("PCI_DMA_region_tables",
ZPCI_TABLE_SIZE, ZPCI_TABLE_ALIGN,
0, NULL);
if (!dma_region_table_cache)
return -ENOMEM;
dma_page_table_cache = kmem_cache_create("PCI_DMA_page_tables",
ZPCI_PT_SIZE, ZPCI_PT_ALIGN,
0, NULL);
if (!dma_page_table_cache) {
kmem_cache_destroy(dma_region_table_cache);
return -ENOMEM;
}
return 0;
}
int __init zpci_dma_init(void)
{
s390_iommu_aperture = (u64)virt_to_phys(high_memory);
if (!s390_iommu_aperture_factor)
s390_iommu_aperture = ULONG_MAX;
else
s390_iommu_aperture *= s390_iommu_aperture_factor;
return dma_alloc_cpu_table_caches();
}
void zpci_dma_exit(void)
{
kmem_cache_destroy(dma_page_table_cache);
kmem_cache_destroy(dma_region_table_cache);
}
const struct dma_map_ops s390_pci_dma_ops = {
.alloc = s390_dma_alloc,
.free = s390_dma_free,
.map_sg = s390_dma_map_sg,
.unmap_sg = s390_dma_unmap_sg,
.map_page = s390_dma_map_pages,
.unmap_page = s390_dma_unmap_pages,
.mmap = dma_common_mmap,
.get_sgtable = dma_common_get_sgtable,
.alloc_pages = dma_common_alloc_pages,
.free_pages = dma_common_free_pages,
/* dma_supported is unconditionally true without a callback */
};
EXPORT_SYMBOL_GPL(s390_pci_dma_ops);
static int __init s390_iommu_setup(char *str)
{
if (!strcmp(str, "strict"))
s390_iommu_strict = 1;
return 1;
}
__setup("s390_iommu=", s390_iommu_setup);
static int __init s390_iommu_aperture_setup(char *str)
{
if (kstrtou32(str, 10, &s390_iommu_aperture_factor))
s390_iommu_aperture_factor = 1;
return 1;
}
__setup("s390_iommu_aperture=", s390_iommu_aperture_setup);

17
arch/s390/pci/pci_event.c
View File

@ -59,9 +59,16 @@ static inline bool ers_result_indicates_abort(pci_ers_result_t ers_res)
}
}
static bool is_passed_through(struct zpci_dev *zdev)
static bool is_passed_through(struct pci_dev *pdev)
{
return zdev->s390_domain;
struct zpci_dev *zdev = to_zpci(pdev);
bool ret;
mutex_lock(&zdev->kzdev_lock);
ret = !!zdev->kzdev;
mutex_unlock(&zdev->kzdev_lock);
return ret;
}
static bool is_driver_supported(struct pci_driver *driver)
@ -176,7 +183,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev)
}
pdev->error_state = pci_channel_io_frozen;
if (is_passed_through(to_zpci(pdev))) {
if (is_passed_through(pdev)) {
pr_info("%s: Cannot be recovered in the host because it is a pass-through device\n",
pci_name(pdev));
goto out_unlock;
@ -239,7 +246,7 @@ static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es)
* we will inject the error event and let the guest recover the device
* itself.
*/
if (is_passed_through(to_zpci(pdev)))
if (is_passed_through(pdev))
goto out;
driver = to_pci_driver(pdev->dev.driver);
if (driver && driver->err_handler && driver->err_handler->error_detected)
@ -306,8 +313,6 @@ static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh)
/* Even though the device is already gone we still
* need to free zPCI resources as part of the disable.
*/
if (zdev->dma_table)
zpci_dma_exit_device(zdev);
if (zdev_enabled(zdev))
zpci_disable_device(zdev);
zdev->state = ZPCI_FN_STATE_STANDBY;

19
arch/s390/pci/pci_sysfs.c
View File

@ -56,6 +56,7 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr,
struct pci_dev *pdev = to_pci_dev(dev);
struct zpci_dev *zdev = to_zpci(pdev);
int ret = 0;
u8 status;
/* Can't use device_remove_self() here as that would lead us to lock
* the pci_rescan_remove_lock while holding the device' kernfs lock.
@ -82,12 +83,6 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr,
pci_lock_rescan_remove();
if (pci_dev_is_added(pdev)) {
pci_stop_and_remove_bus_device(pdev);
if (zdev->dma_table) {
ret = zpci_dma_exit_device(zdev);
if (ret)
goto out;
}
if (zdev_enabled(zdev)) {
ret = zpci_disable_device(zdev);
/*
@ -105,14 +100,16 @@ static ssize_t recover_store(struct device *dev, struct device_attribute *attr,
ret = zpci_enable_device(zdev);
if (ret)
goto out;
ret = zpci_dma_init_device(zdev);
if (ret) {
zpci_disable_device(zdev);
goto out;
if (zdev->dma_table) {
ret = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table), &status);
if (ret)
zpci_disable_device(zdev);
}
pci_rescan_bus(zdev->zbus->bus);
}
out:
pci_rescan_bus(zdev->zbus->bus);
pci_unlock_rescan_remove();
if (kn)
sysfs_unbreak_active_protection(kn);

15
drivers/iommu/Kconfig
View File

@ -95,7 +95,7 @@ config IOMMU_DEBUGFS
choice
prompt "IOMMU default domain type"
depends on IOMMU_API
default IOMMU_DEFAULT_DMA_LAZY if X86
default IOMMU_DEFAULT_DMA_LAZY if X86 || S390
default IOMMU_DEFAULT_DMA_STRICT
help
Choose the type of IOMMU domain used to manage DMA API usage by
@ -150,7 +150,7 @@ config OF_IOMMU
# IOMMU-agnostic DMA-mapping layer
config IOMMU_DMA
def_bool ARM64 || X86
def_bool ARM64 || X86 || S390
select DMA_OPS
select IOMMU_API
select IOMMU_IOVA
@ -240,17 +240,6 @@ config SUN50I_IOMMU
help
Support for the IOMMU introduced in the Allwinner H6 SoCs.
config TEGRA_IOMMU_GART
bool "Tegra GART IOMMU Support"
depends on ARCH_TEGRA_2x_SOC
depends on TEGRA_MC
select IOMMU_API
help
Enables support for remapping discontiguous physical memory
shared with the operating system into contiguous I/O virtual
space through the GART (Graphics Address Relocation Table)
hardware included on Tegra SoCs.
config TEGRA_IOMMU_SMMU
bool "NVIDIA Tegra SMMU Support"
depends on ARCH_TEGRA

1
drivers/iommu/Makefile
View File

@ -20,7 +20,6 @@ obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
obj-$(CONFIG_SUN50I_IOMMU) += sun50i-iommu.o
obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o

9
drivers/iommu/amd/Kconfig
View File

@ -23,15 +23,6 @@ config AMD_IOMMU
your BIOS for an option to enable it or if you have an IVRS ACPI
table.
config AMD_IOMMU_V2
tristate "AMD IOMMU Version 2 driver"
depends on AMD_IOMMU
select MMU_NOTIFIER
help
This option enables support for the AMD IOMMUv2 features of the IOMMU
hardware. Select this option if you want to use devices that support
the PCI PRI and PASID interface.
config AMD_IOMMU_DEBUGFS
bool "Enable AMD IOMMU internals in DebugFS"
depends on AMD_IOMMU && IOMMU_DEBUGFS

1
drivers/iommu/amd/Makefile
View File

@ -1,4 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o

35
drivers/iommu/amd/amd_iommu.h
View File

@ -38,9 +38,6 @@ extern int amd_iommu_guest_ir;
extern enum io_pgtable_fmt amd_iommu_pgtable;
extern int amd_iommu_gpt_level;
/* IOMMUv2 specific functions */
struct iommu_domain;
bool amd_iommu_v2_supported(void);
struct amd_iommu *get_amd_iommu(unsigned int idx);
u8 amd_iommu_pc_get_max_banks(unsigned int idx);
@ -51,10 +48,10 @@ int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
u8 fxn, u64 *value);
int amd_iommu_register_ppr_notifier(struct notifier_block *nb);
int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb);
void amd_iommu_domain_direct_map(struct iommu_domain *dom);
int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
/* Device capabilities */
int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev);
void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev);
int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address);
void amd_iommu_update_and_flush_device_table(struct protection_domain *domain);
void amd_iommu_domain_update(struct protection_domain *domain);
@ -87,9 +84,25 @@ static inline bool is_rd890_iommu(struct pci_dev *pdev)
(pdev->device == PCI_DEVICE_ID_RD890_IOMMU);
}
static inline bool iommu_feature(struct amd_iommu *iommu, u64 mask)
static inline bool check_feature(u64 mask)
{
return !!(iommu->features & mask);
return (amd_iommu_efr & mask);
}
static inline bool check_feature2(u64 mask)
{
return (amd_iommu_efr2 & mask);
}
static inline int check_feature_gpt_level(void)
{
return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK);
}
static inline bool amd_iommu_gt_ppr_supported(void)
{
return (check_feature(FEATURE_GT) &&
check_feature(FEATURE_PPR));
}
static inline u64 iommu_virt_to_phys(void *vaddr)
@ -105,7 +118,6 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
static inline
void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
{
atomic64_set(&domain->iop.pt_root, root);
domain->iop.root = (u64 *)(root & PAGE_MASK);
domain->iop.mode = root & 7; /* lowest 3 bits encode pgtable mode */
}
@ -146,8 +158,5 @@ void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode);
struct dev_table_entry *get_dev_table(struct amd_iommu *iommu);
extern u64 amd_iommu_efr;
extern u64 amd_iommu_efr2;
extern bool amd_iommu_snp_en;
#endif

52
drivers/iommu/amd/amd_iommu_types.h
View File

@ -462,6 +462,10 @@
#define PD_IOMMUV2_MASK BIT(3) /* domain has gcr3 table */
#define PD_GIOV_MASK BIT(4) /* domain enable GIOV support */
/* Timeout stuff */
#define LOOP_TIMEOUT 100000
#define MMIO_STATUS_TIMEOUT 2000000
extern bool amd_iommu_dump;
#define DUMP_printk(format, arg...) \
do { \
@ -516,19 +520,6 @@ extern struct kmem_cache *amd_iommu_irq_cache;
#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT)
#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL)
/*
* This struct is used to pass information about
* incoming PPR faults around.
*/
struct amd_iommu_fault {
u64 address; /* IO virtual address of the fault*/
u32 pasid; /* Address space identifier */
u32 sbdf; /* Originating PCI device id */
u16 tag; /* PPR tag */
u16 flags; /* Fault flags */
};
struct amd_iommu;
struct iommu_domain;
@ -555,7 +546,6 @@ struct amd_io_pgtable {
struct io_pgtable iop;
int mode;
u64 *root;
atomic64_t pt_root; /* pgtable root and pgtable mode */
u64 *pgd; /* v2 pgtable pgd pointer */
};
@ -688,9 +678,6 @@ struct amd_iommu {
/* Extended features 2 */
u64 features2;
/* IOMMUv2 */
bool is_iommu_v2;
/* PCI device id of the IOMMU device */
u16 devid;
@ -811,6 +798,14 @@ struct devid_map {
bool cmd_line;
};
#define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
#define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
#define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
/* Device may request execution on memory pages */
#define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
/* Device may request super-user privileges */
#define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
/*
* This struct contains device specific data for the IOMMU
*/
@ -823,13 +818,15 @@ struct iommu_dev_data {
struct protection_domain *domain; /* Domain the device is bound to */
struct device *dev;
u16 devid; /* PCI Device ID */
bool iommu_v2; /* Device can make use of IOMMUv2 */
struct {
bool enabled;
int qdep;
} ats; /* ATS state */
bool pri_tlp; /* PASID TLB required for
u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
int ats_qdep;
u8 ats_enabled :1; /* ATS state */
u8 pri_enabled :1; /* PRI state */
u8 pasid_enabled:1; /* PASID state */
u8 pri_tlp :1; /* PASID TLB required for
PPR completions */
u8 ppr :1; /* Enable device PPR support */
bool use_vapic; /* Enable device to use vapic mode */
bool defer_attach;
@ -896,16 +893,15 @@ extern unsigned amd_iommu_aperture_order;
/* allocation bitmap for domain ids */
extern unsigned long *amd_iommu_pd_alloc_bitmap;
/* Smallest max PASID supported by any IOMMU in the system */
extern u32 amd_iommu_max_pasid;
extern bool amd_iommu_v2_present;
extern bool amd_iommu_force_isolation;
/* Max levels of glxval supported */
extern int amd_iommu_max_glx_val;
/* Global EFR and EFR2 registers */
extern u64 amd_iommu_efr;
extern u64 amd_iommu_efr2;
/*
* This function flushes all internal caches of
* the IOMMU used by this driver.

121
drivers/iommu/amd/init.c
View File

@ -83,8 +83,6 @@
#define ACPI_DEVFLAG_LINT1 0x80
#define ACPI_DEVFLAG_ATSDIS 0x10000000
#define LOOP_TIMEOUT 2000000
#define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
| ((dev & 0x1f) << 3) | (fn & 0x7))
@ -187,9 +185,6 @@ static int amd_iommus_present;
bool amd_iommu_np_cache __read_mostly;
bool amd_iommu_iotlb_sup __read_mostly = true;
u32 amd_iommu_max_pasid __read_mostly = ~0;
bool amd_iommu_v2_present __read_mostly;
static bool amd_iommu_pc_present __read_mostly;
bool amdr_ivrs_remap_support __read_mostly;
@ -272,7 +267,7 @@ int amd_iommu_get_num_iommus(void)
* Iterate through all the IOMMUs to get common EFR
* masks among all IOMMUs and warn if found inconsistency.
*/
static void get_global_efr(void)
static __init void get_global_efr(void)
{
struct amd_iommu *iommu;
@ -304,16 +299,6 @@ static void get_global_efr(void)
pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
}
static bool check_feature_on_all_iommus(u64 mask)
{
return !!(amd_iommu_efr & mask);
}
static inline int check_feature_gpt_level(void)
{
return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK);
}
/*
* For IVHD type 0x11/0x40, EFR is also available via IVHD.
* Default to IVHD EFR since it is available sooner
@ -399,7 +384,7 @@ static void iommu_set_cwwb_range(struct amd_iommu *iommu)
u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
u64 entry = start & PM_ADDR_MASK;
if (!check_feature_on_all_iommus(FEATURE_SNP))
if (!check_feature(FEATURE_SNP))
return;
/* Note:
@ -869,7 +854,7 @@ static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,
void *buf = (void *)__get_free_pages(gfp, order);
if (buf &&
check_feature_on_all_iommus(FEATURE_SNP) &&
check_feature(FEATURE_SNP) &&
set_memory_4k((unsigned long)buf, (1 << order))) {
free_pages((unsigned long)buf, order);
buf = NULL;
@ -985,14 +970,14 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
iommu_feature_enable(iommu, CONTROL_GAINT_EN);
iommu_feature_enable(iommu, CONTROL_GALOG_EN);
for (i = 0; i < LOOP_TIMEOUT; ++i) {
for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
if (status & (MMIO_STATUS_GALOG_RUN_MASK))
break;
udelay(10);
}
if (WARN_ON(i >= LOOP_TIMEOUT))
if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
return -EINVAL;
return 0;
@ -1048,7 +1033,7 @@ static void iommu_enable_xt(struct amd_iommu *iommu)
static void iommu_enable_gt(struct amd_iommu *iommu)
{
if (!iommu_feature(iommu, FEATURE_GT))
if (!check_feature(FEATURE_GT))
return;
iommu_feature_enable(iommu, CONTROL_GT_EN);
@ -1987,7 +1972,7 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu)
u64 val;
struct pci_dev *pdev = iommu->dev;
if (!iommu_feature(iommu, FEATURE_PC))
if (!check_feature(FEATURE_PC))
return;
amd_iommu_pc_present = true;
@ -2014,8 +1999,7 @@ static ssize_t amd_iommu_show_features(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct amd_iommu *iommu = dev_to_amd_iommu(dev);
return sysfs_emit(buf, "%llx:%llx\n", iommu->features2, iommu->features);
return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2);
}
static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
@ -2051,9 +2035,9 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu)
features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
if (!iommu->features) {
iommu->features = features;
iommu->features2 = features2;
if (!amd_iommu_efr) {
amd_iommu_efr = features;
amd_iommu_efr2 = features2;
return;
}
@ -2061,12 +2045,12 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu)
* Sanity check and warn if EFR values from
* IVHD and MMIO conflict.
*/
if (features != iommu->features ||
features2 != iommu->features2) {
if (features != amd_iommu_efr ||
features2 != amd_iommu_efr2) {
pr_warn(FW_WARN
"EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
features, iommu->features,
features2, iommu->features2);
features, amd_iommu_efr,
features2, amd_iommu_efr2);
}
}
@ -2092,20 +2076,17 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
late_iommu_features_init(iommu);
if (iommu_feature(iommu, FEATURE_GT)) {
if (check_feature(FEATURE_GT)) {
int glxval;
u32 max_pasid;
u64 pasmax;
pasmax = iommu->features & FEATURE_PASID_MASK;
pasmax = amd_iommu_efr & FEATURE_PASID_MASK;
pasmax >>= FEATURE_PASID_SHIFT;
max_pasid = (1 << (pasmax + 1)) - 1;
iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1;
amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK);
BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
glxval = iommu->features & FEATURE_GLXVAL_MASK;
glxval = amd_iommu_efr & FEATURE_GLXVAL_MASK;
glxval >>= FEATURE_GLXVAL_SHIFT;
if (amd_iommu_max_glx_val == -1)
@ -2114,13 +2095,7 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
}
if (iommu_feature(iommu, FEATURE_GT) &&
iommu_feature(iommu, FEATURE_PPR)) {
iommu->is_iommu_v2 = true;
amd_iommu_v2_present = true;
}
if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu))
if (check_feature(FEATURE_PPR) && alloc_ppr_log(iommu))
return -ENOMEM;
if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
@ -2132,13 +2107,10 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
init_iommu_perf_ctr(iommu);
if (amd_iommu_pgtable == AMD_IOMMU_V2) {
if (!iommu_feature(iommu, FEATURE_GIOSUP) ||
!iommu_feature(iommu, FEATURE_GT)) {
if (!check_feature(FEATURE_GIOSUP) ||
!check_feature(FEATURE_GT)) {
pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
amd_iommu_pgtable = AMD_IOMMU_V1;
} else if (iommu_default_passthrough()) {
pr_warn("V2 page table doesn't support passthrough mode. Fallback to v1.\n");
amd_iommu_pgtable = AMD_IOMMU_V1;
}
}
@ -2186,35 +2158,29 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
static void print_iommu_info(void)
{
int i;
static const char * const feat_str[] = {
"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
"IA", "GA", "HE", "PC"
};
struct amd_iommu *iommu;
for_each_iommu(iommu) {
struct pci_dev *pdev = iommu->dev;
int i;
if (amd_iommu_efr) {
pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2);
pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr);
if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
pr_info("Extended features (%#llx, %#llx):", iommu->features, iommu->features2);
for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
if (iommu_feature(iommu, (1ULL << i)))
pr_cont(" %s", feat_str[i]);
}
if (iommu->features & FEATURE_GAM_VAPIC)
pr_cont(" GA_vAPIC");
if (iommu->features & FEATURE_SNP)
pr_cont(" SNP");
pr_cont("\n");
for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
if (check_feature(1ULL << i))
pr_cont(" %s", feat_str[i]);
}
if (check_feature(FEATURE_GAM_VAPIC))
pr_cont(" GA_vAPIC");
if (check_feature(FEATURE_SNP))
pr_cont(" SNP");
pr_cont("\n");
}
if (irq_remapping_enabled) {
pr_info("Interrupt remapping enabled\n");
if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
@ -2900,19 +2866,19 @@ static void enable_iommus_vapic(void)
* Need to set and poll check the GALOGRun bit to zero before
* we can set/ modify GA Log registers safely.
*/
for (i = 0; i < LOOP_TIMEOUT; ++i) {
for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) {
status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
break;
udelay(10);
}
if (WARN_ON(i >= LOOP_TIMEOUT))
if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
return;
}
if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
!check_feature_on_all_iommus(FEATURE_GAM_VAPIC)) {
!check_feature(FEATURE_GAM_VAPIC)) {
amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
return;
}
@ -3698,9 +3664,8 @@ bool amd_iommu_v2_supported(void)
* (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
* setting up IOMMUv1 page table.
*/
return amd_iommu_v2_present && !amd_iommu_snp_en;
return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en;
}
EXPORT_SYMBOL(amd_iommu_v2_supported);
struct amd_iommu *get_amd_iommu(unsigned int idx)
{
@ -3824,7 +3789,7 @@ int amd_iommu_snp_enable(void)
return -EINVAL;
}
amd_iommu_snp_en = check_feature_on_all_iommus(FEATURE_SNP);
amd_iommu_snp_en = check_feature(FEATURE_SNP);
if (!amd_iommu_snp_en)
return -EINVAL;

8
drivers/iommu/amd/io_pgtable_v2.c
View File

@ -363,10 +363,10 @@ static void v2_free_pgtable(struct io_pgtable *iop)
if (!(pdom->flags & PD_IOMMUV2_MASK))
return;
/*
* Make changes visible to IOMMUs. No need to clear gcr3 entry
* as gcr3 table is already freed.
*/
/* Clear gcr3 entry */
amd_iommu_domain_clear_gcr3(&pdom->domain, 0);
/* Make changes visible to IOMMUs */
amd_iommu_domain_update(pdom);
/* Free page table */

581
drivers/iommu/amd/iommu.c
View File

@ -45,8 +45,6 @@
#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
#define LOOP_TIMEOUT 100000
/* IO virtual address start page frame number */
#define IOVA_START_PFN (1)
#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
@ -68,7 +66,6 @@ LIST_HEAD(acpihid_map);
const struct iommu_ops amd_iommu_ops;
const struct iommu_dirty_ops amd_dirty_ops;
static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
int amd_iommu_max_glx_val = -1;
/*
@ -81,7 +78,6 @@ struct iommu_cmd {
struct kmem_cache *amd_iommu_irq_cache;
static void detach_device(struct device *dev);
static int domain_enable_v2(struct protection_domain *domain, int pasids);
/****************************************************************************
*
@ -324,24 +320,141 @@ static struct iommu_group *acpihid_device_group(struct device *dev)
return entry->group;
}
static bool pci_iommuv2_capable(struct pci_dev *pdev)
static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data)
{
static const int caps[] = {
PCI_EXT_CAP_ID_PRI,
PCI_EXT_CAP_ID_PASID,
};
int i, pos;
return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP);
}
if (!pci_ats_supported(pdev))
return false;
static u32 pdev_get_caps(struct pci_dev *pdev)
{
int features;
u32 flags = 0;
for (i = 0; i < 2; ++i) {
pos = pci_find_ext_capability(pdev, caps[i]);
if (pos == 0)
return false;
if (pci_ats_supported(pdev))
flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
if (pci_pri_supported(pdev))
flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
features = pci_pasid_features(pdev);
if (features >= 0) {
flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
if (features & PCI_PASID_CAP_EXEC)
flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
if (features & PCI_PASID_CAP_PRIV)
flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
}
return true;
return flags;
}
static inline int pdev_enable_cap_ats(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
int ret = -EINVAL;
if (dev_data->ats_enabled)
return 0;
if (amd_iommu_iotlb_sup &&
(dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {
ret = pci_enable_ats(pdev, PAGE_SHIFT);
if (!ret) {
dev_data->ats_enabled = 1;
dev_data->ats_qdep = pci_ats_queue_depth(pdev);
}
}
return ret;
}
static inline void pdev_disable_cap_ats(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
if (dev_data->ats_enabled) {
pci_disable_ats(pdev);
dev_data->ats_enabled = 0;
}
}
int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
int ret = -EINVAL;
if (dev_data->pri_enabled)
return 0;
if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) {
/*
* First reset the PRI state of the device.
* FIXME: Hardcode number of outstanding requests for now
*/
if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) {
dev_data->pri_enabled = 1;
dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);
ret = 0;
}
}
return ret;
}
void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
if (dev_data->pri_enabled) {
pci_disable_pri(pdev);
dev_data->pri_enabled = 0;
}
}
static inline int pdev_enable_cap_pasid(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
int ret = -EINVAL;
if (dev_data->pasid_enabled)
return 0;
if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) {
/* Only allow access to user-accessible pages */
ret = pci_enable_pasid(pdev, 0);
if (!ret)
dev_data->pasid_enabled = 1;
}
return ret;
}
static inline void pdev_disable_cap_pasid(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
if (dev_data->pasid_enabled) {
pci_disable_pasid(pdev);
dev_data->pasid_enabled = 0;
}
}
static void pdev_enable_caps(struct pci_dev *pdev)
{
pdev_enable_cap_ats(pdev);
pdev_enable_cap_pasid(pdev);
amd_iommu_pdev_enable_cap_pri(pdev);
}
static void pdev_disable_caps(struct pci_dev *pdev)
{
pdev_disable_cap_ats(pdev);
pdev_disable_cap_pasid(pdev);
amd_iommu_pdev_disable_cap_pri(pdev);
}
/*
@ -401,8 +514,8 @@ static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
* it'll be forced to go into translation mode.
*/
if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
dev_data->iommu_v2 = iommu->is_iommu_v2;
dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
dev_data->flags = pdev_get_caps(to_pci_dev(dev));
}
dev_iommu_priv_set(dev, dev_data);
@ -703,24 +816,6 @@ static void iommu_poll_events(struct amd_iommu *iommu)
writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
}
static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw)
{
struct amd_iommu_fault fault;
if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
pr_err_ratelimited("Unknown PPR request received\n");
return;
}
fault.address = raw[1];
fault.pasid = PPR_PASID(raw[0]);
fault.sbdf = PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, PPR_DEVID(raw[0]));
fault.tag = PPR_TAG(raw[0]);
fault.flags = PPR_FLAGS(raw[0]);
atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
}
static void iommu_poll_ppr_log(struct amd_iommu *iommu)
{
u32 head, tail;
@ -766,8 +861,7 @@ static void iommu_poll_ppr_log(struct amd_iommu *iommu)
head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
/* Handle PPR entry */
iommu_handle_ppr_entry(iommu, entry);
/* TODO: PPR Handler will be added when we add IOPF support */
/* Refresh ring-buffer information */
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
@ -1096,7 +1190,7 @@ static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, u32 pasid,
}
static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
int status, int tag, bool gn)
int status, int tag, u8 gn)
{
memset(cmd, 0, sizeof(*cmd));
@ -1300,7 +1394,7 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
if (iommu_feature(iommu, FEATURE_IA)) {
if (check_feature(FEATURE_IA)) {
amd_iommu_flush_all(iommu);
} else {
amd_iommu_flush_dte_all(iommu);
@ -1319,7 +1413,7 @@ static int device_flush_iotlb(struct iommu_dev_data *dev_data,
struct iommu_cmd cmd;
int qdep;
qdep = dev_data->ats.qdep;
qdep = dev_data->ats_qdep;
iommu = rlookup_amd_iommu(dev_data->dev);
if (!iommu)
return -EINVAL;
@ -1370,7 +1464,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data)
return ret;
}
if (dev_data->ats.enabled)
if (dev_data->ats_enabled)
ret = device_flush_iotlb(dev_data, 0, ~0UL);
return ret;
@ -1403,7 +1497,7 @@ static void __domain_flush_pages(struct protection_domain *domain,
list_for_each_entry(dev_data, &domain->dev_list, list) {
if (!dev_data->ats.enabled)
if (!dev_data->ats_enabled)
continue;
ret |= device_flush_iotlb(dev_data, address, size);
@ -1579,6 +1673,42 @@ static void free_gcr3_table(struct protection_domain *domain)
free_page((unsigned long)domain->gcr3_tbl);
}
/*
* Number of GCR3 table levels required. Level must be 4-Kbyte
* page and can contain up to 512 entries.
*/
static int get_gcr3_levels(int pasids)
{
int levels;
if (pasids == -1)
return amd_iommu_max_glx_val;
levels = get_count_order(pasids);
return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels;
}
/* Note: This function expects iommu_domain->lock to be held prior calling the function. */
static int setup_gcr3_table(struct protection_domain *domain, int pasids)
{
int levels = get_gcr3_levels(pasids);
if (levels > amd_iommu_max_glx_val)
return -EINVAL;
domain->gcr3_tbl = alloc_pgtable_page(domain->nid, GFP_ATOMIC);
if (domain->gcr3_tbl == NULL)
return -ENOMEM;
domain->glx = levels;
domain->flags |= PD_IOMMUV2_MASK;
amd_iommu_domain_update(domain);
return 0;
}
static void set_dte_entry(struct amd_iommu *iommu, u16 devid,
struct protection_domain *domain, bool ats, bool ppr)
{
@ -1607,10 +1737,8 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid,
if (ats)
flags |= DTE_FLAG_IOTLB;
if (ppr) {
if (iommu_feature(iommu, FEATURE_EPHSUP))
pte_root |= 1ULL << DEV_ENTRY_PPR;
}
if (ppr)
pte_root |= 1ULL << DEV_ENTRY_PPR;
if (domain->dirty_tracking)
pte_root |= DTE_FLAG_HAD;
@ -1690,7 +1818,7 @@ static void do_attach(struct iommu_dev_data *dev_data,
iommu = rlookup_amd_iommu(dev_data->dev);
if (!iommu)
return;
ats = dev_data->ats.enabled;
ats = dev_data->ats_enabled;
/* Update data structures */
dev_data->domain = domain;
@ -1706,7 +1834,7 @@ static void do_attach(struct iommu_dev_data *dev_data,
/* Update device table */
set_dte_entry(iommu, dev_data->devid, domain,
ats, dev_data->iommu_v2);
ats, dev_data->ppr);
clone_aliases(iommu, dev_data->dev);
device_flush_dte(dev_data);
@ -1741,48 +1869,6 @@ static void do_detach(struct iommu_dev_data *dev_data)
domain->dev_cnt -= 1;
}
static void pdev_iommuv2_disable(struct pci_dev *pdev)
{
pci_disable_ats(pdev);
pci_disable_pri(pdev);
pci_disable_pasid(pdev);
}
static int pdev_pri_ats_enable(struct pci_dev *pdev)
{
int ret;
/* Only allow access to user-accessible pages */
ret = pci_enable_pasid(pdev, 0);
if (ret)
return ret;
/* First reset the PRI state of the device */
ret = pci_reset_pri(pdev);
if (ret)
goto out_err_pasid;
/* Enable PRI */
/* FIXME: Hardcode number of outstanding requests for now */
ret = pci_enable_pri(pdev, 32);
if (ret)
goto out_err_pasid;
ret = pci_enable_ats(pdev, PAGE_SHIFT);
if (ret)
goto out_err_pri;
return 0;
out_err_pri:
pci_disable_pri(pdev);
out_err_pasid:
pci_disable_pasid(pdev);
return ret;
}
/*
* If a device is not yet associated with a domain, this function makes the
* device visible in the domain
@ -1791,9 +1877,8 @@ static int attach_device(struct device *dev,
struct protection_domain *domain)
{
struct iommu_dev_data *dev_data;
struct pci_dev *pdev;
unsigned long flags;
int ret;
int ret = 0;
spin_lock_irqsave(&domain->lock, flags);
@ -1801,45 +1886,13 @@ static int attach_device(struct device *dev,
spin_lock(&dev_data->lock);
ret = -EBUSY;
if (dev_data->domain != NULL)
if (dev_data->domain != NULL) {
ret = -EBUSY;
goto out;
if (!dev_is_pci(dev))
goto skip_ats_check;
pdev = to_pci_dev(dev);
if (domain->flags & PD_IOMMUV2_MASK) {
struct iommu_domain *def_domain = iommu_get_dma_domain(dev);
ret = -EINVAL;
/*
* In case of using AMD_IOMMU_V1 page table mode and the device
* is enabling for PPR/ATS support (using v2 table),
* we need to make sure that the domain type is identity map.
*/
if ((amd_iommu_pgtable == AMD_IOMMU_V1) &&
def_domain->type != IOMMU_DOMAIN_IDENTITY) {
goto out;
}
if (dev_data->iommu_v2) {
if (pdev_pri_ats_enable(pdev) != 0)
goto out;
dev_data->ats.enabled = true;
dev_data->ats.qdep = pci_ats_queue_depth(pdev);
dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);
}
} else if (amd_iommu_iotlb_sup &&
pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
dev_data->ats.enabled = true;
dev_data->ats.qdep = pci_ats_queue_depth(pdev);
}
skip_ats_check:
ret = 0;
if (dev_is_pci(dev))
pdev_enable_caps(to_pci_dev(dev));
do_attach(dev_data, domain);
@ -1887,15 +1940,8 @@ static void detach_device(struct device *dev)
do_detach(dev_data);
if (!dev_is_pci(dev))
goto out;
if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
pdev_iommuv2_disable(to_pci_dev(dev));
else if (dev_data->ats.enabled)
pci_disable_ats(to_pci_dev(dev));
dev_data->ats.enabled = false;
if (dev_is_pci(dev))
pdev_disable_caps(to_pci_dev(dev));
out:
spin_unlock(&dev_data->lock);
@ -1985,7 +2031,7 @@ static void update_device_table(struct protection_domain *domain)
if (!iommu)
continue;
set_dte_entry(iommu, dev_data->devid, domain,
dev_data->ats.enabled, dev_data->iommu_v2);
dev_data->ats_enabled, dev_data->ppr);
clone_aliases(iommu, dev_data->dev);
}
}
@ -2019,9 +2065,11 @@ void amd_iommu_domain_update(struct protection_domain *domain)
static void cleanup_domain(struct protection_domain *domain)
{
struct iommu_dev_data *entry;
unsigned long flags;
spin_lock_irqsave(&domain->lock, flags);
lockdep_assert_held(&domain->lock);
if (!domain->dev_cnt)
return;
while (!list_empty(&domain->dev_list)) {
entry = list_first_entry(&domain->dev_list,
@ -2029,8 +2077,7 @@ static void cleanup_domain(struct protection_domain *domain)
BUG_ON(!entry->domain);
do_detach(entry);
}
spin_unlock_irqrestore(&domain->lock, flags);
WARN_ON(domain->dev_cnt != 0);
}
static void protection_domain_free(struct protection_domain *domain)
@ -2041,6 +2088,12 @@ static void protection_domain_free(struct protection_domain *domain)
if (domain->iop.pgtbl_cfg.tlb)
free_io_pgtable_ops(&domain->iop.iop.ops);
if (domain->flags & PD_IOMMUV2_MASK)
free_gcr3_table(domain);
if (domain->iop.root)
free_page((unsigned long)domain->iop.root);
if (domain->id)
domain_id_free(domain->id);
@ -2053,18 +2106,10 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode)
BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL);
spin_lock_init(&domain->lock);
domain->id = domain_id_alloc();
if (!domain->id)
return -ENOMEM;
INIT_LIST_HEAD(&domain->dev_list);
if (mode != PAGE_MODE_NONE) {
pt_root = (void *)get_zeroed_page(GFP_KERNEL);
if (!pt_root) {
domain_id_free(domain->id);
if (!pt_root)
return -ENOMEM;
}
}
amd_iommu_domain_set_pgtable(domain, pt_root, mode);
@ -2074,20 +2119,12 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode)
static int protection_domain_init_v2(struct protection_domain *domain)
{
spin_lock_init(&domain->lock);
domain->id = domain_id_alloc();
if (!domain->id)
return -ENOMEM;
INIT_LIST_HEAD(&domain->dev_list);
domain->flags |= PD_GIOV_MASK;
domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2;
if (domain_enable_v2(domain, 1)) {
domain_id_free(domain->id);
if (setup_gcr3_table(domain, 1))
return -ENOMEM;
}
return 0;
}
@ -2097,57 +2134,60 @@ static struct protection_domain *protection_domain_alloc(unsigned int type)
struct io_pgtable_ops *pgtbl_ops;
struct protection_domain *domain;
int pgtable;
int mode = DEFAULT_PGTABLE_LEVEL;
int ret;
/*
* Force IOMMU v1 page table when iommu=pt and
* when allocating domain for pass-through devices.
*/
if (type == IOMMU_DOMAIN_IDENTITY) {
pgtable = AMD_IOMMU_V1;
mode = PAGE_MODE_NONE;
} else if (type == IOMMU_DOMAIN_UNMANAGED) {
pgtable = AMD_IOMMU_V1;
} else if (type == IOMMU_DOMAIN_DMA || type == IOMMU_DOMAIN_DMA_FQ) {
pgtable = amd_iommu_pgtable;
} else {
return NULL;
}
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
domain->id = domain_id_alloc();
if (!domain->id)
goto out_err;
spin_lock_init(&domain->lock);
INIT_LIST_HEAD(&domain->dev_list);
domain->nid = NUMA_NO_NODE;
switch (type) {
/* No need to allocate io pgtable ops in passthrough mode */
case IOMMU_DOMAIN_IDENTITY:
return domain;
case IOMMU_DOMAIN_DMA:
pgtable = amd_iommu_pgtable;
break;
/*
* Force IOMMU v1 page table when allocating
* domain for pass-through devices.
*/
case IOMMU_DOMAIN_UNMANAGED:
pgtable = AMD_IOMMU_V1;
break;
default:
goto out_err;
}
switch (pgtable) {
case AMD_IOMMU_V1:
ret = protection_domain_init_v1(domain, mode);
ret = protection_domain_init_v1(domain, DEFAULT_PGTABLE_LEVEL);
break;
case AMD_IOMMU_V2:
ret = protection_domain_init_v2(domain);
break;
default:
ret = -EINVAL;
break;
}
if (ret)
goto out_err;
/* No need to allocate io pgtable ops in passthrough mode */
if (type == IOMMU_DOMAIN_IDENTITY)
return domain;
domain->nid = NUMA_NO_NODE;
pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain);
if (!pgtbl_ops) {
domain_id_free(domain->id);
if (!pgtbl_ops)
goto out_err;
}
return domain;
out_err:
kfree(domain);
protection_domain_free(domain);
return NULL;
}
@ -2236,19 +2276,18 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags,
static void amd_iommu_domain_free(struct iommu_domain *dom)
{
struct protection_domain *domain;
domain = to_pdomain(dom);
if (domain->dev_cnt > 0)
cleanup_domain(domain);
BUG_ON(domain->dev_cnt != 0);
unsigned long flags;
if (!dom)
return;
if (domain->flags & PD_IOMMUV2_MASK)
free_gcr3_table(domain);
domain = to_pdomain(dom);
spin_lock_irqsave(&domain->lock, flags);
cleanup_domain(domain);
spin_unlock_irqrestore(&domain->lock, flags);
protection_domain_free(domain);
}
@ -2296,14 +2335,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
return ret;
}
static void amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
unsigned long iova, size_t size)
static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
unsigned long iova, size_t size)
{
struct protection_domain *domain = to_pdomain(dom);
struct io_pgtable_ops *ops = &domain->iop.iop.ops;
if (ops->map_pages)
domain_flush_np_cache(domain, iova, size);
return 0;
}
static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,
@ -2541,7 +2581,6 @@ bool amd_iommu_is_attach_deferred(struct device *dev)
return dev_data->defer_attach;
}
EXPORT_SYMBOL_GPL(amd_iommu_is_attach_deferred);
static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
@ -2581,7 +2620,7 @@ static int amd_iommu_def_domain_type(struct device *dev)
* and require remapping.
* - SNP is enabled, because it prohibits DTE[Mode]=0.
*/
if (dev_data->iommu_v2 &&
if (pdev_pasid_supported(dev_data) &&
!cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
!amd_iommu_snp_en) {
return IOMMU_DOMAIN_IDENTITY;
@ -2626,93 +2665,6 @@ const struct iommu_ops amd_iommu_ops = {
}
};
/*****************************************************************************
*
* The next functions do a basic initialization of IOMMU for pass through
* mode
*
* In passthrough mode the IOMMU is initialized and enabled but not used for
* DMA-API translation.
*
*****************************************************************************/
/* IOMMUv2 specific functions */
int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_register(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&ppr_notifier, nb);
}
EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
void amd_iommu_domain_direct_map(struct iommu_domain *dom)
{
struct protection_domain *domain = to_pdomain(dom);
unsigned long flags;
spin_lock_irqsave(&domain->lock, flags);
if (domain->iop.pgtbl_cfg.tlb)
free_io_pgtable_ops(&domain->iop.iop.ops);
spin_unlock_irqrestore(&domain->lock, flags);
}
EXPORT_SYMBOL(amd_iommu_domain_direct_map);
/* Note: This function expects iommu_domain->lock to be held prior calling the function. */
static int domain_enable_v2(struct protection_domain *domain, int pasids)
{
int levels;
/* Number of GCR3 table levels required */
for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
levels += 1;
if (levels > amd_iommu_max_glx_val)
return -EINVAL;
domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
if (domain->gcr3_tbl == NULL)
return -ENOMEM;
domain->glx = levels;
domain->flags |= PD_IOMMUV2_MASK;
amd_iommu_domain_update(domain);
return 0;
}
int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
{
struct protection_domain *pdom = to_pdomain(dom);
unsigned long flags;
int ret;
spin_lock_irqsave(&pdom->lock, flags);
/*
* Save us all sanity checks whether devices already in the
* domain support IOMMUv2. Just force that the domain has no
* devices attached when it is switched into IOMMUv2 mode.
*/
ret = -EBUSY;
if (pdom->dev_cnt > 0 || pdom->flags & PD_IOMMUV2_MASK)
goto out;
if (!pdom->gcr3_tbl)
ret = domain_enable_v2(pdom, pasids);
out:
spin_unlock_irqrestore(&pdom->lock, flags);
return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
static int __flush_pasid(struct protection_domain *domain, u32 pasid,
u64 address, bool size)
{
@ -2750,10 +2702,10 @@ static int __flush_pasid(struct protection_domain *domain, u32 pasid,
There might be non-IOMMUv2 capable devices in an IOMMUv2
* domain.
*/
if (!dev_data->ats.enabled)
if (!dev_data->ats_enabled)
continue;
qdep = dev_data->ats.qdep;
qdep = dev_data->ats_qdep;
iommu = rlookup_amd_iommu(dev_data->dev);
if (!iommu)
continue;
@ -2794,7 +2746,6 @@ int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,
return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_page);
static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid)
{
@ -2814,7 +2765,6 @@ int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)
return ret;
}
EXPORT_SYMBOL(amd_iommu_flush_tlb);
static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)
{
@ -2894,7 +2844,6 @@ int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)
{
@ -2908,7 +2857,6 @@ int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)
return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
int status, int tag)
@ -2927,49 +2875,6 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
return iommu_queue_command(iommu, &cmd);
}
EXPORT_SYMBOL(amd_iommu_complete_ppr);
int amd_iommu_device_info(struct pci_dev *pdev,
struct amd_iommu_device_info *info)
{
int max_pasids;
int pos;
if (pdev == NULL || info == NULL)
return -EINVAL;
if (!amd_iommu_v2_supported())
return -EINVAL;
memset(info, 0, sizeof(*info));
if (pci_ats_supported(pdev))
info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
if (pos)
info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
if (pos) {
int features;
max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
max_pasids = min(max_pasids, (1 << 20));
info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
features = pci_pasid_features(pdev);
if (features & PCI_PASID_CAP_EXEC)
info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
if (features & PCI_PASID_CAP_PRIV)
info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
}
return 0;
}
EXPORT_SYMBOL(amd_iommu_device_info);
#ifdef CONFIG_IRQ_REMAP

996
drivers/iommu/amd/iommu_v2.c
View File

@ -1,996 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
* Author: Joerg Roedel <jroedel@suse.de>
*/
#define pr_fmt(fmt) "AMD-Vi: " fmt
#include <linux/refcount.h>
#include <linux/mmu_notifier.h>
#include <linux/amd-iommu.h>
#include <linux/mm_types.h>
#include <linux/profile.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/wait.h>
#include <linux/pci.h>
#include <linux/gfp.h>
#include <linux/cc_platform.h>
#include "amd_iommu.h"
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
#define PRI_QUEUE_SIZE 512
struct pri_queue {
atomic_t inflight;
bool finish;
int status;
};
struct pasid_state {
struct list_head list; /* For global state-list */
refcount_t count; /* Reference count */
unsigned mmu_notifier_count; /* Counting nested mmu_notifier
calls */
struct mm_struct *mm; /* mm_struct for the faults */
struct mmu_notifier mn; /* mmu_notifier handle */
struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
struct device_state *device_state; /* Link to our device_state */
u32 pasid; /* PASID index */
bool invalid; /* Used during setup and
teardown of the pasid */
spinlock_t lock; /* Protect pri_queues and
mmu_notifer_count */
wait_queue_head_t wq; /* To wait for count == 0 */
};
struct device_state {
struct list_head list;
u32 sbdf;
atomic_t count;
struct pci_dev *pdev;
struct pasid_state **states;
struct iommu_domain *domain;
int pasid_levels;
int max_pasids;
amd_iommu_invalid_ppr_cb inv_ppr_cb;
amd_iommu_invalidate_ctx inv_ctx_cb;
spinlock_t lock;
wait_queue_head_t wq;
};
struct fault {
struct work_struct work;
struct device_state *dev_state;
struct pasid_state *state;
struct mm_struct *mm;
u64 address;
u32 pasid;
u16 tag;
u16 finish;
u16 flags;
};
static LIST_HEAD(state_list);
static DEFINE_SPINLOCK(state_lock);
static struct workqueue_struct *iommu_wq;
static void free_pasid_states(struct device_state *dev_state);
static struct device_state *__get_device_state(u32 sbdf)
{
struct device_state *dev_state;
list_for_each_entry(dev_state, &state_list, list) {
if (dev_state->sbdf == sbdf)
return dev_state;
}
return NULL;
}
static struct device_state *get_device_state(u32 sbdf)
{
struct device_state *dev_state;
unsigned long flags;
spin_lock_irqsave(&state_lock, flags);
dev_state = __get_device_state(sbdf);
if (dev_state != NULL)
atomic_inc(&dev_state->count);
spin_unlock_irqrestore(&state_lock, flags);
return dev_state;
}
static void free_device_state(struct device_state *dev_state)
{
struct iommu_group *group;
/* Get rid of any remaining pasid states */
free_pasid_states(dev_state);
/*
* Wait until the last reference is dropped before freeing
* the device state.
*/
wait_event(dev_state->wq, !atomic_read(&dev_state->count));
/*
* First detach device from domain - No more PRI requests will arrive
* from that device after it is unbound from the IOMMUv2 domain.
*/
group = iommu_group_get(&dev_state->pdev->dev);
if (WARN_ON(!group))
return;
iommu_detach_group(dev_state->domain, group);
iommu_group_put(group);
/* Everything is down now, free the IOMMUv2 domain */
iommu_domain_free(dev_state->domain);
/* Finally get rid of the device-state */
kfree(dev_state);
}
static void put_device_state(struct device_state *dev_state)
{
if (atomic_dec_and_test(&dev_state->count))
wake_up(&dev_state->wq);
}
/* Must be called under dev_state->lock */
static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
u32 pasid, bool alloc)
{
struct pasid_state **root, **ptr;
int level, index;
level = dev_state->pasid_levels;
root = dev_state->states;
while (true) {
index = (pasid >> (9 * level)) & 0x1ff;
ptr = &root[index];
if (level == 0)
break;
if (*ptr == NULL) {
if (!alloc)
return NULL;
*ptr = (void *)get_zeroed_page(GFP_ATOMIC);
if (*ptr == NULL)
return NULL;
}
root = (struct pasid_state **)*ptr;
level -= 1;
}
return ptr;
}
static int set_pasid_state(struct device_state *dev_state,
struct pasid_state *pasid_state,
u32 pasid)
{
struct pasid_state **ptr;
unsigned long flags;
int ret;
spin_lock_irqsave(&dev_state->lock, flags);
ptr = __get_pasid_state_ptr(dev_state, pasid, true);
ret = -ENOMEM;
if (ptr == NULL)
goto out_unlock;
ret = -ENOMEM;
if (*ptr != NULL)
goto out_unlock;
*ptr = pasid_state;
ret = 0;
out_unlock:
spin_unlock_irqrestore(&dev_state->lock, flags);
return ret;
}
static void clear_pasid_state(struct device_state *dev_state, u32 pasid)
{
struct pasid_state **ptr;
unsigned long flags;
spin_lock_irqsave(&dev_state->lock, flags);
ptr = __get_pasid_state_ptr(dev_state, pasid, true);
if (ptr == NULL)
goto out_unlock;
*ptr = NULL;
out_unlock:
spin_unlock_irqrestore(&dev_state->lock, flags);
}
static struct pasid_state *get_pasid_state(struct device_state *dev_state,
u32 pasid)
{
struct pasid_state **ptr, *ret = NULL;
unsigned long flags;
spin_lock_irqsave(&dev_state->lock, flags);
ptr = __get_pasid_state_ptr(dev_state, pasid, false);
if (ptr == NULL)
goto out_unlock;
ret = *ptr;
if (ret)
refcount_inc(&ret->count);
out_unlock:
spin_unlock_irqrestore(&dev_state->lock, flags);
return ret;
}
static void free_pasid_state(struct pasid_state *pasid_state)
{
kfree(pasid_state);
}
static void put_pasid_state(struct pasid_state *pasid_state)
{
if (refcount_dec_and_test(&pasid_state->count))
wake_up(&pasid_state->wq);
}
static void put_pasid_state_wait(struct pasid_state *pasid_state)
{
if (!refcount_dec_and_test(&pasid_state->count))
wait_event(pasid_state->wq, !refcount_read(&pasid_state->count));
free_pasid_state(pasid_state);
}
static void unbind_pasid(struct pasid_state *pasid_state)
{
struct iommu_domain *domain;
domain = pasid_state->device_state->domain;
/*
* Mark pasid_state as invalid, no more faults will we added to the
* work queue after this is visible everywhere.
*/
pasid_state->invalid = true;
/* Make sure this is visible */
smp_wmb();
/* After this the device/pasid can't access the mm anymore */
amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
/* Make sure no more pending faults are in the queue */
flush_workqueue(iommu_wq);
}
static void free_pasid_states_level1(struct pasid_state **tbl)
{
int i;
for (i = 0; i < 512; ++i) {
if (tbl[i] == NULL)
continue;
free_page((unsigned long)tbl[i]);
}
}
static void free_pasid_states_level2(struct pasid_state **tbl)
{
struct pasid_state **ptr;
int i;
for (i = 0; i < 512; ++i) {
if (tbl[i] == NULL)
continue;
ptr = (struct pasid_state **)tbl[i];
free_pasid_states_level1(ptr);
}
}
static void free_pasid_states(struct device_state *dev_state)
{
struct pasid_state *pasid_state;
int i;
for (i = 0; i < dev_state->max_pasids; ++i) {
pasid_state = get_pasid_state(dev_state, i);
if (pasid_state == NULL)
continue;
put_pasid_state(pasid_state);
/* Clear the pasid state so that the pasid can be re-used */
clear_pasid_state(dev_state, pasid_state->pasid);
/*
* This will call the mn_release function and
* unbind the PASID
*/
mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
put_pasid_state_wait(pasid_state); /* Reference taken in
amd_iommu_bind_pasid */
/* Drop reference taken in amd_iommu_bind_pasid */
put_device_state(dev_state);
}
if (dev_state->pasid_levels == 2)
free_pasid_states_level2(dev_state->states);
else if (dev_state->pasid_levels == 1)
free_pasid_states_level1(dev_state->states);
else
BUG_ON(dev_state->pasid_levels != 0);
free_page((unsigned long)dev_state->states);
}
static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
{
return container_of(mn, struct pasid_state, mn);
}
static void mn_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end)
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
pasid_state = mn_to_state(mn);
dev_state = pasid_state->device_state;
if ((start ^ (end - 1)) < PAGE_SIZE)
amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
start);
else
amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
}
static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
bool run_inv_ctx_cb;
might_sleep();
pasid_state = mn_to_state(mn);
dev_state = pasid_state->device_state;
run_inv_ctx_cb = !pasid_state->invalid;
if (run_inv_ctx_cb && dev_state->inv_ctx_cb)
dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
unbind_pasid(pasid_state);
}
static const struct mmu_notifier_ops iommu_mn = {
.release = mn_release,
.arch_invalidate_secondary_tlbs = mn_arch_invalidate_secondary_tlbs,
};
static void set_pri_tag_status(struct pasid_state *pasid_state,
u16 tag, int status)
{
unsigned long flags;
spin_lock_irqsave(&pasid_state->lock, flags);
pasid_state->pri[tag].status = status;
spin_unlock_irqrestore(&pasid_state->lock, flags);
}
static void finish_pri_tag(struct device_state *dev_state,
struct pasid_state *pasid_state,
u16 tag)
{
unsigned long flags;
spin_lock_irqsave(&pasid_state->lock, flags);
if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
pasid_state->pri[tag].finish) {
amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
pasid_state->pri[tag].status, tag);
pasid_state->pri[tag].finish = false;
pasid_state->pri[tag].status = PPR_SUCCESS;
}
spin_unlock_irqrestore(&pasid_state->lock, flags);
}
static void handle_fault_error(struct fault *fault)
{
int status;
if (!fault->dev_state->inv_ppr_cb) {
set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
return;
}
status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
fault->pasid,
fault->address,
fault->flags);
switch (status) {
case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
break;
case AMD_IOMMU_INV_PRI_RSP_INVALID:
set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
break;
case AMD_IOMMU_INV_PRI_RSP_FAIL:
set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
break;
default:
BUG();
}
}
static bool access_error(struct vm_area_struct *vma, struct fault *fault)
{
unsigned long requested = 0;
if (fault->flags & PPR_FAULT_EXEC)
requested |= VM_EXEC;
if (fault->flags & PPR_FAULT_READ)
requested |= VM_READ;
if (fault->flags & PPR_FAULT_WRITE)
requested |= VM_WRITE;
return (requested & ~vma->vm_flags) != 0;
}
static void do_fault(struct work_struct *work)
{
struct fault *fault = container_of(work, struct fault, work);
struct vm_area_struct *vma;
vm_fault_t ret = VM_FAULT_ERROR;
unsigned int flags = 0;
struct mm_struct *mm;
u64 address;
mm = fault->state->mm;
address = fault->address;
if (fault->flags & PPR_FAULT_USER)
flags |= FAULT_FLAG_USER;
if (fault->flags & PPR_FAULT_WRITE)
flags |= FAULT_FLAG_WRITE;
flags |= FAULT_FLAG_REMOTE;
mmap_read_lock(mm);
vma = vma_lookup(mm, address);
if (!vma)
/* failed to get a vma in the right range */
goto out;
/* Check if we have the right permissions on the vma */
if (access_error(vma, fault))
goto out;
ret = handle_mm_fault(vma, address, flags, NULL);
out:
mmap_read_unlock(mm);
if (ret & VM_FAULT_ERROR)
/* failed to service fault */
handle_fault_error(fault);
finish_pri_tag(fault->dev_state, fault->state, fault->tag);
put_pasid_state(fault->state);
kfree(fault);
}
static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
{
struct amd_iommu_fault *iommu_fault;
struct pasid_state *pasid_state;
struct device_state *dev_state;
struct pci_dev *pdev = NULL;
unsigned long flags;
struct fault *fault;
bool finish;
u16 tag, devid, seg_id;
int ret;
iommu_fault = data;
tag = iommu_fault->tag & 0x1ff;
finish = (iommu_fault->tag >> 9) & 1;
seg_id = PCI_SBDF_TO_SEGID(iommu_fault->sbdf);
devid = PCI_SBDF_TO_DEVID(iommu_fault->sbdf);
pdev = pci_get_domain_bus_and_slot(seg_id, PCI_BUS_NUM(devid),
devid & 0xff);
if (!pdev)
return -ENODEV;
ret = NOTIFY_DONE;
/* In kdump kernel pci dev is not initialized yet -> send INVALID */
if (amd_iommu_is_attach_deferred(&pdev->dev)) {
amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
PPR_INVALID, tag);
goto out;
}
dev_state = get_device_state(iommu_fault->sbdf);
if (dev_state == NULL)
goto out;
pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
if (pasid_state == NULL || pasid_state->invalid) {
/* We know the device but not the PASID -> send INVALID */
amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
PPR_INVALID, tag);
goto out_drop_state;
}
spin_lock_irqsave(&pasid_state->lock, flags);
atomic_inc(&pasid_state->pri[tag].inflight);
if (finish)
pasid_state->pri[tag].finish = true;
spin_unlock_irqrestore(&pasid_state->lock, flags);
fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
if (fault == NULL) {
/* We are OOM - send success and let the device re-fault */
finish_pri_tag(dev_state, pasid_state, tag);
goto out_drop_state;
}
fault->dev_state = dev_state;
fault->address = iommu_fault->address;
fault->state = pasid_state;
fault->tag = tag;
fault->finish = finish;
fault->pasid = iommu_fault->pasid;
fault->flags = iommu_fault->flags;
INIT_WORK(&fault->work, do_fault);
queue_work(iommu_wq, &fault->work);
ret = NOTIFY_OK;
out_drop_state:
if (ret != NOTIFY_OK && pasid_state)
put_pasid_state(pasid_state);
put_device_state(dev_state);
out:
pci_dev_put(pdev);
return ret;
}
static struct notifier_block ppr_nb = {
.notifier_call = ppr_notifier,
};
int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
struct task_struct *task)
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
struct mm_struct *mm;
u32 sbdf;
int ret;
might_sleep();
if (!amd_iommu_v2_supported())
return -ENODEV;
sbdf = get_pci_sbdf_id(pdev);
dev_state = get_device_state(sbdf);
if (dev_state == NULL)
return -EINVAL;
ret = -EINVAL;
if (pasid >= dev_state->max_pasids)
goto out;
ret = -ENOMEM;
pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
if (pasid_state == NULL)
goto out;
refcount_set(&pasid_state->count, 1);
init_waitqueue_head(&pasid_state->wq);
spin_lock_init(&pasid_state->lock);
mm = get_task_mm(task);
pasid_state->mm = mm;
pasid_state->device_state = dev_state;
pasid_state->pasid = pasid;
pasid_state->invalid = true; /* Mark as valid only if we are
done with setting up the pasid */
pasid_state->mn.ops = &iommu_mn;
if (pasid_state->mm == NULL)
goto out_free;
ret = mmu_notifier_register(&pasid_state->mn, mm);
if (ret)
goto out_free;
ret = set_pasid_state(dev_state, pasid_state, pasid);
if (ret)
goto out_unregister;
ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
__pa(pasid_state->mm->pgd));
if (ret)
goto out_clear_state;
/* Now we are ready to handle faults */
pasid_state->invalid = false;
/*
* Drop the reference to the mm_struct here. We rely on the
* mmu_notifier release call-back to inform us when the mm
* is going away.
*/
mmput(mm);
return 0;
out_clear_state:
clear_pasid_state(dev_state, pasid);
out_unregister:
mmu_notifier_unregister(&pasid_state->mn, mm);
mmput(mm);
out_free:
free_pasid_state(pasid_state);
out:
put_device_state(dev_state);
return ret;
}
EXPORT_SYMBOL(amd_iommu_bind_pasid);
void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid)
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
u32 sbdf;
might_sleep();
if (!amd_iommu_v2_supported())
return;
sbdf = get_pci_sbdf_id(pdev);
dev_state = get_device_state(sbdf);
if (dev_state == NULL)
return;
if (pasid >= dev_state->max_pasids)
goto out;
pasid_state = get_pasid_state(dev_state, pasid);
if (pasid_state == NULL)
goto out;
/*
* Drop reference taken here. We are safe because we still hold
* the reference taken in the amd_iommu_bind_pasid function.
*/
put_pasid_state(pasid_state);
/* Clear the pasid state so that the pasid can be re-used */
clear_pasid_state(dev_state, pasid_state->pasid);
/*
* Call mmu_notifier_unregister to drop our reference
* to pasid_state->mm
*/
mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
put_pasid_state_wait(pasid_state); /* Reference taken in
amd_iommu_bind_pasid */
out:
/* Drop reference taken in this function */
put_device_state(dev_state);
/* Drop reference taken in amd_iommu_bind_pasid */
put_device_state(dev_state);
}
EXPORT_SYMBOL(amd_iommu_unbind_pasid);
int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
{
struct device_state *dev_state;
struct iommu_group *group;
unsigned long flags;
int ret, tmp;
u32 sbdf;
might_sleep();
/*
* When memory encryption is active the device is likely not in a
* direct-mapped domain. Forbid using IOMMUv2 functionality for now.
*/
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
return -ENODEV;
if (!amd_iommu_v2_supported())
return -ENODEV;
if (pasids <= 0 || pasids > (PASID_MASK + 1))
return -EINVAL;
sbdf = get_pci_sbdf_id(pdev);
dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
if (dev_state == NULL)
return -ENOMEM;
spin_lock_init(&dev_state->lock);
init_waitqueue_head(&dev_state->wq);
dev_state->pdev = pdev;
dev_state->sbdf = sbdf;
tmp = pasids;
for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
dev_state->pasid_levels += 1;
atomic_set(&dev_state->count, 1);
dev_state->max_pasids = pasids;
ret = -ENOMEM;
dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
if (dev_state->states == NULL)
goto out_free_dev_state;
dev_state->domain = iommu_domain_alloc(&pci_bus_type);
if (dev_state->domain == NULL)
goto out_free_states;
/* See iommu_is_default_domain() */
dev_state->domain->type = IOMMU_DOMAIN_IDENTITY;
amd_iommu_domain_direct_map(dev_state->domain);
ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
if (ret)
goto out_free_domain;
group = iommu_group_get(&pdev->dev);
if (!group) {
ret = -EINVAL;
goto out_free_domain;
}
ret = iommu_attach_group(dev_state->domain, group);
if (ret != 0)
goto out_drop_group;
iommu_group_put(group);
spin_lock_irqsave(&state_lock, flags);
if (__get_device_state(sbdf) != NULL) {
spin_unlock_irqrestore(&state_lock, flags);
ret = -EBUSY;
goto out_free_domain;
}
list_add_tail(&dev_state->list, &state_list);
spin_unlock_irqrestore(&state_lock, flags);
return 0;
out_drop_group:
iommu_group_put(group);
out_free_domain:
iommu_domain_free(dev_state->domain);
out_free_states:
free_page((unsigned long)dev_state->states);
out_free_dev_state:
kfree(dev_state);
return ret;
}
EXPORT_SYMBOL(amd_iommu_init_device);
void amd_iommu_free_device(struct pci_dev *pdev)
{
struct device_state *dev_state;
unsigned long flags;
u32 sbdf;
if (!amd_iommu_v2_supported())
return;
sbdf = get_pci_sbdf_id(pdev);
spin_lock_irqsave(&state_lock, flags);
dev_state = __get_device_state(sbdf);
if (dev_state == NULL) {
spin_unlock_irqrestore(&state_lock, flags);
return;
}
list_del(&dev_state->list);
spin_unlock_irqrestore(&state_lock, flags);
put_device_state(dev_state);
free_device_state(dev_state);
}
EXPORT_SYMBOL(amd_iommu_free_device);
int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
amd_iommu_invalid_ppr_cb cb)
{
struct device_state *dev_state;
unsigned long flags;
u32 sbdf;
int ret;
if (!amd_iommu_v2_supported())
return -ENODEV;
sbdf = get_pci_sbdf_id(pdev);
spin_lock_irqsave(&state_lock, flags);
ret = -EINVAL;
dev_state = __get_device_state(sbdf);
if (dev_state == NULL)
goto out_unlock;
dev_state->inv_ppr_cb = cb;
ret = 0;
out_unlock:
spin_unlock_irqrestore(&state_lock, flags);
return ret;
}
EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
amd_iommu_invalidate_ctx cb)
{
struct device_state *dev_state;
unsigned long flags;
u32 sbdf;
int ret;
if (!amd_iommu_v2_supported())
return -ENODEV;
sbdf = get_pci_sbdf_id(pdev);
spin_lock_irqsave(&state_lock, flags);
ret = -EINVAL;
dev_state = __get_device_state(sbdf);
if (dev_state == NULL)
goto out_unlock;
dev_state->inv_ctx_cb = cb;
ret = 0;
out_unlock:
spin_unlock_irqrestore(&state_lock, flags);
return ret;
}
EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
static int __init amd_iommu_v2_init(void)
{
int ret;
if (!amd_iommu_v2_supported()) {
pr_info("AMD IOMMUv2 functionality not available on this system - This is not a bug.\n");
/*
* Load anyway to provide the symbols to other modules
* which may use AMD IOMMUv2 optionally.
*/
return 0;
}
ret = -ENOMEM;
iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0);
if (iommu_wq == NULL)
goto out;
amd_iommu_register_ppr_notifier(&ppr_nb);
pr_info("AMD IOMMUv2 loaded and initialized\n");
return 0;
out:
return ret;
}
static void __exit amd_iommu_v2_exit(void)
{
struct device_state *dev_state, *next;
unsigned long flags;
LIST_HEAD(freelist);
if (!amd_iommu_v2_supported())
return;
amd_iommu_unregister_ppr_notifier(&ppr_nb);
flush_workqueue(iommu_wq);
/*
* The loop below might call flush_workqueue(), so call
* destroy_workqueue() after it
*/
spin_lock_irqsave(&state_lock, flags);
list_for_each_entry_safe(dev_state, next, &state_list, list) {
WARN_ON_ONCE(1);
put_device_state(dev_state);
list_del(&dev_state->list);
list_add_tail(&dev_state->list, &freelist);
}
spin_unlock_irqrestore(&state_lock, flags);
/*
* Since free_device_state waits on the count to be zero,
* we need to free dev_state outside the spinlock.
*/
list_for_each_entry_safe(dev_state, next, &freelist, list) {
list_del(&dev_state->list);
free_device_state(dev_state);
}
destroy_workqueue(iommu_wq);
}
module_init(amd_iommu_v2_init);
module_exit(amd_iommu_v2_exit);

142
drivers/iommu/apple-dart.c
View File

@ -196,7 +196,6 @@ struct apple_dart_hw {
* @lock: lock for hardware operations involving this dart
* @pgsize: pagesize supported by this DART
* @supports_bypass: indicates if this DART supports bypass mode
* @force_bypass: force bypass mode due to pagesize mismatch?
* @sid2group: maps stream ids to iommu_groups
* @iommu: iommu core device
*/
@ -217,7 +216,6 @@ struct apple_dart {
u32 pgsize;
u32 num_streams;
u32 supports_bypass : 1;
u32 force_bypass : 1;
struct iommu_group *sid2group[DART_MAX_STREAMS];
struct iommu_device iommu;
@ -506,10 +504,11 @@ static void apple_dart_iotlb_sync(struct iommu_domain *domain,
apple_dart_domain_flush_tlb(to_dart_domain(domain));
}
static void apple_dart_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
apple_dart_domain_flush_tlb(to_dart_domain(domain));
return 0;
}
static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
@ -568,15 +567,17 @@ apple_dart_setup_translation(struct apple_dart_domain *domain,
stream_map->dart->hw->invalidate_tlb(stream_map);
}
static int apple_dart_finalize_domain(struct iommu_domain *domain,
static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
struct apple_dart_master_cfg *cfg)
{
struct apple_dart_domain *dart_domain = to_dart_domain(domain);
struct apple_dart *dart = cfg->stream_maps[0].dart;
struct io_pgtable_cfg pgtbl_cfg;
int ret = 0;
int i, j;
if (dart->pgsize > PAGE_SIZE)
return -EINVAL;
mutex_lock(&dart_domain->init_lock);
if (dart_domain->finalized)
@ -597,17 +598,18 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain,
.iommu_dev = dart->dev,
};
dart_domain->pgtbl_ops =
alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain);
dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
&dart_domain->domain);
if (!dart_domain->pgtbl_ops) {
ret = -ENOMEM;
goto done;
}
domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
domain->geometry.aperture_start = 0;
domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(dart->ias);
domain->geometry.force_aperture = true;
dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
dart_domain->domain.geometry.aperture_start = 0;
dart_domain->domain.geometry.aperture_end =
(dma_addr_t)DMA_BIT_MASK(dart->ias);
dart_domain->domain.geometry.force_aperture = true;
dart_domain->finalized = true;
@ -651,47 +653,72 @@ static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
true);
}
static int apple_dart_attach_dev(struct iommu_domain *domain,
struct device *dev)
static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
struct device *dev)
{
int ret, i;
struct apple_dart_stream_map *stream_map;
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
struct apple_dart_domain *dart_domain = to_dart_domain(domain);
if (cfg->stream_maps[0].dart->force_bypass &&
domain->type != IOMMU_DOMAIN_IDENTITY)
return -EINVAL;
if (!cfg->stream_maps[0].dart->supports_bypass &&
domain->type == IOMMU_DOMAIN_IDENTITY)
return -EINVAL;
ret = apple_dart_finalize_domain(domain, cfg);
ret = apple_dart_finalize_domain(dart_domain, cfg);
if (ret)
return ret;
switch (domain->type) {
default:
ret = apple_dart_domain_add_streams(dart_domain, cfg);
if (ret)
return ret;
ret = apple_dart_domain_add_streams(dart_domain, cfg);
if (ret)
return ret;
for_each_stream_map(i, cfg, stream_map)
apple_dart_setup_translation(dart_domain, stream_map);
break;
case IOMMU_DOMAIN_BLOCKED:
for_each_stream_map(i, cfg, stream_map)
apple_dart_hw_disable_dma(stream_map);
break;
case IOMMU_DOMAIN_IDENTITY:
for_each_stream_map(i, cfg, stream_map)
apple_dart_hw_enable_bypass(stream_map);
break;
}
return ret;
for_each_stream_map(i, cfg, stream_map)
apple_dart_setup_translation(dart_domain, stream_map);
return 0;
}
static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
struct device *dev)
{
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
struct apple_dart_stream_map *stream_map;
int i;
if (!cfg->stream_maps[0].dart->supports_bypass)
return -EINVAL;
for_each_stream_map(i, cfg, stream_map)
apple_dart_hw_enable_bypass(stream_map);
return 0;
}
static const struct iommu_domain_ops apple_dart_identity_ops = {
.attach_dev = apple_dart_attach_dev_identity,
};
static struct iommu_domain apple_dart_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &apple_dart_identity_ops,
};
static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
struct device *dev)
{
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
struct apple_dart_stream_map *stream_map;
int i;
for_each_stream_map(i, cfg, stream_map)
apple_dart_hw_disable_dma(stream_map);
return 0;
}
static const struct iommu_domain_ops apple_dart_blocked_ops = {
.attach_dev = apple_dart_attach_dev_blocked,
};
static struct iommu_domain apple_dart_blocked_domain = {
.type = IOMMU_DOMAIN_BLOCKED,
.ops = &apple_dart_blocked_ops,
};
static struct iommu_device *apple_dart_probe_device(struct device *dev)
{
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
@ -717,24 +744,26 @@ static void apple_dart_release_device(struct device *dev)
kfree(cfg);
}
static struct iommu_domain *apple_dart_domain_alloc(unsigned int type)
static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
{
struct apple_dart_domain *dart_domain;
if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED &&
type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_BLOCKED)
return NULL;
dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
if (!dart_domain)
return NULL;
mutex_init(&dart_domain->init_lock);
/* no need to allocate pgtbl_ops or do any other finalization steps */
if (type == IOMMU_DOMAIN_IDENTITY || type == IOMMU_DOMAIN_BLOCKED)
dart_domain->finalized = true;
if (dev) {
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
int ret;
ret = apple_dart_finalize_domain(dart_domain, cfg);
if (ret) {
kfree(dart_domain);
return ERR_PTR(ret);
}
}
return &dart_domain->domain;
}
@ -770,8 +799,6 @@ static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args)
if (cfg_dart) {
if (cfg_dart->supports_bypass != dart->supports_bypass)
return -EINVAL;
if (cfg_dart->force_bypass != dart->force_bypass)
return -EINVAL;
if (cfg_dart->pgsize != dart->pgsize)
return -EINVAL;
}
@ -913,7 +940,7 @@ static int apple_dart_def_domain_type(struct device *dev)
{
struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
if (cfg->stream_maps[0].dart->force_bypass)
if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
return IOMMU_DOMAIN_IDENTITY;
if (!cfg->stream_maps[0].dart->supports_bypass)
return IOMMU_DOMAIN_DMA;
@ -947,7 +974,9 @@ static void apple_dart_get_resv_regions(struct device *dev,
}
static const struct iommu_ops apple_dart_iommu_ops = {
.domain_alloc = apple_dart_domain_alloc,
.identity_domain = &apple_dart_identity_domain,
.blocked_domain = &apple_dart_blocked_domain,
.domain_alloc_paging = apple_dart_domain_alloc_paging,
.probe_device = apple_dart_probe_device,
.release_device = apple_dart_release_device,
.device_group = apple_dart_device_group,
@ -957,7 +986,7 @@ static const struct iommu_ops apple_dart_iommu_ops = {
.pgsize_bitmap = -1UL, /* Restricted during dart probe */
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = apple_dart_attach_dev,
.attach_dev = apple_dart_attach_dev_paging,
.map_pages = apple_dart_map_pages,
.unmap_pages = apple_dart_unmap_pages,
.flush_iotlb_all = apple_dart_flush_iotlb_all,
@ -1111,8 +1140,6 @@ static int apple_dart_probe(struct platform_device *pdev)
goto err_clk_disable;
}
dart->force_bypass = dart->pgsize > PAGE_SIZE;
ret = apple_dart_hw_reset(dart);
if (ret)
goto err_clk_disable;
@ -1136,7 +1163,8 @@ static int apple_dart_probe(struct platform_device *pdev)
dev_info(
&pdev->dev,
"DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n",
dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass);
dart->pgsize, dart->num_streams, dart->supports_bypass,
dart->pgsize > PAGE_SIZE);
return 0;
err_sysfs_remove:

71
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
View File

@ -25,11 +25,9 @@ struct arm_smmu_mmu_notifier {
#define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
struct arm_smmu_bond {
struct iommu_sva sva;
struct mm_struct *mm;
struct arm_smmu_mmu_notifier *smmu_mn;
struct list_head list;
refcount_t refs;
};
#define sva_to_bond(handle) \
@ -37,6 +35,25 @@ struct arm_smmu_bond {
static DEFINE_MUTEX(sva_lock);
/*
* Write the CD to the CD tables for all masters that this domain is attached
* to. Note that this is only used to update existing CD entries in the target
* CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail.
*/
static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain,
int ssid,
struct arm_smmu_ctx_desc *cd)
{
struct arm_smmu_master *master;
unsigned long flags;
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
list_for_each_entry(master, &smmu_domain->devices, domain_head) {
arm_smmu_write_ctx_desc(master, ssid, cd);
}
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
}
/*
* Check if the CPU ASID is available on the SMMU side. If a private context
* descriptor is using it, try to replace it.
@ -62,7 +79,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
return cd;
}
smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd);
smmu_domain = container_of(cd, struct arm_smmu_domain, cd);
smmu = smmu_domain->smmu;
ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd,
@ -80,7 +97,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
* be some overlap between use of both ASIDs, until we invalidate the
* TLB.
*/
arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, cd);
arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd);
/* Invalidate TLB entries previously associated with that context */
arm_smmu_tlb_inv_asid(smmu, asid);
@ -247,7 +264,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
* DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
* but disable translation.
*/
arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd);
arm_smmu_update_ctx_desc_devices(smmu_domain, mm->pasid, &quiet_cd);
arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
@ -273,8 +290,10 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
struct mm_struct *mm)
{
int ret;
unsigned long flags;
struct arm_smmu_ctx_desc *cd;
struct arm_smmu_mmu_notifier *smmu_mn;
struct arm_smmu_master *master;
list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
if (smmu_mn->mn.mm == mm) {
@ -304,7 +323,16 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
goto err_free_cd;
}
ret = arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, cd);
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
list_for_each_entry(master, &smmu_domain->devices, domain_head) {
ret = arm_smmu_write_ctx_desc(master, mm->pasid, cd);
if (ret) {
list_for_each_entry_from_reverse(master, &smmu_domain->devices, domain_head)
arm_smmu_write_ctx_desc(master, mm->pasid, NULL);
break;
}
}
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
if (ret)
goto err_put_notifier;
@ -329,7 +357,8 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
return;
list_del(&smmu_mn->list);
arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, NULL);
arm_smmu_update_ctx_desc_devices(smmu_domain, mm->pasid, NULL);
/*
* If we went through clear(), we've already invalidated, and no
@ -345,8 +374,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
arm_smmu_free_shared_cd(cd);
}
static struct iommu_sva *
__arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
{
int ret;
struct arm_smmu_bond *bond;
@ -355,23 +383,13 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
if (!master || !master->sva_enabled)
return ERR_PTR(-ENODEV);
/* If bind() was already called for this {dev, mm} pair, reuse it. */
list_for_each_entry(bond, &master->bonds, list) {
if (bond->mm == mm) {
refcount_inc(&bond->refs);
return &bond->sva;
}
}
return -ENODEV;
bond = kzalloc(sizeof(*bond), GFP_KERNEL);
if (!bond)
return ERR_PTR(-ENOMEM);
return -ENOMEM;
bond->mm = mm;
bond->sva.dev = dev;
refcount_set(&bond->refs, 1);
bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
if (IS_ERR(bond->smmu_mn)) {
@ -380,11 +398,11 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
}
list_add(&bond->list, &master->bonds);
return &bond->sva;
return 0;
err_free_bond:
kfree(bond);
return ERR_PTR(ret);
return ret;
}
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
@ -550,7 +568,7 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
}
}
if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
if (!WARN_ON(!bond)) {
list_del(&bond->list);
arm_smmu_mmu_notifier_put(bond->smmu_mn);
kfree(bond);
@ -562,13 +580,10 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
int ret = 0;
struct iommu_sva *handle;
struct mm_struct *mm = domain->mm;
mutex_lock(&sva_lock);
handle = __arm_smmu_sva_bind(dev, mm);
if (IS_ERR(handle))
ret = PTR_ERR(handle);
ret = __arm_smmu_sva_bind(dev, mm);
mutex_unlock(&sva_lock);
return ret;

253
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@ -971,14 +971,12 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
}
static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
static void arm_smmu_sync_cd(struct arm_smmu_master *master,
int ssid, bool leaf)
{
size_t i;
unsigned long flags;
struct arm_smmu_master *master;
struct arm_smmu_cmdq_batch cmds;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_cmdq_ent cmd = {
.opcode = CMDQ_OP_CFGI_CD,
.cfgi = {
@ -988,15 +986,10 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
};
cmds.num = 0;
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
list_for_each_entry(master, &smmu_domain->devices, domain_head) {
for (i = 0; i < master->num_streams; i++) {
cmd.cfgi.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
}
for (i = 0; i < master->num_streams; i++) {
cmd.cfgi.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
}
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
arm_smmu_cmdq_batch_submit(smmu, &cmds);
}
@ -1026,34 +1019,33 @@ static void arm_smmu_write_cd_l1_desc(__le64 *dst,
WRITE_ONCE(*dst, cpu_to_le64(val));
}
static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
u32 ssid)
static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid)
{
__le64 *l1ptr;
unsigned int idx;
struct arm_smmu_l1_ctx_desc *l1_desc;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
return cd_table->cdtab + ssid * CTXDESC_CD_DWORDS;
idx = ssid >> CTXDESC_SPLIT;
l1_desc = &cdcfg->l1_desc[idx];
l1_desc = &cd_table->l1_desc[idx];
if (!l1_desc->l2ptr) {
if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
return NULL;
l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
l1ptr = cd_table->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
/* An invalid L1CD can be cached */
arm_smmu_sync_cd(smmu_domain, ssid, false);
arm_smmu_sync_cd(master, ssid, false);
}
idx = ssid & (CTXDESC_L2_ENTRIES - 1);
return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
}
int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
int arm_smmu_write_ctx_desc(struct arm_smmu_master *master, int ssid,
struct arm_smmu_ctx_desc *cd)
{
/*
@ -1070,11 +1062,12 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
u64 val;
bool cd_live;
__le64 *cdptr;
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
if (WARN_ON(ssid >= (1 << cd_table->s1cdmax)))
return -E2BIG;
cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
cdptr = arm_smmu_get_cd_ptr(master, ssid);
if (!cdptr)
return -ENOMEM;
@ -1098,11 +1091,11 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
cdptr[3] = cpu_to_le64(cd->mair);
/*
* STE is live, and the SMMU might read dwords of this CD in any
* STE may be live, and the SMMU might read dwords of this CD in any
* order. Ensure that it observes valid values before reading
* V=1.
*/
arm_smmu_sync_cd(smmu_domain, ssid, true);
arm_smmu_sync_cd(master, ssid, true);
val = cd->tcr |
#ifdef __BIG_ENDIAN
@ -1114,7 +1107,7 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
CTXDESC_CD_0_V;
if (smmu_domain->stall_enabled)
if (cd_table->stall_enabled)
val |= CTXDESC_CD_0_S;
}
@ -1128,44 +1121,45 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
* without first making the structure invalid.
*/
WRITE_ONCE(cdptr[0], cpu_to_le64(val));
arm_smmu_sync_cd(smmu_domain, ssid, true);
arm_smmu_sync_cd(master, ssid, true);
return 0;
}
static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
{
int ret;
size_t l1size;
size_t max_contexts;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
max_contexts = 1 << cfg->s1cdmax;
cd_table->stall_enabled = master->stall_enabled;
cd_table->s1cdmax = master->ssid_bits;
max_contexts = 1 << cd_table->s1cdmax;
if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
max_contexts <= CTXDESC_L2_ENTRIES) {
cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
cdcfg->num_l1_ents = max_contexts;
cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
cd_table->num_l1_ents = max_contexts;
l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
} else {
cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
cd_table->num_l1_ents = DIV_ROUND_UP(max_contexts,
CTXDESC_L2_ENTRIES);
cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
sizeof(*cdcfg->l1_desc),
cd_table->l1_desc = devm_kcalloc(smmu->dev, cd_table->num_l1_ents,
sizeof(*cd_table->l1_desc),
GFP_KERNEL);
if (!cdcfg->l1_desc)
if (!cd_table->l1_desc)
return -ENOMEM;
l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
l1size = cd_table->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
}
cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
cd_table->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cd_table->cdtab_dma,
GFP_KERNEL);
if (!cdcfg->cdtab) {
if (!cd_table->cdtab) {
dev_warn(smmu->dev, "failed to allocate context descriptor\n");
ret = -ENOMEM;
goto err_free_l1;
@ -1174,42 +1168,42 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
return 0;
err_free_l1:
if (cdcfg->l1_desc) {
devm_kfree(smmu->dev, cdcfg->l1_desc);
cdcfg->l1_desc = NULL;
if (cd_table->l1_desc) {
devm_kfree(smmu->dev, cd_table->l1_desc);
cd_table->l1_desc = NULL;
}
return ret;
}
static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
{
int i;
size_t size, l1size;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
if (cdcfg->l1_desc) {
if (cd_table->l1_desc) {
size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
for (i = 0; i < cdcfg->num_l1_ents; i++) {
if (!cdcfg->l1_desc[i].l2ptr)
for (i = 0; i < cd_table->num_l1_ents; i++) {
if (!cd_table->l1_desc[i].l2ptr)
continue;
dmam_free_coherent(smmu->dev, size,
cdcfg->l1_desc[i].l2ptr,
cdcfg->l1_desc[i].l2ptr_dma);
cd_table->l1_desc[i].l2ptr,
cd_table->l1_desc[i].l2ptr_dma);
}
devm_kfree(smmu->dev, cdcfg->l1_desc);
cdcfg->l1_desc = NULL;
devm_kfree(smmu->dev, cd_table->l1_desc);
cd_table->l1_desc = NULL;
l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
l1size = cd_table->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
} else {
l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
l1size = cd_table->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
}
dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
cdcfg->cdtab_dma = 0;
cdcfg->cdtab = NULL;
dmam_free_coherent(smmu->dev, l1size, cd_table->cdtab, cd_table->cdtab_dma);
cd_table->cdtab_dma = 0;
cd_table->cdtab = NULL;
}
bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
@ -1276,7 +1270,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
u64 val = le64_to_cpu(dst[0]);
bool ste_live = false;
struct arm_smmu_device *smmu = NULL;
struct arm_smmu_s1_cfg *s1_cfg = NULL;
struct arm_smmu_ctx_desc_cfg *cd_table = NULL;
struct arm_smmu_s2_cfg *s2_cfg = NULL;
struct arm_smmu_domain *smmu_domain = NULL;
struct arm_smmu_cmdq_ent prefetch_cmd = {
@ -1294,7 +1288,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
if (smmu_domain) {
switch (smmu_domain->stage) {
case ARM_SMMU_DOMAIN_S1:
s1_cfg = &smmu_domain->s1_cfg;
cd_table = &master->cd_table;
break;
case ARM_SMMU_DOMAIN_S2:
case ARM_SMMU_DOMAIN_NESTED:
@ -1325,7 +1319,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
val = STRTAB_STE_0_V;
/* Bypass/fault */
if (!smmu_domain || !(s1_cfg || s2_cfg)) {
if (!smmu_domain || !(cd_table || s2_cfg)) {
if (!smmu_domain && disable_bypass)
val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
else
@ -1344,7 +1338,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
return;
}
if (s1_cfg) {
if (cd_table) {
u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
@ -1360,10 +1354,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
!master->stall_enabled)
dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
val |= (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax) |
FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt);
}
if (s2_cfg) {
@ -1869,7 +1863,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
* careful, 007.
*/
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
} else {
cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
@ -1962,7 +1956,7 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
cmd.tlbi.asid = smmu_domain->cd.asid;
} else {
cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
@ -2067,15 +2061,11 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
free_io_pgtable_ops(smmu_domain->pgtbl_ops);
/* Free the CD and ASID, if we allocated them */
/* Free the ASID or VMID */
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
/* Prevent SVA from touching the CD while we're freeing it */
mutex_lock(&arm_smmu_asid_lock);
if (cfg->cdcfg.cdtab)
arm_smmu_free_cd_tables(smmu_domain);
arm_smmu_free_asid(&cfg->cd);
arm_smmu_free_asid(&smmu_domain->cd);
mutex_unlock(&arm_smmu_asid_lock);
} else {
struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
@ -2087,66 +2077,43 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
}
static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_master *master,
struct io_pgtable_cfg *pgtbl_cfg)
{
int ret;
u32 asid;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
refcount_set(&cfg->cd.refs, 1);
refcount_set(&cd->refs, 1);
/* Prevent SVA from modifying the ASID until it is written to the CD */
mutex_lock(&arm_smmu_asid_lock);
ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
ret = xa_alloc(&arm_smmu_asid_xa, &asid, cd,
XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
if (ret)
goto out_unlock;
cfg->s1cdmax = master->ssid_bits;
smmu_domain->stall_enabled = master->stall_enabled;
ret = arm_smmu_alloc_cd_tables(smmu_domain);
if (ret)
goto out_free_asid;
cfg->cd.asid = (u16)asid;
cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
cd->asid = (u16)asid;
cd->ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
cd->tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
/*
* Note that this will end up calling arm_smmu_sync_cd() before
* the master has been added to the devices list for this domain.
* This isn't an issue because the STE hasn't been installed yet.
*/
ret = arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, &cfg->cd);
if (ret)
goto out_free_cd_tables;
cd->mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
mutex_unlock(&arm_smmu_asid_lock);
return 0;
out_free_cd_tables:
arm_smmu_free_cd_tables(smmu_domain);
out_free_asid:
arm_smmu_free_asid(&cfg->cd);
out_unlock:
mutex_unlock(&arm_smmu_asid_lock);
return ret;
}
static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_master *master,
struct io_pgtable_cfg *pgtbl_cfg)
{
int vmid;
@ -2173,8 +2140,7 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
return 0;
}
static int arm_smmu_domain_finalise(struct iommu_domain *domain,
struct arm_smmu_master *master)
static int arm_smmu_domain_finalise(struct iommu_domain *domain)
{
int ret;
unsigned long ias, oas;
@ -2182,7 +2148,6 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain,
struct io_pgtable_cfg pgtbl_cfg;
struct io_pgtable_ops *pgtbl_ops;
int (*finalise_stage_fn)(struct arm_smmu_domain *,
struct arm_smmu_master *,
struct io_pgtable_cfg *);
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
@ -2234,7 +2199,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain,
domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
domain->geometry.force_aperture = true;
ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
if (ret < 0) {
free_io_pgtable_ops(pgtbl_ops);
return ret;
@ -2403,6 +2368,14 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
master->domain = NULL;
master->ats_enabled = false;
arm_smmu_install_ste_for_dev(master);
/*
* Clearing the CD entry isn't strictly required to detach the domain
* since the table is uninstalled anyway, but it helps avoid confusion
* in the call to arm_smmu_write_ctx_desc on the next attach (which
* expects the entry to be empty).
*/
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && master->cd_table.cdtab)
arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL);
}
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
@ -2436,23 +2409,15 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
if (!smmu_domain->smmu) {
smmu_domain->smmu = smmu;
ret = arm_smmu_domain_finalise(domain, master);
if (ret) {
ret = arm_smmu_domain_finalise(domain);
if (ret)
smmu_domain->smmu = NULL;
goto out_unlock;
}
} else if (smmu_domain->smmu != smmu) {
} else if (smmu_domain->smmu != smmu)
ret = -EINVAL;
goto out_unlock;
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
ret = -EINVAL;
goto out_unlock;
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
smmu_domain->stall_enabled != master->stall_enabled) {
ret = -EINVAL;
goto out_unlock;
}
mutex_unlock(&smmu_domain->init_mutex);
if (ret)
return ret;
master->domain = smmu_domain;
@ -2466,16 +2431,42 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
master->ats_enabled = arm_smmu_ats_supported(master);
arm_smmu_install_ste_for_dev(master);
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
list_add(&master->domain_head, &smmu_domain->devices);
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
arm_smmu_enable_ats(master);
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
if (!master->cd_table.cdtab) {
ret = arm_smmu_alloc_cd_tables(master);
if (ret) {
master->domain = NULL;
goto out_list_del;
}
}
/*
* Prevent SVA from concurrently modifying the CD or writing to
* the CD entry
*/
mutex_lock(&arm_smmu_asid_lock);
ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd);
mutex_unlock(&arm_smmu_asid_lock);
if (ret) {
master->domain = NULL;
goto out_list_del;
}
}
arm_smmu_install_ste_for_dev(master);
arm_smmu_enable_ats(master);
return 0;
out_list_del:
spin_lock_irqsave(&smmu_domain->devices_lock, flags);
list_del(&master->domain_head);
spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
out_unlock:
mutex_unlock(&smmu_domain->init_mutex);
return ret;
}
@ -2720,6 +2711,8 @@ static void arm_smmu_release_device(struct device *dev)
arm_smmu_detach_dev(master);
arm_smmu_disable_pasid(master);
arm_smmu_remove_master(master);
if (master->cd_table.cdtab)
arm_smmu_free_cd_tables(master);
kfree(master);
}

17
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@ -595,13 +595,11 @@ struct arm_smmu_ctx_desc_cfg {
dma_addr_t cdtab_dma;
struct arm_smmu_l1_ctx_desc *l1_desc;
unsigned int num_l1_ents;
};
struct arm_smmu_s1_cfg {
struct arm_smmu_ctx_desc_cfg cdcfg;
struct arm_smmu_ctx_desc cd;
u8 s1fmt;
/* log2 of the maximum number of CDs supported by this table */
u8 s1cdmax;
/* Whether CD entries in this table have the stall bit set. */
u8 stall_enabled:1;
};
struct arm_smmu_s2_cfg {
@ -697,6 +695,8 @@ struct arm_smmu_master {
struct arm_smmu_domain *domain;
struct list_head domain_head;
struct arm_smmu_stream *streams;
/* Locked by the iommu core using the group mutex */
struct arm_smmu_ctx_desc_cfg cd_table;
unsigned int num_streams;
bool ats_enabled;
bool stall_enabled;
@ -719,13 +719,12 @@ struct arm_smmu_domain {
struct mutex init_mutex; /* Protects smmu pointer */
struct io_pgtable_ops *pgtbl_ops;
bool stall_enabled;
atomic_t nr_ats_masters;
enum arm_smmu_domain_stage stage;
union {
struct arm_smmu_s1_cfg s1_cfg;
struct arm_smmu_s2_cfg s2_cfg;
struct arm_smmu_ctx_desc cd;
struct arm_smmu_s2_cfg s2_cfg;
};
struct iommu_domain domain;
@ -745,7 +744,7 @@ extern struct xarray arm_smmu_asid_xa;
extern struct mutex arm_smmu_asid_lock;
extern struct arm_smmu_ctx_desc quiet_cd;
int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
int arm_smmu_write_ctx_desc(struct arm_smmu_master *smmu_master, int ssid,
struct arm_smmu_ctx_desc *cd);
void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,

2
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
View File

@ -251,6 +251,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
{ .compatible = "qcom,sc7280-mss-pil" },
{ .compatible = "qcom,sc8180x-mdss" },
{ .compatible = "qcom,sc8280xp-mdss" },
{ .compatible = "qcom,sdm670-mdss" },
{ .compatible = "qcom,sdm845-mdss" },
{ .compatible = "qcom,sdm845-mss-pil" },
{ .compatible = "qcom,sm6350-mdss" },
@ -532,6 +533,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
{ .compatible = "qcom,sm6350-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm6375-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sm6375-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm7150-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sm8150-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8250-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8350-smmu-500", .data = &qcom_smmu_500_impl0_data },

45
drivers/iommu/arm/arm-smmu/qcom_iommu.c
View File

@ -332,12 +332,10 @@ out_unlock:
return ret;
}
static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
static struct iommu_domain *qcom_iommu_domain_alloc_paging(struct device *dev)
{
struct qcom_iommu_domain *qcom_domain;
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
return NULL;
/*
* Allocate the domain and initialise some of its data structures.
* We can't really do anything meaningful until we've added a
@ -400,6 +398,44 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev
return 0;
}
static int qcom_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct qcom_iommu_domain *qcom_domain;
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
unsigned int i;
if (domain == identity_domain || !domain)
return 0;
qcom_domain = to_qcom_iommu_domain(domain);
if (WARN_ON(!qcom_domain->iommu))
return -EINVAL;
pm_runtime_get_sync(qcom_iommu->dev);
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]);
/* Disable the context bank: */
iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
ctx->domain = NULL;
}
pm_runtime_put_sync(qcom_iommu->dev);
return 0;
}
static struct iommu_domain_ops qcom_iommu_identity_ops = {
.attach_dev = qcom_iommu_identity_attach,
};
static struct iommu_domain qcom_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &qcom_iommu_identity_ops,
};
static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
@ -565,8 +601,9 @@ static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
}
static const struct iommu_ops qcom_iommu_ops = {
.identity_domain = &qcom_iommu_identity_domain,
.capable = qcom_iommu_capable,
.domain_alloc = qcom_iommu_domain_alloc,
.domain_alloc_paging = qcom_iommu_domain_alloc_paging,
.probe_device = qcom_iommu_probe_device,
.device_group = generic_device_group,
.of_xlate = qcom_iommu_of_xlate,

200
drivers/iommu/dma-iommu.c
View File

@ -43,14 +43,28 @@ enum iommu_dma_cookie_type {
IOMMU_DMA_MSI_COOKIE,
};
enum iommu_dma_queue_type {
IOMMU_DMA_OPTS_PER_CPU_QUEUE,
IOMMU_DMA_OPTS_SINGLE_QUEUE,
};
struct iommu_dma_options {
enum iommu_dma_queue_type qt;
size_t fq_size;
unsigned int fq_timeout;
};
struct iommu_dma_cookie {
enum iommu_dma_cookie_type type;
union {
/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
struct {
struct iova_domain iovad;
struct iova_fq __percpu *fq; /* Flush queue */
/* Flush queue */
union {
struct iova_fq *single_fq;
struct iova_fq __percpu *percpu_fq;
};
/* Number of TLB flushes that have been started */
atomic64_t fq_flush_start_cnt;
/* Number of TLB flushes that have been finished */
@ -67,6 +81,8 @@ struct iommu_dma_cookie {
/* Domain for flush queue callback; NULL if flush queue not in use */
struct iommu_domain *fq_domain;
/* Options for dma-iommu use */
struct iommu_dma_options options;
struct mutex mutex;
};
@ -84,10 +100,12 @@ static int __init iommu_dma_forcedac_setup(char *str)
early_param("iommu.forcedac", iommu_dma_forcedac_setup);
/* Number of entries per flush queue */
#define IOVA_FQ_SIZE 256
#define IOVA_DEFAULT_FQ_SIZE 256
#define IOVA_SINGLE_FQ_SIZE 32768
/* Timeout (in ms) after which entries are flushed from the queue */
#define IOVA_FQ_TIMEOUT 10
#define IOVA_DEFAULT_FQ_TIMEOUT 10
#define IOVA_SINGLE_FQ_TIMEOUT 1000
/* Flush queue entry for deferred flushing */
struct iova_fq_entry {
@ -99,18 +117,19 @@ struct iova_fq_entry {
/* Per-CPU flush queue structure */
struct iova_fq {
struct iova_fq_entry entries[IOVA_FQ_SIZE];
unsigned int head, tail;
spinlock_t lock;
unsigned int head, tail;
unsigned int mod_mask;
struct iova_fq_entry entries[];
};
#define fq_ring_for_each(i, fq) \
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) & (fq)->mod_mask)
static inline bool fq_full(struct iova_fq *fq)
{
assert_spin_locked(&fq->lock);
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
return (((fq->tail + 1) & fq->mod_mask) == fq->head);
}
static inline unsigned int fq_ring_add(struct iova_fq *fq)
@ -119,12 +138,12 @@ static inline unsigned int fq_ring_add(struct iova_fq *fq)
assert_spin_locked(&fq->lock);
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
fq->tail = (idx + 1) & fq->mod_mask;
return idx;
}
static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
static void fq_ring_free_locked(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
{
u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt);
unsigned int idx;
@ -141,10 +160,19 @@ static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
fq->entries[idx].iova_pfn,
fq->entries[idx].pages);
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
fq->head = (fq->head + 1) & fq->mod_mask;
}
}
static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
{
unsigned long flags;
spin_lock_irqsave(&fq->lock, flags);
fq_ring_free_locked(cookie, fq);
spin_unlock_irqrestore(&fq->lock, flags);
}
static void fq_flush_iotlb(struct iommu_dma_cookie *cookie)
{
atomic64_inc(&cookie->fq_flush_start_cnt);
@ -160,14 +188,11 @@ static void fq_flush_timeout(struct timer_list *t)
atomic_set(&cookie->fq_timer_on, 0);
fq_flush_iotlb(cookie);
for_each_possible_cpu(cpu) {
unsigned long flags;
struct iova_fq *fq;
fq = per_cpu_ptr(cookie->fq, cpu);
spin_lock_irqsave(&fq->lock, flags);
fq_ring_free(cookie, fq);
spin_unlock_irqrestore(&fq->lock, flags);
if (cookie->options.qt == IOMMU_DMA_OPTS_SINGLE_QUEUE) {
fq_ring_free(cookie, cookie->single_fq);
} else {
for_each_possible_cpu(cpu)
fq_ring_free(cookie, per_cpu_ptr(cookie->percpu_fq, cpu));
}
}
@ -188,7 +213,11 @@ static void queue_iova(struct iommu_dma_cookie *cookie,
*/
smp_mb();
fq = raw_cpu_ptr(cookie->fq);
if (cookie->options.qt == IOMMU_DMA_OPTS_SINGLE_QUEUE)
fq = cookie->single_fq;
else
fq = raw_cpu_ptr(cookie->percpu_fq);
spin_lock_irqsave(&fq->lock, flags);
/*
@ -196,11 +225,11 @@ static void queue_iova(struct iommu_dma_cookie *cookie,
* flushed out on another CPU. This makes the fq_full() check below less
* likely to be true.
*/
fq_ring_free(cookie, fq);
fq_ring_free_locked(cookie, fq);
if (fq_full(fq)) {
fq_flush_iotlb(cookie);
fq_ring_free(cookie, fq);
fq_ring_free_locked(cookie, fq);
}
idx = fq_ring_add(fq);
@ -216,34 +245,95 @@ static void queue_iova(struct iommu_dma_cookie *cookie,
if (!atomic_read(&cookie->fq_timer_on) &&
!atomic_xchg(&cookie->fq_timer_on, 1))
mod_timer(&cookie->fq_timer,
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
jiffies + msecs_to_jiffies(cookie->options.fq_timeout));
}
static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
static void iommu_dma_free_fq_single(struct iova_fq *fq)
{
int idx;
fq_ring_for_each(idx, fq)
put_pages_list(&fq->entries[idx].freelist);
vfree(fq);
}
static void iommu_dma_free_fq_percpu(struct iova_fq __percpu *percpu_fq)
{
int cpu, idx;
if (!cookie->fq)
return;
del_timer_sync(&cookie->fq_timer);
/* The IOVAs will be torn down separately, so just free our queued pages */
for_each_possible_cpu(cpu) {
struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu);
struct iova_fq *fq = per_cpu_ptr(percpu_fq, cpu);
fq_ring_for_each(idx, fq)
put_pages_list(&fq->entries[idx].freelist);
}
free_percpu(cookie->fq);
free_percpu(percpu_fq);
}
static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
{
if (!cookie->fq_domain)
return;
del_timer_sync(&cookie->fq_timer);
if (cookie->options.qt == IOMMU_DMA_OPTS_SINGLE_QUEUE)
iommu_dma_free_fq_single(cookie->single_fq);
else
iommu_dma_free_fq_percpu(cookie->percpu_fq);
}
static void iommu_dma_init_one_fq(struct iova_fq *fq, size_t fq_size)
{
int i;
fq->head = 0;
fq->tail = 0;
fq->mod_mask = fq_size - 1;
spin_lock_init(&fq->lock);
for (i = 0; i < fq_size; i++)
INIT_LIST_HEAD(&fq->entries[i].freelist);
}
static int iommu_dma_init_fq_single(struct iommu_dma_cookie *cookie)
{
size_t fq_size = cookie->options.fq_size;
struct iova_fq *queue;
queue = vmalloc(struct_size(queue, entries, fq_size));
if (!queue)
return -ENOMEM;
iommu_dma_init_one_fq(queue, fq_size);
cookie->single_fq = queue;
return 0;
}
static int iommu_dma_init_fq_percpu(struct iommu_dma_cookie *cookie)
{
size_t fq_size = cookie->options.fq_size;
struct iova_fq __percpu *queue;
int cpu;
queue = __alloc_percpu(struct_size(queue, entries, fq_size),
__alignof__(*queue));
if (!queue)
return -ENOMEM;
for_each_possible_cpu(cpu)
iommu_dma_init_one_fq(per_cpu_ptr(queue, cpu), fq_size);
cookie->percpu_fq = queue;
return 0;
}
/* sysfs updates are serialised by the mutex of the group owning @domain */
int iommu_dma_init_fq(struct iommu_domain *domain)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_fq __percpu *queue;
int i, cpu;
int rc;
if (cookie->fq_domain)
return 0;
@ -251,26 +341,16 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
atomic64_set(&cookie->fq_flush_start_cnt, 0);
atomic64_set(&cookie->fq_flush_finish_cnt, 0);
queue = alloc_percpu(struct iova_fq);
if (!queue) {
if (cookie->options.qt == IOMMU_DMA_OPTS_SINGLE_QUEUE)
rc = iommu_dma_init_fq_single(cookie);
else
rc = iommu_dma_init_fq_percpu(cookie);
if (rc) {
pr_warn("iova flush queue initialization failed\n");
return -ENOMEM;
}
for_each_possible_cpu(cpu) {
struct iova_fq *fq = per_cpu_ptr(queue, cpu);
fq->head = 0;
fq->tail = 0;
spin_lock_init(&fq->lock);
for (i = 0; i < IOVA_FQ_SIZE; i++)
INIT_LIST_HEAD(&fq->entries[i].freelist);
}
cookie->fq = queue;
timer_setup(&cookie->fq_timer, fq_flush_timeout, 0);
atomic_set(&cookie->fq_timer_on, 0);
/*
@ -554,6 +634,28 @@ static bool dev_use_sg_swiotlb(struct device *dev, struct scatterlist *sg,
return false;
}
/**
* iommu_dma_init_options - Initialize dma-iommu options
* @options: The options to be initialized
* @dev: Device the options are set for
*
* This allows tuning dma-iommu specific to device properties
*/
static void iommu_dma_init_options(struct iommu_dma_options *options,
struct device *dev)
{
/* Shadowing IOTLB flushes do better with a single large queue */
if (dev->iommu->shadow_on_flush) {
options->qt = IOMMU_DMA_OPTS_SINGLE_QUEUE;
options->fq_timeout = IOVA_SINGLE_FQ_TIMEOUT;
options->fq_size = IOVA_SINGLE_FQ_SIZE;
} else {
options->qt = IOMMU_DMA_OPTS_PER_CPU_QUEUE;
options->fq_size = IOVA_DEFAULT_FQ_SIZE;
options->fq_timeout = IOVA_DEFAULT_FQ_TIMEOUT;
}
}
/**
* iommu_dma_init_domain - Initialise a DMA mapping domain
* @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
@ -614,6 +716,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
if (ret)
goto done_unlock;
iommu_dma_init_options(&cookie->options, dev);
/* If the FQ fails we can simply fall back to strict mode */
if (domain->type == IOMMU_DOMAIN_DMA_FQ &&
(!device_iommu_capable(dev, IOMMU_CAP_DEFERRED_FLUSH) || iommu_dma_init_fq(domain)))

83
drivers/iommu/exynos-iommu.c
View File

@ -24,6 +24,7 @@
typedef u32 sysmmu_iova_t;
typedef u32 sysmmu_pte_t;
static struct iommu_domain exynos_identity_domain;
/* We do not consider super section mapping (16MB) */
#define SECT_ORDER 20
@ -829,7 +830,7 @@ static int __maybe_unused exynos_sysmmu_suspend(struct device *dev)
struct exynos_iommu_owner *owner = dev_iommu_priv_get(master);
mutex_lock(&owner->rpm_lock);
if (data->domain) {
if (&data->domain->domain != &exynos_identity_domain) {
dev_dbg(data->sysmmu, "saving state\n");
__sysmmu_disable(data);
}
@ -847,7 +848,7 @@ static int __maybe_unused exynos_sysmmu_resume(struct device *dev)
struct exynos_iommu_owner *owner = dev_iommu_priv_get(master);
mutex_lock(&owner->rpm_lock);
if (data->domain) {
if (&data->domain->domain != &exynos_identity_domain) {
dev_dbg(data->sysmmu, "restoring state\n");
__sysmmu_enable(data);
}
@ -886,7 +887,7 @@ static inline void exynos_iommu_set_pte(sysmmu_pte_t *ent, sysmmu_pte_t val)
DMA_TO_DEVICE);
}
static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
static struct iommu_domain *exynos_iommu_domain_alloc_paging(struct device *dev)
{
struct exynos_iommu_domain *domain;
dma_addr_t handle;
@ -895,9 +896,6 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
/* Check if correct PTE offsets are initialized */
BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev);
if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
@ -980,17 +978,20 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
kfree(domain);
}
static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
struct device *dev)
static int exynos_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev);
phys_addr_t pagetable = virt_to_phys(domain->pgtable);
struct exynos_iommu_domain *domain;
phys_addr_t pagetable;
struct sysmmu_drvdata *data, *next;
unsigned long flags;
if (!has_sysmmu(dev) || owner->domain != iommu_domain)
return;
if (owner->domain == identity_domain)
return 0;
domain = to_exynos_domain(owner->domain);
pagetable = virt_to_phys(domain->pgtable);
mutex_lock(&owner->rpm_lock);
@ -1009,15 +1010,25 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
list_del_init(&data->domain_node);
spin_unlock(&data->lock);
}
owner->domain = NULL;
owner->domain = identity_domain;
spin_unlock_irqrestore(&domain->lock, flags);
mutex_unlock(&owner->rpm_lock);
dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__,
&pagetable);
dev_dbg(dev, "%s: Restored IOMMU to IDENTITY from pgtable %pa\n",
__func__, &pagetable);
return 0;
}
static struct iommu_domain_ops exynos_identity_ops = {
.attach_dev = exynos_iommu_identity_attach,
};
static struct iommu_domain exynos_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &exynos_identity_ops,
};
static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
struct device *dev)
{
@ -1026,12 +1037,11 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
struct sysmmu_drvdata *data;
phys_addr_t pagetable = virt_to_phys(domain->pgtable);
unsigned long flags;
int err;
if (!has_sysmmu(dev))
return -ENODEV;
if (owner->domain)
exynos_iommu_detach_device(owner->domain, dev);
err = exynos_iommu_identity_attach(&exynos_identity_domain, dev);
if (err)
return err;
mutex_lock(&owner->rpm_lock);
@ -1219,7 +1229,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
*/
static int exynos_iommu_map(struct iommu_domain *iommu_domain,
unsigned long l_iova, phys_addr_t paddr, size_t size,
int prot, gfp_t gfp)
size_t count, int prot, gfp_t gfp, size_t *mapped)
{
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_pte_t *entry;
@ -1253,6 +1263,8 @@ static int exynos_iommu_map(struct iommu_domain *iommu_domain,
if (ret)
pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
__func__, ret, size, iova);
else
*mapped = size;
spin_unlock_irqrestore(&domain->pgtablelock, flags);
@ -1274,7 +1286,7 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
}
static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
unsigned long l_iova, size_t size,
unsigned long l_iova, size_t size, size_t count,
struct iommu_iotlb_gather *gather)
{
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
@ -1407,26 +1419,12 @@ static struct iommu_device *exynos_iommu_probe_device(struct device *dev)
return &data->iommu;
}
static void exynos_iommu_set_platform_dma(struct device *dev)
{
struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev);
if (owner->domain) {
struct iommu_group *group = iommu_group_get(dev);
if (group) {
exynos_iommu_detach_device(owner->domain, dev);
iommu_group_put(group);
}
}
}
static void exynos_iommu_release_device(struct device *dev)
{
struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev);
struct sysmmu_drvdata *data;
exynos_iommu_set_platform_dma(dev);
WARN_ON(exynos_iommu_identity_attach(&exynos_identity_domain, dev));
list_for_each_entry(data, &owner->controllers, owner_node)
device_link_del(data->link);
@ -1457,6 +1455,7 @@ static int exynos_iommu_of_xlate(struct device *dev,
INIT_LIST_HEAD(&owner->controllers);
mutex_init(&owner->rpm_lock);
owner->domain = &exynos_identity_domain;
dev_iommu_priv_set(dev, owner);
}
@ -1471,19 +1470,17 @@ static int exynos_iommu_of_xlate(struct device *dev,
}
static const struct iommu_ops exynos_iommu_ops = {
.domain_alloc = exynos_iommu_domain_alloc,
.identity_domain = &exynos_identity_domain,
.domain_alloc_paging = exynos_iommu_domain_alloc_paging,
.device_group = generic_device_group,
#ifdef CONFIG_ARM
.set_platform_dma_ops = exynos_iommu_set_platform_dma,
#endif
.probe_device = exynos_iommu_probe_device,
.release_device = exynos_iommu_release_device,
.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
.of_xlate = exynos_iommu_of_xlate,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = exynos_iommu_attach_device,
.map = exynos_iommu_map,
.unmap = exynos_iommu_unmap,
.map_pages = exynos_iommu_map,
.unmap_pages = exynos_iommu_unmap,
.iova_to_phys = exynos_iommu_iova_to_phys,
.free = exynos_iommu_domain_free,
}

41
drivers/iommu/fsl_pamu_domain.c
View File

@ -196,6 +196,13 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
{
struct fsl_dma_domain *dma_domain;
/*
* FIXME: This isn't creating an unmanaged domain since the
* default_domain_ops do not have any map/unmap function it doesn't meet
* the requirements for __IOMMU_DOMAIN_PAGING. The only purpose seems to
* allow drivers/soc/fsl/qbman/qman_portal.c to do
* fsl_pamu_configure_l1_stash()
*/
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
@ -283,15 +290,33 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
return ret;
}
static void fsl_pamu_set_platform_dma(struct device *dev)
/*
* FIXME: fsl/pamu is completely broken in terms of how it works with the iommu
* API. Immediately after probe the HW is left in an IDENTITY translation and
* the driver provides a non-working UNMANAGED domain that it can switch over
* to. However it cannot switch back to an IDENTITY translation, instead it
* switches to what looks like BLOCKING.
*/
static int fsl_pamu_platform_attach(struct iommu_domain *platform_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
struct fsl_dma_domain *dma_domain;
const u32 *prop;
int len;
struct pci_dev *pdev = NULL;
struct pci_controller *pci_ctl;
/*
* Hack to keep things working as they always have, only leaving an
* UNMANAGED domain makes it BLOCKING.
*/
if (domain == platform_domain || !domain ||
domain->type != IOMMU_DOMAIN_UNMANAGED)
return 0;
dma_domain = to_fsl_dma_domain(domain);
/*
* Use LIODN of the PCI controller while detaching a
* PCI device.
@ -312,8 +337,18 @@ static void fsl_pamu_set_platform_dma(struct device *dev)
detach_device(dev, dma_domain);
else
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
return 0;
}
static struct iommu_domain_ops fsl_pamu_platform_ops = {
.attach_dev = fsl_pamu_platform_attach,
};
static struct iommu_domain fsl_pamu_platform_domain = {
.type = IOMMU_DOMAIN_PLATFORM,
.ops = &fsl_pamu_platform_ops,
};
/* Set the domain stash attribute */
int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu)
{
@ -395,11 +430,11 @@ static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
}
static const struct iommu_ops fsl_pamu_ops = {
.default_domain = &fsl_pamu_platform_domain,
.capable = fsl_pamu_capable,
.domain_alloc = fsl_pamu_domain_alloc,
.probe_device = fsl_pamu_probe_device,
.device_group = fsl_pamu_device_group,
.set_platform_dma_ops = fsl_pamu_set_platform_dma,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = fsl_pamu_attach_device,
.iova_to_phys = fsl_pamu_iova_to_phys,

215
drivers/iommu/intel/debugfs.c
View File

@ -111,6 +111,8 @@ static const struct iommu_regset iommu_regs_64[] = {
IOMMU_REGSET_ENTRY(VCRSP),
};
static struct dentry *intel_iommu_debug;
static int iommu_regset_show(struct seq_file *m, void *unused)
{
struct dmar_drhd_unit *drhd;
@ -311,9 +313,14 @@ static inline unsigned long level_to_directory_size(int level)
static inline void
dump_page_info(struct seq_file *m, unsigned long iova, u64 *path)
{
seq_printf(m, "0x%013lx |\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\t0x%016llx\n",
iova >> VTD_PAGE_SHIFT, path[5], path[4],
path[3], path[2], path[1]);
seq_printf(m, "0x%013lx |\t0x%016llx\t0x%016llx\t0x%016llx",
iova >> VTD_PAGE_SHIFT, path[5], path[4], path[3]);
if (path[2]) {
seq_printf(m, "\t0x%016llx", path[2]);
if (path[1])
seq_printf(m, "\t0x%016llx", path[1]);
}
seq_putc(m, '\n');
}
static void pgtable_walk_level(struct seq_file *m, struct dma_pte *pde,
@ -340,58 +347,140 @@ static void pgtable_walk_level(struct seq_file *m, struct dma_pte *pde,
}
}
static int __show_device_domain_translation(struct device *dev, void *data)
static int domain_translation_struct_show(struct seq_file *m,
struct device_domain_info *info,
ioasid_t pasid)
{
struct dmar_domain *domain;
struct seq_file *m = data;
u64 path[6] = { 0 };
bool scalable, found = false;
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
u16 devfn, bus, seg;
domain = to_dmar_domain(iommu_get_domain_for_dev(dev));
if (!domain)
return 0;
bus = info->bus;
devfn = info->devfn;
seg = info->segment;
seq_printf(m, "Device %s @0x%llx\n", dev_name(dev),
(u64)virt_to_phys(domain->pgd));
seq_puts(m, "IOVA_PFN\t\tPML5E\t\t\tPML4E\t\t\tPDPE\t\t\tPDE\t\t\tPTE\n");
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
struct context_entry *context;
u64 pgd, path[6] = { 0 };
u32 sts, agaw;
pgtable_walk_level(m, domain->pgd, domain->agaw + 2, 0, path);
seq_putc(m, '\n');
if (seg != iommu->segment)
continue;
/* Don't iterate */
return 1;
}
sts = dmar_readl(iommu->reg + DMAR_GSTS_REG);
if (!(sts & DMA_GSTS_TES)) {
seq_printf(m, "DMA Remapping is not enabled on %s\n",
iommu->name);
continue;
}
if (dmar_readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT)
scalable = true;
else
scalable = false;
static int show_device_domain_translation(struct device *dev, void *data)
{
struct iommu_group *group;
group = iommu_group_get(dev);
if (group) {
/*
* The group->mutex is held across the callback, which will
* block calls to iommu_attach/detach_group/device. Hence,
* The iommu->lock is held across the callback, which will
* block calls to domain_attach/domain_detach. Hence,
* the domain of the device will not change during traversal.
*
* All devices in an iommu group share a single domain, hence
* we only dump the domain of the first device. Even though,
* this code still possibly races with the iommu_unmap()
* Traversing page table possibly races with the iommu_unmap()
* interface. This could be solved by RCU-freeing the page
* table pages in the iommu_unmap() path.
*/
iommu_group_for_each_dev(group, data,
__show_device_domain_translation);
iommu_group_put(group);
spin_lock(&iommu->lock);
context = iommu_context_addr(iommu, bus, devfn, 0);
if (!context || !context_present(context))
goto iommu_unlock;
if (scalable) { /* scalable mode */
struct pasid_entry *pasid_tbl, *pasid_tbl_entry;
struct pasid_dir_entry *dir_tbl, *dir_entry;
u16 dir_idx, tbl_idx, pgtt;
u64 pasid_dir_ptr;
pasid_dir_ptr = context->lo & VTD_PAGE_MASK;
/* Dump specified device domain mappings with PASID. */
dir_idx = pasid >> PASID_PDE_SHIFT;
tbl_idx = pasid & PASID_PTE_MASK;
dir_tbl = phys_to_virt(pasid_dir_ptr);
dir_entry = &dir_tbl[dir_idx];
pasid_tbl = get_pasid_table_from_pde(dir_entry);
if (!pasid_tbl)
goto iommu_unlock;
pasid_tbl_entry = &pasid_tbl[tbl_idx];
if (!pasid_pte_is_present(pasid_tbl_entry))
goto iommu_unlock;
/*
* According to PASID Granular Translation Type(PGTT),
* get the page table pointer.
*/
pgtt = (u16)(pasid_tbl_entry->val[0] & GENMASK_ULL(8, 6)) >> 6;
agaw = (u8)(pasid_tbl_entry->val[0] & GENMASK_ULL(4, 2)) >> 2;
switch (pgtt) {
case PASID_ENTRY_PGTT_FL_ONLY:
pgd = pasid_tbl_entry->val[2];
break;
case PASID_ENTRY_PGTT_SL_ONLY:
case PASID_ENTRY_PGTT_NESTED:
pgd = pasid_tbl_entry->val[0];
break;
default:
goto iommu_unlock;
}
pgd &= VTD_PAGE_MASK;
} else { /* legacy mode */
pgd = context->lo & VTD_PAGE_MASK;
agaw = context->hi & 7;
}
seq_printf(m, "Device %04x:%02x:%02x.%x ",
iommu->segment, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
if (scalable)
seq_printf(m, "with pasid %x @0x%llx\n", pasid, pgd);
else
seq_printf(m, "@0x%llx\n", pgd);
seq_printf(m, "%-17s\t%-18s\t%-18s\t%-18s\t%-18s\t%-s\n",
"IOVA_PFN", "PML5E", "PML4E", "PDPE", "PDE", "PTE");
pgtable_walk_level(m, phys_to_virt(pgd), agaw + 2, 0, path);
found = true;
iommu_unlock:
spin_unlock(&iommu->lock);
if (found)
break;
}
rcu_read_unlock();
return 0;
}
static int domain_translation_struct_show(struct seq_file *m, void *unused)
static int dev_domain_translation_struct_show(struct seq_file *m, void *unused)
{
return bus_for_each_dev(&pci_bus_type, NULL, m,
show_device_domain_translation);
struct device_domain_info *info = (struct device_domain_info *)m->private;
return domain_translation_struct_show(m, info, IOMMU_NO_PASID);
}
DEFINE_SHOW_ATTRIBUTE(domain_translation_struct);
DEFINE_SHOW_ATTRIBUTE(dev_domain_translation_struct);
static int pasid_domain_translation_struct_show(struct seq_file *m, void *unused)
{
struct dev_pasid_info *dev_pasid = (struct dev_pasid_info *)m->private;
struct device_domain_info *info = dev_iommu_priv_get(dev_pasid->dev);
return domain_translation_struct_show(m, info, dev_pasid->pasid);
}
DEFINE_SHOW_ATTRIBUTE(pasid_domain_translation_struct);
static void invalidation_queue_entry_show(struct seq_file *m,
struct intel_iommu *iommu)
@ -666,16 +755,12 @@ static const struct file_operations dmar_perf_latency_fops = {
void __init intel_iommu_debugfs_init(void)
{
struct dentry *intel_iommu_debug = debugfs_create_dir("intel",
iommu_debugfs_dir);
intel_iommu_debug = debugfs_create_dir("intel", iommu_debugfs_dir);
debugfs_create_file("iommu_regset", 0444, intel_iommu_debug, NULL,
&iommu_regset_fops);
debugfs_create_file("dmar_translation_struct", 0444, intel_iommu_debug,
NULL, &dmar_translation_struct_fops);
debugfs_create_file("domain_translation_struct", 0444,
intel_iommu_debug, NULL,
&domain_translation_struct_fops);
debugfs_create_file("invalidation_queue", 0444, intel_iommu_debug,
NULL, &invalidation_queue_fops);
#ifdef CONFIG_IRQ_REMAP
@ -685,3 +770,51 @@ void __init intel_iommu_debugfs_init(void)
debugfs_create_file("dmar_perf_latency", 0644, intel_iommu_debug,
NULL, &dmar_perf_latency_fops);
}
/*
* Create a debugfs directory for each device, and then create a
* debugfs file in this directory for users to dump the page table
* of the default domain. e.g.
* /sys/kernel/debug/iommu/intel/0000:00:01.0/domain_translation_struct
*/
void intel_iommu_debugfs_create_dev(struct device_domain_info *info)
{
info->debugfs_dentry = debugfs_create_dir(dev_name(info->dev), intel_iommu_debug);
debugfs_create_file("domain_translation_struct", 0444, info->debugfs_dentry,
info, &dev_domain_translation_struct_fops);
}
/* Remove the device debugfs directory. */
void intel_iommu_debugfs_remove_dev(struct device_domain_info *info)
{
debugfs_remove_recursive(info->debugfs_dentry);
}
/*
* Create a debugfs directory per pair of {device, pasid}, then create the
* corresponding debugfs file in this directory for users to dump its page
* table. e.g.
* /sys/kernel/debug/iommu/intel/0000:00:01.0/1/domain_translation_struct
*
* The debugfs only dumps the page tables whose mappings are created and
* destroyed by the iommu_map/unmap() interfaces. Check the mapping type
* of the domain before creating debugfs directory.
*/
void intel_iommu_debugfs_create_dev_pasid(struct dev_pasid_info *dev_pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev_pasid->dev);
char dir_name[10];
sprintf(dir_name, "%x", dev_pasid->pasid);
dev_pasid->debugfs_dentry = debugfs_create_dir(dir_name, info->debugfs_dentry);
debugfs_create_file("domain_translation_struct", 0444, dev_pasid->debugfs_dentry,
dev_pasid, &pasid_domain_translation_struct_fops);
}
/* Remove the device pasid debugfs directory. */
void intel_iommu_debugfs_remove_dev_pasid(struct dev_pasid_info *dev_pasid)
{
debugfs_remove_recursive(dev_pasid->debugfs_dentry);
}

19
drivers/iommu/intel/iommu.c
View File

@ -4016,9 +4016,9 @@ static int blocking_domain_attach_dev(struct iommu_domain *domain,
}
static struct iommu_domain blocking_domain = {
.type = IOMMU_DOMAIN_BLOCKED,
.ops = &(const struct iommu_domain_ops) {
.attach_dev = blocking_domain_attach_dev,
.free = intel_iommu_domain_free
}
};
@ -4028,8 +4028,6 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
struct iommu_domain *domain;
switch (type) {
case IOMMU_DOMAIN_BLOCKED:
return &blocking_domain;
case IOMMU_DOMAIN_DMA:
case IOMMU_DOMAIN_UNMANAGED:
dmar_domain = alloc_domain(type);
@ -4111,7 +4109,7 @@ intel_iommu_domain_alloc_user(struct device *dev, u32 flags,
static void intel_iommu_domain_free(struct iommu_domain *domain)
{
if (domain != &si_domain->domain && domain != &blocking_domain)
if (domain != &si_domain->domain)
domain_exit(to_dmar_domain(domain));
}
@ -4465,6 +4463,8 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
}
}
intel_iommu_debugfs_create_dev(info);
return &iommu->iommu;
}
@ -4474,6 +4474,7 @@ static void intel_iommu_release_device(struct device *dev)
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
intel_iommu_debugfs_remove_dev(info);
dev_iommu_priv_set(dev, NULL);
kfree(info);
set_dma_ops(dev, NULL);
@ -4718,8 +4719,8 @@ static bool risky_device(struct pci_dev *pdev)
return false;
}
static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
unsigned long pages = aligned_nrpages(iova, size);
@ -4729,6 +4730,7 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
xa_for_each(&dmar_domain->iommu_array, i, info)
__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
return 0;
}
static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
@ -4766,6 +4768,7 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
spin_unlock_irqrestore(&dmar_domain->lock, flags);
domain_detach_iommu(dmar_domain, iommu);
intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
kfree(dev_pasid);
out_tear_down:
intel_pasid_tear_down_entry(iommu, dev, pasid, false);
@ -4821,6 +4824,9 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
spin_unlock_irqrestore(&dmar_domain->lock, flags);
if (domain->type & __IOMMU_DOMAIN_PAGING)
intel_iommu_debugfs_create_dev_pasid(dev_pasid);
return 0;
out_detach_iommu:
domain_detach_iommu(dmar_domain, iommu);
@ -4925,6 +4931,7 @@ const struct iommu_dirty_ops intel_dirty_ops = {
};
const struct iommu_ops intel_iommu_ops = {
.blocked_domain = &blocking_domain,
.capable = intel_iommu_capable,
.hw_info = intel_iommu_hw_info,
.domain_alloc = intel_iommu_domain_alloc,

14
drivers/iommu/intel/iommu.h
View File

@ -749,12 +749,18 @@ struct device_domain_info {
struct intel_iommu *iommu; /* IOMMU used by this device */
struct dmar_domain *domain; /* pointer to domain */
struct pasid_table *pasid_table; /* pasid table */
#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
struct dentry *debugfs_dentry; /* pointer to device directory dentry */
#endif
};
struct dev_pasid_info {
struct list_head link_domain; /* link to domain siblings */
struct device *dev;
ioasid_t pasid;
#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
struct dentry *debugfs_dentry; /* pointer to pasid directory dentry */
#endif
};
static inline void __iommu_flush_cache(
@ -935,8 +941,16 @@ static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid
#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
void intel_iommu_debugfs_init(void);
void intel_iommu_debugfs_create_dev(struct device_domain_info *info);
void intel_iommu_debugfs_remove_dev(struct device_domain_info *info);
void intel_iommu_debugfs_create_dev_pasid(struct dev_pasid_info *dev_pasid);
void intel_iommu_debugfs_remove_dev_pasid(struct dev_pasid_info *dev_pasid);
#else
static inline void intel_iommu_debugfs_init(void) {}
static inline void intel_iommu_debugfs_create_dev(struct device_domain_info *info) {}
static inline void intel_iommu_debugfs_remove_dev(struct device_domain_info *info) {}
static inline void intel_iommu_debugfs_create_dev_pasid(struct dev_pasid_info *dev_pasid) {}
static inline void intel_iommu_debugfs_remove_dev_pasid(struct dev_pasid_info *dev_pasid) {}
#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */
extern const struct attribute_group *intel_iommu_groups[];

465
drivers/iommu/iommu.c
View File

@ -37,7 +37,6 @@
#include "iommu-priv.h"
#include "iommu-sva.h"
#include "iommu-priv.h"
static struct kset *iommu_group_kset;
static DEFINE_IDA(iommu_group_ida);
@ -96,8 +95,8 @@ static const char * const iommu_group_resv_type_string[] = {
static int iommu_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data);
static void iommu_release_device(struct device *dev);
static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
unsigned type);
static struct iommu_domain *
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type);
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
static int __iommu_attach_group(struct iommu_domain *domain,
@ -184,6 +183,8 @@ static const char *iommu_domain_type_str(unsigned int t)
case IOMMU_DOMAIN_DMA:
case IOMMU_DOMAIN_DMA_FQ:
return "Translated";
case IOMMU_DOMAIN_PLATFORM:
return "Platform";
default:
return "Unknown";
}
@ -290,6 +291,10 @@ void iommu_device_unregister(struct iommu_device *iommu)
spin_lock(&iommu_device_lock);
list_del(&iommu->list);
spin_unlock(&iommu_device_lock);
/* Pairs with the alloc in generic_single_device_group() */
iommu_group_put(iommu->singleton_group);
iommu->singleton_group = NULL;
}
EXPORT_SYMBOL_GPL(iommu_device_unregister);
@ -404,6 +409,7 @@ static int iommu_init_device(struct device *dev, const struct iommu_ops *ops)
ret = PTR_ERR(iommu_dev);
goto err_module_put;
}
dev->iommu->iommu_dev = iommu_dev;
ret = iommu_device_link(iommu_dev, dev);
if (ret)
@ -418,7 +424,6 @@ static int iommu_init_device(struct device *dev, const struct iommu_ops *ops)
}
dev->iommu_group = group;
dev->iommu->iommu_dev = iommu_dev;
dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
if (ops->is_attach_deferred)
dev->iommu->attach_deferred = ops->is_attach_deferred(dev);
@ -432,6 +437,7 @@ err_release:
err_module_put:
module_put(ops->owner);
err_free:
dev->iommu->iommu_dev = NULL;
dev_iommu_free(dev);
return ret;
}
@ -1636,6 +1642,27 @@ struct iommu_group *generic_device_group(struct device *dev)
}
EXPORT_SYMBOL_GPL(generic_device_group);
/*
* Generic device_group call-back function. It just allocates one
* iommu-group per iommu driver instance shared by every device
* probed by that iommu driver.
*/
struct iommu_group *generic_single_device_group(struct device *dev)
{
struct iommu_device *iommu = dev->iommu->iommu_dev;
if (!iommu->singleton_group) {
struct iommu_group *group;
group = iommu_group_alloc();
if (IS_ERR(group))
return group;
iommu->singleton_group = group;
}
return iommu_group_ref_get(iommu->singleton_group);
}
EXPORT_SYMBOL_GPL(generic_single_device_group);
/*
* Use standard PCI bus topology, isolation features, and DMA alias quirks
* to find or create an IOMMU group for a device.
@ -1717,26 +1744,29 @@ struct iommu_group *fsl_mc_device_group(struct device *dev)
}
EXPORT_SYMBOL_GPL(fsl_mc_device_group);
static int iommu_get_def_domain_type(struct device *dev)
{
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
return IOMMU_DOMAIN_DMA;
if (ops->def_domain_type)
return ops->def_domain_type(dev);
return 0;
}
static struct iommu_domain *
__iommu_group_alloc_default_domain(const struct bus_type *bus,
struct iommu_group *group, int req_type)
__iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
{
if (group->default_domain && group->default_domain->type == req_type)
return group->default_domain;
return __iommu_domain_alloc(bus, req_type);
return __iommu_group_domain_alloc(group, req_type);
}
/*
* Returns the iommu_ops for the devices in an iommu group.
*
* It is assumed that all devices in an iommu group are managed by a single
* IOMMU unit. Therefore, this returns the dev_iommu_ops of the first device
* in the group.
*/
static const struct iommu_ops *group_iommu_ops(struct iommu_group *group)
{
struct group_device *device =
list_first_entry(&group->devices, struct group_device, list);
lockdep_assert_held(&group->mutex);
return dev_iommu_ops(device->dev);
}
/*
@ -1746,25 +1776,34 @@ __iommu_group_alloc_default_domain(const struct bus_type *bus,
static struct iommu_domain *
iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
{
const struct bus_type *bus =
list_first_entry(&group->devices, struct group_device, list)
->dev->bus;
const struct iommu_ops *ops = group_iommu_ops(group);
struct iommu_domain *dom;
lockdep_assert_held(&group->mutex);
/*
* Allow legacy drivers to specify the domain that will be the default
* domain. This should always be either an IDENTITY/BLOCKED/PLATFORM
* domain. Do not use in new drivers.
*/
if (ops->default_domain) {
if (req_type)
return NULL;
return ops->default_domain;
}
if (req_type)
return __iommu_group_alloc_default_domain(bus, group, req_type);
return __iommu_group_alloc_default_domain(group, req_type);
/* The driver gave no guidance on what type to use, try the default */
dom = __iommu_group_alloc_default_domain(bus, group, iommu_def_domain_type);
dom = __iommu_group_alloc_default_domain(group, iommu_def_domain_type);
if (dom)
return dom;
/* Otherwise IDENTITY and DMA_FQ defaults will try DMA */
if (iommu_def_domain_type == IOMMU_DOMAIN_DMA)
return NULL;
dom = __iommu_group_alloc_default_domain(bus, group, IOMMU_DOMAIN_DMA);
dom = __iommu_group_alloc_default_domain(group, IOMMU_DOMAIN_DMA);
if (!dom)
return NULL;
@ -1808,40 +1847,109 @@ static int iommu_bus_notifier(struct notifier_block *nb,
return 0;
}
/* A target_type of 0 will select the best domain type and cannot fail */
/*
* Combine the driver's chosen def_domain_type across all the devices in a
* group. Drivers must give a consistent result.
*/
static int iommu_get_def_domain_type(struct iommu_group *group,
struct device *dev, int cur_type)
{
const struct iommu_ops *ops = group_iommu_ops(group);
int type;
if (!ops->def_domain_type)
return cur_type;
type = ops->def_domain_type(dev);
if (!type || cur_type == type)
return cur_type;
if (!cur_type)
return type;
dev_err_ratelimited(
dev,
"IOMMU driver error, requesting conflicting def_domain_type, %s and %s, for devices in group %u.\n",
iommu_domain_type_str(cur_type), iommu_domain_type_str(type),
group->id);
/*
* Try to recover, drivers are allowed to force IDENITY or DMA, IDENTITY
* takes precedence.
*/
if (type == IOMMU_DOMAIN_IDENTITY)
return type;
return cur_type;
}
/*
* A target_type of 0 will select the best domain type. 0 can be returned in
* this case meaning the global default should be used.
*/
static int iommu_get_default_domain_type(struct iommu_group *group,
int target_type)
{
int best_type = target_type;
struct device *untrusted = NULL;
struct group_device *gdev;
struct device *last_dev;
int driver_type = 0;
lockdep_assert_held(&group->mutex);
for_each_group_device(group, gdev) {
unsigned int type = iommu_get_def_domain_type(gdev->dev);
if (best_type && type && best_type != type) {
if (target_type) {
dev_err_ratelimited(
gdev->dev,
"Device cannot be in %s domain\n",
iommu_domain_type_str(target_type));
return -1;
}
dev_warn(
gdev->dev,
"Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
iommu_domain_type_str(type), dev_name(last_dev),
iommu_domain_type_str(best_type));
return 0;
}
if (!best_type)
best_type = type;
last_dev = gdev->dev;
/*
* ARM32 drivers supporting CONFIG_ARM_DMA_USE_IOMMU can declare an
* identity_domain and it will automatically become their default
* domain. Later on ARM_DMA_USE_IOMMU will install its UNMANAGED domain.
* Override the selection to IDENTITY.
*/
if (IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)) {
static_assert(!(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) &&
IS_ENABLED(CONFIG_IOMMU_DMA)));
driver_type = IOMMU_DOMAIN_IDENTITY;
}
return best_type;
for_each_group_device(group, gdev) {
driver_type = iommu_get_def_domain_type(group, gdev->dev,
driver_type);
if (dev_is_pci(gdev->dev) && to_pci_dev(gdev->dev)->untrusted) {
/*
* No ARM32 using systems will set untrusted, it cannot
* work.
*/
if (WARN_ON(IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)))
return -1;
untrusted = gdev->dev;
}
}
/*
* If the common dma ops are not selected in kconfig then we cannot use
* IOMMU_DOMAIN_DMA at all. Force IDENTITY if nothing else has been
* selected.
*/
if (!IS_ENABLED(CONFIG_IOMMU_DMA)) {
if (WARN_ON(driver_type == IOMMU_DOMAIN_DMA))
return -1;
if (!driver_type)
driver_type = IOMMU_DOMAIN_IDENTITY;
}
if (untrusted) {
if (driver_type && driver_type != IOMMU_DOMAIN_DMA) {
dev_err_ratelimited(
untrusted,
"Device is not trusted, but driver is overriding group %u to %s, refusing to probe.\n",
group->id, iommu_domain_type_str(driver_type));
return -1;
}
driver_type = IOMMU_DOMAIN_DMA;
}
if (target_type) {
if (driver_type && target_type != driver_type)
return -1;
return target_type;
}
return driver_type;
}
static void iommu_group_do_probe_finalize(struct device *dev)
@ -1970,16 +2078,24 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
unsigned type)
static struct iommu_domain *__iommu_domain_alloc(const struct iommu_ops *ops,
struct device *dev,
unsigned int type)
{
struct iommu_domain *domain;
unsigned int alloc_type = type & IOMMU_DOMAIN_ALLOC_FLAGS;
if (bus == NULL || bus->iommu_ops == NULL)
if (alloc_type == IOMMU_DOMAIN_IDENTITY && ops->identity_domain)
return ops->identity_domain;
else if (alloc_type == IOMMU_DOMAIN_BLOCKED && ops->blocked_domain)
return ops->blocked_domain;
else if (type & __IOMMU_DOMAIN_PAGING && ops->domain_alloc_paging)
domain = ops->domain_alloc_paging(dev);
else if (ops->domain_alloc)
domain = ops->domain_alloc(alloc_type);
else
return NULL;
domain = bus->iommu_ops->domain_alloc(alloc_type);
if (!domain)
return NULL;
@ -1989,10 +2105,10 @@ static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
* may override this later
*/
if (!domain->pgsize_bitmap)
domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
domain->pgsize_bitmap = ops->pgsize_bitmap;
if (!domain->ops)
domain->ops = bus->iommu_ops->default_domain_ops;
domain->ops = ops->default_domain_ops;
if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) {
iommu_domain_free(domain);
@ -2001,9 +2117,22 @@ static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus,
return domain;
}
static struct iommu_domain *
__iommu_group_domain_alloc(struct iommu_group *group, unsigned int type)
{
struct device *dev =
list_first_entry(&group->devices, struct group_device, list)
->dev;
return __iommu_domain_alloc(group_iommu_ops(group), dev, type);
}
struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
{
return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
if (bus == NULL || bus->iommu_ops == NULL)
return NULL;
return __iommu_domain_alloc(bus->iommu_ops, NULL,
IOMMU_DOMAIN_UNMANAGED);
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
@ -2012,7 +2141,8 @@ void iommu_domain_free(struct iommu_domain *domain)
if (domain->type == IOMMU_DOMAIN_SVA)
mmdrop(domain->mm);
iommu_put_dma_cookie(domain);
domain->ops->free(domain);
if (domain->ops->free)
domain->ops->free(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);
@ -2062,10 +2192,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
*/
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_group *group;
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
int ret;
group = iommu_group_get(dev);
if (!group)
return -ENODEV;
@ -2082,8 +2212,6 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device);
@ -2098,9 +2226,9 @@ int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain)
void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_group *group;
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
group = iommu_group_get(dev);
if (!group)
return;
@ -2112,24 +2240,18 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);
struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
{
struct iommu_domain *domain;
struct iommu_group *group;
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
group = iommu_group_get(dev);
if (!group)
return NULL;
domain = group->domain;
iommu_group_put(group);
return domain;
return group->domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
@ -2275,21 +2397,8 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group,
if (group->domain == new_domain)
return 0;
/*
* New drivers should support default domains, so set_platform_dma()
* op will never be called. Otherwise the NULL domain represents some
* platform specific behavior.
*/
if (!new_domain) {
for_each_group_device(group, gdev) {
const struct iommu_ops *ops = dev_iommu_ops(gdev->dev);
if (!WARN_ON(!ops->set_platform_dma_ops))
ops->set_platform_dma_ops(gdev->dev);
}
group->domain = NULL;
return 0;
}
if (WARN_ON(!new_domain))
return -EINVAL;
/*
* Changing the domain is done by calling attach_dev() on the new
@ -2325,19 +2434,15 @@ err_revert:
*/
last_gdev = gdev;
for_each_group_device(group, gdev) {
const struct iommu_ops *ops = dev_iommu_ops(gdev->dev);
/*
* If set_platform_dma_ops is not present a NULL domain can
* happen only for first probe, in which case we leave
* group->domain as NULL and let release clean everything up.
* A NULL domain can happen only for first probe, in which case
* we leave group->domain as NULL and let release clean
* everything up.
*/
if (group->domain)
WARN_ON(__iommu_device_set_domain(
group, gdev->dev, group->domain,
IOMMU_SET_DOMAIN_MUST_SUCCEED));
else if (ops->set_platform_dma_ops)
ops->set_platform_dma_ops(gdev->dev);
if (gdev == last_gdev)
break;
}
@ -2418,30 +2523,6 @@ out_set_count:
return pgsize;
}
static int __iommu_map_pages(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot,
gfp_t gfp, size_t *mapped)
{
const struct iommu_domain_ops *ops = domain->ops;
size_t pgsize, count;
int ret;
pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
iova, &paddr, pgsize, count);
if (ops->map_pages) {
ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
gfp, mapped);
} else {
ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
*mapped = ret ? 0 : pgsize;
}
return ret;
}
static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
@ -2452,13 +2533,12 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t orig_paddr = paddr;
int ret = 0;
if (unlikely(!(ops->map || ops->map_pages) ||
domain->pgsize_bitmap == 0UL))
return -ENODEV;
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
return -EINVAL;
if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL))
return -ENODEV;
/* find out the minimum page size supported */
min_pagesz = 1 << __ffs(domain->pgsize_bitmap);
@ -2476,10 +2556,14 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova,
pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
while (size) {
size_t mapped = 0;
size_t pgsize, count, mapped = 0;
ret = __iommu_map_pages(domain, iova, paddr, size, prot, gfp,
&mapped);
pgsize = iommu_pgsize(domain, iova, paddr, size, &count);
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n",
iova, &paddr, pgsize, count);
ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot,
gfp, &mapped);
/*
* Some pages may have been mapped, even if an error occurred,
* so we should account for those so they can be unmapped.
@ -2516,26 +2600,22 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
return -EINVAL;
ret = __iommu_map(domain, iova, paddr, size, prot, gfp);
if (ret == 0 && ops->iotlb_sync_map)
ops->iotlb_sync_map(domain, iova, size);
if (ret == 0 && ops->iotlb_sync_map) {
ret = ops->iotlb_sync_map(domain, iova, size);
if (ret)
goto out_err;
}
return ret;
out_err:
/* undo mappings already done */
iommu_unmap(domain, iova, size);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);
static size_t __iommu_unmap_pages(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *iotlb_gather)
{
const struct iommu_domain_ops *ops = domain->ops;
size_t pgsize, count;
pgsize = iommu_pgsize(domain, iova, iova, size, &count);
return ops->unmap_pages ?
ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather) :
ops->unmap(domain, iova, pgsize, iotlb_gather);
}
static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *iotlb_gather)
@ -2545,11 +2625,10 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long orig_iova = iova;
unsigned int min_pagesz;
if (unlikely(!(ops->unmap || ops->unmap_pages) ||
domain->pgsize_bitmap == 0UL))
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
return 0;
if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL))
return 0;
/* find out the minimum page size supported */
@ -2573,9 +2652,10 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
* or we hit an area that isn't mapped.
*/
while (unmapped < size) {
unmapped_page = __iommu_unmap_pages(domain, iova,
size - unmapped,
iotlb_gather);
size_t pgsize, count;
pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count);
unmapped_page = ops->unmap_pages(domain, iova, pgsize, count, iotlb_gather);
if (!unmapped_page)
break;
@ -2658,8 +2738,11 @@ next:
sg = sg_next(sg);
}
if (ops->iotlb_sync_map)
ops->iotlb_sync_map(domain, iova, mapped);
if (ops->iotlb_sync_map) {
ret = ops->iotlb_sync_map(domain, iova, mapped);
if (ret)
goto out_err;
}
return mapped;
out_err:
@ -2957,21 +3040,9 @@ static int iommu_setup_default_domain(struct iommu_group *group,
if (req_type < 0)
return -EINVAL;
/*
* There are still some drivers which don't support default domains, so
* we ignore the failure and leave group->default_domain NULL.
*
* We assume that the iommu driver starts up the device in
* 'set_platform_dma_ops' mode if it does not support default domains.
*/
dom = iommu_group_alloc_default_domain(group, req_type);
if (!dom) {
/* Once in default_domain mode we never leave */
if (group->default_domain)
return -ENODEV;
group->default_domain = NULL;
return 0;
}
if (!dom)
return -ENODEV;
if (group->default_domain == dom)
return 0;
@ -3114,24 +3185,6 @@ out_unlock:
return ret ?: count;
}
static bool iommu_is_default_domain(struct iommu_group *group)
{
if (group->domain == group->default_domain)
return true;
/*
* If the default domain was set to identity and it is still an identity
* domain then we consider this a pass. This happens because of
* amd_iommu_init_device() replacing the default idenytity domain with an
* identity domain that has a different configuration for AMDGPU.
*/
if (group->default_domain &&
group->default_domain->type == IOMMU_DOMAIN_IDENTITY &&
group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY)
return true;
return false;
}
/**
* iommu_device_use_default_domain() - Device driver wants to handle device
* DMA through the kernel DMA API.
@ -3142,7 +3195,8 @@ static bool iommu_is_default_domain(struct iommu_group *group)
*/
int iommu_device_use_default_domain(struct device *dev)
{
struct iommu_group *group = iommu_group_get(dev);
/* Caller is the driver core during the pre-probe path */
struct iommu_group *group = dev->iommu_group;
int ret = 0;
if (!group)
@ -3150,7 +3204,7 @@ int iommu_device_use_default_domain(struct device *dev)
mutex_lock(&group->mutex);
if (group->owner_cnt) {
if (group->owner || !iommu_is_default_domain(group) ||
if (group->domain != group->default_domain || group->owner ||
!xa_empty(&group->pasid_array)) {
ret = -EBUSY;
goto unlock_out;
@ -3161,8 +3215,6 @@ int iommu_device_use_default_domain(struct device *dev)
unlock_out:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
@ -3176,7 +3228,8 @@ unlock_out:
*/
void iommu_device_unuse_default_domain(struct device *dev)
{
struct iommu_group *group = iommu_group_get(dev);
/* Caller is the driver core during the post-probe path */
struct iommu_group *group = dev->iommu_group;
if (!group)
return;
@ -3186,26 +3239,22 @@ void iommu_device_unuse_default_domain(struct device *dev)
group->owner_cnt--;
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)
{
struct group_device *dev =
list_first_entry(&group->devices, struct group_device, list);
if (group->blocking_domain)
return 0;
group->blocking_domain =
__iommu_domain_alloc(dev->dev->bus, IOMMU_DOMAIN_BLOCKED);
__iommu_group_domain_alloc(group, IOMMU_DOMAIN_BLOCKED);
if (!group->blocking_domain) {
/*
* For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED
* create an empty domain instead.
*/
group->blocking_domain = __iommu_domain_alloc(
dev->dev->bus, IOMMU_DOMAIN_UNMANAGED);
group->blocking_domain = __iommu_group_domain_alloc(
group, IOMMU_DOMAIN_UNMANAGED);
if (!group->blocking_domain)
return -EINVAL;
}
@ -3273,13 +3322,13 @@ EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner);
*/
int iommu_device_claim_dma_owner(struct device *dev, void *owner)
{
struct iommu_group *group;
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
int ret = 0;
if (WARN_ON(!owner))
return -EINVAL;
group = iommu_group_get(dev);
if (!group)
return -ENODEV;
@ -3296,8 +3345,6 @@ int iommu_device_claim_dma_owner(struct device *dev, void *owner)
ret = __iommu_take_dma_ownership(group, owner);
unlock_out:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_device_claim_dma_owner);
@ -3335,7 +3382,8 @@ EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner);
*/
void iommu_device_release_dma_owner(struct device *dev)
{
struct iommu_group *group = iommu_group_get(dev);
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
mutex_lock(&group->mutex);
if (group->owner_cnt > 1)
@ -3343,7 +3391,6 @@ void iommu_device_release_dma_owner(struct device *dev)
else
__iommu_release_dma_ownership(group);
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_device_release_dma_owner);
@ -3404,14 +3451,14 @@ static void __iommu_remove_group_pasid(struct iommu_group *group,
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
struct iommu_group *group;
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
void *curr;
int ret;
if (!domain->ops->set_dev_pasid)
return -EOPNOTSUPP;
group = iommu_group_get(dev);
if (!group)
return -ENODEV;
@ -3429,8 +3476,6 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
}
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
@ -3447,14 +3492,13 @@ EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
ioasid_t pasid)
{
struct iommu_group *group = iommu_group_get(dev);
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
mutex_lock(&group->mutex);
__iommu_remove_group_pasid(group, pasid);
WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
@ -3476,10 +3520,10 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
ioasid_t pasid,
unsigned int type)
{
/* Caller must be a probed driver on dev */
struct iommu_group *group = dev->iommu_group;
struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get(dev);
if (!group)
return NULL;
@ -3488,7 +3532,6 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
if (type && domain && domain->type != type)
domain = ERR_PTR(-EBUSY);
xa_unlock(&group->pasid_array);
iommu_group_put(group);
return domain;
}

31
drivers/iommu/iommufd/selftest.c
View File

@ -123,10 +123,6 @@ struct selftest_obj {
};
};
static void mock_domain_blocking_free(struct iommu_domain *domain)
{
}
static int mock_domain_nop_attach(struct iommu_domain *domain,
struct device *dev)
{
@ -139,7 +135,6 @@ static int mock_domain_nop_attach(struct iommu_domain *domain,
}
static const struct iommu_domain_ops mock_blocking_ops = {
.free = mock_domain_blocking_free,
.attach_dev = mock_domain_nop_attach,
};
@ -258,15 +253,6 @@ __mock_domain_alloc_nested(struct mock_iommu_domain *mock_parent,
return &mock_nested->domain;
}
static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type)
{
if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED)
return &mock_blocking_domain;
if (iommu_domain_type == IOMMU_DOMAIN_UNMANAGED)
return mock_domain_alloc_paging(NULL);
return NULL;
}
static struct iommu_domain *
mock_domain_alloc_user(struct device *dev, u32 flags,
struct iommu_domain *parent,
@ -446,14 +432,6 @@ static bool mock_domain_capable(struct device *dev, enum iommu_cap cap)
return false;
}
static void mock_domain_set_plaform_dma_ops(struct device *dev)
{
/*
* mock doesn't setup default domains because we can't hook into the
* normal probe path
*/
}
static struct iommu_device mock_iommu_device = {
};
@ -463,13 +441,18 @@ static struct iommu_device *mock_probe_device(struct device *dev)
}
static const struct iommu_ops mock_ops = {
/*
* IOMMU_DOMAIN_BLOCKED cannot be returned from def_domain_type()
* because it is zero.
*/
.default_domain = &mock_blocking_domain,
.blocked_domain = &mock_blocking_domain,
.owner = THIS_MODULE,
.pgsize_bitmap = MOCK_IO_PAGE_SIZE,
.hw_info = mock_domain_hw_info,
.domain_alloc = mock_domain_alloc,
.domain_alloc_paging = mock_domain_alloc_paging,
.domain_alloc_user = mock_domain_alloc_user,
.capable = mock_domain_capable,
.set_platform_dma_ops = mock_domain_set_plaform_dma_ops,
.device_group = generic_device_group,
.probe_device = mock_probe_device,
.default_domain_ops =

95
drivers/iommu/iova.c
View File

@ -11,6 +11,7 @@
#include <linux/smp.h>
#include <linux/bitops.h>
#include <linux/cpu.h>
#include <linux/workqueue.h>
/* The anchor node sits above the top of the usable address space */
#define IOVA_ANCHOR ~0UL
@ -622,15 +623,21 @@ EXPORT_SYMBOL_GPL(reserve_iova);
/*
* As kmalloc's buffer size is fixed to power of 2, 127 is chosen to
* assure size of 'iova_magazine' to be 1024 bytes, so that no memory
* will be wasted.
* will be wasted. Since only full magazines are inserted into the depot,
* we don't need to waste PFN capacity on a separate list head either.
*/
#define IOVA_MAG_SIZE 127
#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
#define IOVA_DEPOT_DELAY msecs_to_jiffies(100)
struct iova_magazine {
unsigned long size;
union {
unsigned long size;
struct iova_magazine *next;
};
unsigned long pfns[IOVA_MAG_SIZE];
};
static_assert(!(sizeof(struct iova_magazine) & (sizeof(struct iova_magazine) - 1)));
struct iova_cpu_rcache {
spinlock_t lock;
@ -640,9 +647,11 @@ struct iova_cpu_rcache {
struct iova_rcache {
spinlock_t lock;
unsigned long depot_size;
struct iova_magazine *depot[MAX_GLOBAL_MAGS];
unsigned int depot_size;
struct iova_magazine *depot;
struct iova_cpu_rcache __percpu *cpu_rcaches;
struct iova_domain *iovad;
struct delayed_work work;
};
static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
@ -717,6 +726,41 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
mag->pfns[mag->size++] = pfn;
}
static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache)
{
struct iova_magazine *mag = rcache->depot;
rcache->depot = mag->next;
mag->size = IOVA_MAG_SIZE;
rcache->depot_size--;
return mag;
}
static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag)
{
mag->next = rcache->depot;
rcache->depot = mag;
rcache->depot_size++;
}
static void iova_depot_work_func(struct work_struct *work)
{
struct iova_rcache *rcache = container_of(work, typeof(*rcache), work.work);
struct iova_magazine *mag = NULL;
unsigned long flags;
spin_lock_irqsave(&rcache->lock, flags);
if (rcache->depot_size > num_online_cpus())
mag = iova_depot_pop(rcache);
spin_unlock_irqrestore(&rcache->lock, flags);
if (mag) {
iova_magazine_free_pfns(mag, rcache->iovad);
iova_magazine_free(mag);
schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
}
}
int iova_domain_init_rcaches(struct iova_domain *iovad)
{
unsigned int cpu;
@ -734,7 +778,8 @@ int iova_domain_init_rcaches(struct iova_domain *iovad)
rcache = &iovad->rcaches[i];
spin_lock_init(&rcache->lock);
rcache->depot_size = 0;
rcache->iovad = iovad;
INIT_DELAYED_WORK(&rcache->work, iova_depot_work_func);
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache),
cache_line_size());
if (!rcache->cpu_rcaches) {
@ -776,7 +821,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
struct iova_rcache *rcache,
unsigned long iova_pfn)
{
struct iova_magazine *mag_to_free = NULL;
struct iova_cpu_rcache *cpu_rcache;
bool can_insert = false;
unsigned long flags;
@ -794,13 +838,9 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
if (new_mag) {
spin_lock(&rcache->lock);
if (rcache->depot_size < MAX_GLOBAL_MAGS) {
rcache->depot[rcache->depot_size++] =
cpu_rcache->loaded;
} else {
mag_to_free = cpu_rcache->loaded;
}
iova_depot_push(rcache, cpu_rcache->loaded);
spin_unlock(&rcache->lock);
schedule_delayed_work(&rcache->work, IOVA_DEPOT_DELAY);
cpu_rcache->loaded = new_mag;
can_insert = true;
@ -812,11 +852,6 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
if (mag_to_free) {
iova_magazine_free_pfns(mag_to_free, iovad);
iova_magazine_free(mag_to_free);
}
return can_insert;
}
@ -854,9 +889,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
has_pfn = true;
} else {
spin_lock(&rcache->lock);
if (rcache->depot_size > 0) {
if (rcache->depot) {
iova_magazine_free(cpu_rcache->loaded);
cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
cpu_rcache->loaded = iova_depot_pop(rcache);
has_pfn = true;
}
spin_unlock(&rcache->lock);
@ -895,9 +930,8 @@ static void free_iova_rcaches(struct iova_domain *iovad)
struct iova_rcache *rcache;
struct iova_cpu_rcache *cpu_rcache;
unsigned int cpu;
int i, j;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
rcache = &iovad->rcaches[i];
if (!rcache->cpu_rcaches)
break;
@ -907,8 +941,9 @@ static void free_iova_rcaches(struct iova_domain *iovad)
iova_magazine_free(cpu_rcache->prev);
}
free_percpu(rcache->cpu_rcaches);
for (j = 0; j < rcache->depot_size; ++j)
iova_magazine_free(rcache->depot[j]);
cancel_delayed_work_sync(&rcache->work);
while (rcache->depot)
iova_magazine_free(iova_depot_pop(rcache));
}
kfree(iovad->rcaches);
@ -942,16 +977,16 @@ static void free_global_cached_iovas(struct iova_domain *iovad)
{
struct iova_rcache *rcache;
unsigned long flags;
int i, j;
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
for (int i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
rcache = &iovad->rcaches[i];
spin_lock_irqsave(&rcache->lock, flags);
for (j = 0; j < rcache->depot_size; ++j) {
iova_magazine_free_pfns(rcache->depot[j], iovad);
iova_magazine_free(rcache->depot[j]);
while (rcache->depot) {
struct iova_magazine *mag = iova_depot_pop(rcache);
iova_magazine_free_pfns(mag, iovad);
iova_magazine_free(mag);
}
rcache->depot_size = 0;
spin_unlock_irqrestore(&rcache->lock, flags);
}
}

72
drivers/iommu/ipmmu-vmsa.c
View File

@ -64,7 +64,6 @@ struct ipmmu_vmsa_device {
struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX];
s8 utlb_ctx[IPMMU_UTLB_MAX];
struct iommu_group *group;
struct dma_iommu_mapping *mapping;
};
@ -295,6 +294,18 @@ static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
mmu->utlb_ctx[utlb] = domain->context_id;
}
/*
* Disable MMU translation for the microTLB.
*/
static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
unsigned int utlb)
{
struct ipmmu_vmsa_device *mmu = domain->mmu;
ipmmu_imuctr_write(mmu, utlb, 0);
mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;
}
static void ipmmu_tlb_flush_all(void *cookie)
{
struct ipmmu_vmsa_domain *domain = cookie;
@ -551,13 +562,10 @@ static irqreturn_t ipmmu_irq(int irq, void *dev)
* IOMMU Operations
*/
static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
static struct iommu_domain *ipmmu_domain_alloc_paging(struct device *dev)
{
struct ipmmu_vmsa_domain *domain;
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
return NULL;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
@ -627,6 +635,36 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
return 0;
}
static int ipmmu_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *io_domain = iommu_get_domain_for_dev(dev);
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct ipmmu_vmsa_domain *domain;
unsigned int i;
if (io_domain == identity_domain || !io_domain)
return 0;
domain = to_vmsa_domain(io_domain);
for (i = 0; i < fwspec->num_ids; ++i)
ipmmu_utlb_disable(domain, fwspec->ids[i]);
/*
* TODO: Optimize by disabling the context when no device is attached.
*/
return 0;
}
static struct iommu_domain_ops ipmmu_iommu_identity_ops = {
.attach_dev = ipmmu_iommu_identity_attach,
};
static struct iommu_domain ipmmu_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &ipmmu_iommu_identity_ops,
};
static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
@ -833,28 +871,18 @@ static void ipmmu_release_device(struct device *dev)
arm_iommu_release_mapping(mmu->mapping);
}
static struct iommu_group *ipmmu_find_group(struct device *dev)
{
struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
struct iommu_group *group;
if (mmu->group)
return iommu_group_ref_get(mmu->group);
group = iommu_group_alloc();
if (!IS_ERR(group))
mmu->group = group;
return group;
}
static const struct iommu_ops ipmmu_ops = {
.domain_alloc = ipmmu_domain_alloc,
.identity_domain = &ipmmu_iommu_identity_domain,
.domain_alloc_paging = ipmmu_domain_alloc_paging,
.probe_device = ipmmu_probe_device,
.release_device = ipmmu_release_device,
.probe_finalize = ipmmu_probe_finalize,
/*
* FIXME: The device grouping is a fixed property of the hardware's
* ability to isolate and control DMA, it should not depend on kconfig.
*/
.device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)
? generic_device_group : ipmmu_find_group,
? generic_device_group : generic_single_device_group,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
.of_xlate = ipmmu_of_xlate,
.default_domain_ops = &(const struct iommu_domain_ops) {

35
drivers/iommu/msm_iommu.c
View File

@ -302,13 +302,10 @@ static void __program_context(void __iomem *base, int ctx,
SET_M(base, ctx, 1);
}
static struct iommu_domain *msm_iommu_domain_alloc(unsigned type)
static struct iommu_domain *msm_iommu_domain_alloc_paging(struct device *dev)
{
struct msm_priv *priv;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
goto fail_nomem;
@ -443,15 +440,20 @@ fail:
return ret;
}
static void msm_iommu_set_platform_dma(struct device *dev)
static int msm_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct msm_priv *priv = to_msm_priv(domain);
struct msm_priv *priv;
unsigned long flags;
struct msm_iommu_dev *iommu;
struct msm_iommu_ctx_dev *master;
int ret;
int ret = 0;
if (domain == identity_domain || !domain)
return 0;
priv = to_msm_priv(domain);
free_io_pgtable_ops(priv->iop);
spin_lock_irqsave(&msm_iommu_lock, flags);
@ -468,8 +470,18 @@ static void msm_iommu_set_platform_dma(struct device *dev)
}
fail:
spin_unlock_irqrestore(&msm_iommu_lock, flags);
return ret;
}
static struct iommu_domain_ops msm_iommu_identity_ops = {
.attach_dev = msm_iommu_identity_attach,
};
static struct iommu_domain msm_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &msm_iommu_identity_ops,
};
static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t pa, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
@ -486,12 +498,13 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
return ret;
}
static void msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
size_t size)
static int msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
size_t size)
{
struct msm_priv *priv = to_msm_priv(domain);
__flush_iotlb_range(iova, size, SZ_4K, false, priv);
return 0;
}
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
@ -675,10 +688,10 @@ fail:
}
static struct iommu_ops msm_iommu_ops = {
.domain_alloc = msm_iommu_domain_alloc,
.identity_domain = &msm_iommu_identity_domain,
.domain_alloc_paging = msm_iommu_domain_alloc_paging,
.probe_device = msm_iommu_probe_device,
.device_group = generic_device_group,
.set_platform_dma_ops = msm_iommu_set_platform_dma,
.pgsize_bitmap = MSM_IOMMU_PGSIZES,
.of_xlate = qcom_iommu_of_xlate,
.default_domain_ops = &(const struct iommu_domain_ops) {

35
drivers/iommu/mtk_iommu.c
View File

@ -688,13 +688,10 @@ update_iova_region:
return 0;
}
static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
static struct iommu_domain *mtk_iommu_domain_alloc_paging(struct device *dev)
{
struct mtk_iommu_domain *dom;
if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
dom = kzalloc(sizeof(*dom), GFP_KERNEL);
if (!dom)
return NULL;
@ -776,6 +773,28 @@ err_unlock:
return ret;
}
static int mtk_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
if (domain == identity_domain || !domain)
return 0;
mtk_iommu_config(data, dev, false, 0);
return 0;
}
static struct iommu_domain_ops mtk_iommu_identity_ops = {
.attach_dev = mtk_iommu_identity_attach,
};
static struct iommu_domain mtk_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &mtk_iommu_identity_ops,
};
static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
@ -817,12 +836,13 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank);
}
static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
size_t size)
static int mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
size_t size)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank);
return 0;
}
static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
@ -995,7 +1015,8 @@ static void mtk_iommu_get_resv_regions(struct device *dev,
}
static const struct iommu_ops mtk_iommu_ops = {
.domain_alloc = mtk_iommu_domain_alloc,
.identity_domain = &mtk_iommu_identity_domain,
.domain_alloc_paging = mtk_iommu_domain_alloc_paging,
.probe_device = mtk_iommu_probe_device,
.release_device = mtk_iommu_release_device,
.device_group = mtk_iommu_device_group,

28
drivers/iommu/mtk_iommu_v1.c
View File

@ -270,13 +270,10 @@ static int mtk_iommu_v1_domain_finalise(struct mtk_iommu_v1_data *data)
return 0;
}
static struct iommu_domain *mtk_iommu_v1_domain_alloc(unsigned type)
static struct iommu_domain *mtk_iommu_v1_domain_alloc_paging(struct device *dev)
{
struct mtk_iommu_v1_domain *dom;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
dom = kzalloc(sizeof(*dom), GFP_KERNEL);
if (!dom)
return NULL;
@ -319,13 +316,24 @@ static int mtk_iommu_v1_attach_device(struct iommu_domain *domain, struct device
return 0;
}
static void mtk_iommu_v1_set_platform_dma(struct device *dev)
static int mtk_iommu_v1_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct mtk_iommu_v1_data *data = dev_iommu_priv_get(dev);
mtk_iommu_v1_config(data, dev, false);
return 0;
}
static struct iommu_domain_ops mtk_iommu_v1_identity_ops = {
.attach_dev = mtk_iommu_v1_identity_attach,
};
static struct iommu_domain mtk_iommu_v1_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &mtk_iommu_v1_identity_ops,
};
static int mtk_iommu_v1_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
@ -441,11 +449,6 @@ static int mtk_iommu_v1_create_mapping(struct device *dev, struct of_phandle_arg
return 0;
}
static int mtk_iommu_v1_def_domain_type(struct device *dev)
{
return IOMMU_DOMAIN_UNMANAGED;
}
static struct iommu_device *mtk_iommu_v1_probe_device(struct device *dev)
{
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
@ -578,14 +581,13 @@ static int mtk_iommu_v1_hw_init(const struct mtk_iommu_v1_data *data)
}
static const struct iommu_ops mtk_iommu_v1_ops = {
.domain_alloc = mtk_iommu_v1_domain_alloc,
.identity_domain = &mtk_iommu_v1_identity_domain,
.domain_alloc_paging = mtk_iommu_v1_domain_alloc_paging,
.probe_device = mtk_iommu_v1_probe_device,
.probe_finalize = mtk_iommu_v1_probe_finalize,
.release_device = mtk_iommu_v1_release_device,
.def_domain_type = mtk_iommu_v1_def_domain_type,
.device_group = generic_device_group,
.pgsize_bitmap = MT2701_IOMMU_PAGE_SIZE,
.set_platform_dma_ops = mtk_iommu_v1_set_platform_dma,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = mtk_iommu_v1_attach_device,

69
drivers/iommu/omap-iommu.c
View File

@ -1225,18 +1225,15 @@ static int omap_iommu_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, obj);
if (omap_iommu_can_register(pdev)) {
obj->group = iommu_group_alloc();
if (IS_ERR(obj->group))
return PTR_ERR(obj->group);
err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL,
obj->name);
if (err)
goto out_group;
return err;
err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
if (err)
goto out_sysfs;
obj->has_iommu_driver = true;
}
pm_runtime_enable(obj->dev);
@ -1252,8 +1249,6 @@ static int omap_iommu_probe(struct platform_device *pdev)
out_sysfs:
iommu_device_sysfs_remove(&obj->iommu);
out_group:
iommu_group_put(obj->group);
return err;
}
@ -1261,10 +1256,7 @@ static void omap_iommu_remove(struct platform_device *pdev)
{
struct omap_iommu *obj = platform_get_drvdata(pdev);
if (obj->group) {
iommu_group_put(obj->group);
obj->group = NULL;
if (obj->has_iommu_driver) {
iommu_device_sysfs_remove(&obj->iommu);
iommu_device_unregister(&obj->iommu);
}
@ -1318,7 +1310,8 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
}
static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
phys_addr_t pa, size_t bytes, size_t count,
int prot, gfp_t gfp, size_t *mapped)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct device *dev = omap_domain->dev;
@ -1356,13 +1349,15 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
oiommu = iommu->iommu_dev;
iopgtable_clear_entry(oiommu, da);
}
} else {
*mapped = bytes;
}
return ret;
}
static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
size_t size, struct iommu_iotlb_gather *gather)
size_t size, size_t count, struct iommu_iotlb_gather *gather)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct device *dev = omap_domain->dev;
@ -1555,23 +1550,35 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
omap_domain->dev = NULL;
}
static void omap_iommu_set_platform_dma(struct device *dev)
static int omap_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct omap_iommu_domain *omap_domain;
if (domain == identity_domain || !domain)
return 0;
omap_domain = to_omap_domain(domain);
spin_lock(&omap_domain->lock);
_omap_iommu_detach_dev(omap_domain, dev);
spin_unlock(&omap_domain->lock);
return 0;
}
static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
static struct iommu_domain_ops omap_iommu_identity_ops = {
.attach_dev = omap_iommu_identity_attach,
};
static struct iommu_domain omap_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &omap_iommu_identity_ops,
};
static struct iommu_domain *omap_iommu_domain_alloc_paging(struct device *dev)
{
struct omap_iommu_domain *omap_domain;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
if (!omap_domain)
return NULL;
@ -1717,31 +1724,17 @@ static void omap_iommu_release_device(struct device *dev)
}
static struct iommu_group *omap_iommu_device_group(struct device *dev)
{
struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
struct iommu_group *group = ERR_PTR(-EINVAL);
if (!arch_data)
return ERR_PTR(-ENODEV);
if (arch_data->iommu_dev)
group = iommu_group_ref_get(arch_data->iommu_dev->group);
return group;
}
static const struct iommu_ops omap_iommu_ops = {
.domain_alloc = omap_iommu_domain_alloc,
.identity_domain = &omap_iommu_identity_domain,
.domain_alloc_paging = omap_iommu_domain_alloc_paging,
.probe_device = omap_iommu_probe_device,
.release_device = omap_iommu_release_device,
.device_group = omap_iommu_device_group,
.set_platform_dma_ops = omap_iommu_set_platform_dma,
.device_group = generic_single_device_group,
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = omap_iommu_attach_dev,
.map = omap_iommu_map,
.unmap = omap_iommu_unmap,
.map_pages = omap_iommu_map,
.unmap_pages = omap_iommu_unmap,
.iova_to_phys = omap_iommu_iova_to_phys,
.free = omap_iommu_domain_free,
}

2
drivers/iommu/omap-iommu.h
View File

@ -80,7 +80,7 @@ struct omap_iommu {
u32 id;
struct iommu_device iommu;
struct iommu_group *group;
bool has_iommu_driver;
u8 pwrst;
};

59
drivers/iommu/rockchip-iommu.c
View File

@ -113,7 +113,6 @@ struct rk_iommu {
struct iommu_device iommu;
struct list_head node; /* entry in rk_iommu_domain.iommus */
struct iommu_domain *domain; /* domain to which iommu is attached */
struct iommu_group *group;
};
struct rk_iommudata {
@ -817,7 +816,8 @@ unwind:
}
static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t size, size_t count,
int prot, gfp_t gfp, size_t *mapped)
{
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
unsigned long flags;
@ -850,12 +850,14 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
paddr, size, prot);
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
if (!ret)
*mapped = size;
return ret;
}
static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t size, size_t count, struct iommu_iotlb_gather *gather)
{
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
unsigned long flags;
@ -989,13 +991,8 @@ static int rk_iommu_identity_attach(struct iommu_domain *identity_domain,
return 0;
}
static void rk_iommu_identity_free(struct iommu_domain *domain)
{
}
static struct iommu_domain_ops rk_identity_ops = {
.attach_dev = rk_iommu_identity_attach,
.free = rk_iommu_identity_free,
};
static struct iommu_domain rk_identity_domain = {
@ -1003,13 +1000,6 @@ static struct iommu_domain rk_identity_domain = {
.ops = &rk_identity_ops,
};
#ifdef CONFIG_ARM
static void rk_iommu_set_platform_dma(struct device *dev)
{
WARN_ON(rk_iommu_identity_attach(&rk_identity_domain, dev));
}
#endif
static int rk_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
@ -1055,16 +1045,10 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
return ret;
}
static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
static struct iommu_domain *rk_iommu_domain_alloc_paging(struct device *dev)
{
struct rk_iommu_domain *rk_domain;
if (type == IOMMU_DOMAIN_IDENTITY)
return &rk_identity_domain;
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
return NULL;
if (!dma_dev)
return NULL;
@ -1155,15 +1139,6 @@ static void rk_iommu_release_device(struct device *dev)
device_link_del(data->link);
}
static struct iommu_group *rk_iommu_device_group(struct device *dev)
{
struct rk_iommu *iommu;
iommu = rk_iommu_from_dev(dev);
return iommu_group_ref_get(iommu->group);
}
static int rk_iommu_of_xlate(struct device *dev,
struct of_phandle_args *args)
{
@ -1186,19 +1161,17 @@ static int rk_iommu_of_xlate(struct device *dev,
}
static const struct iommu_ops rk_iommu_ops = {
.domain_alloc = rk_iommu_domain_alloc,
.identity_domain = &rk_identity_domain,
.domain_alloc_paging = rk_iommu_domain_alloc_paging,
.probe_device = rk_iommu_probe_device,
.release_device = rk_iommu_release_device,
.device_group = rk_iommu_device_group,
#ifdef CONFIG_ARM
.set_platform_dma_ops = rk_iommu_set_platform_dma,
#endif
.device_group = generic_single_device_group,
.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
.of_xlate = rk_iommu_of_xlate,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = rk_iommu_attach_device,
.map = rk_iommu_map,
.unmap = rk_iommu_unmap,
.map_pages = rk_iommu_map,
.unmap_pages = rk_iommu_unmap,
.iova_to_phys = rk_iommu_iova_to_phys,
.free = rk_iommu_domain_free,
}
@ -1280,15 +1253,9 @@ static int rk_iommu_probe(struct platform_device *pdev)
if (err)
return err;
iommu->group = iommu_group_alloc();
if (IS_ERR(iommu->group)) {
err = PTR_ERR(iommu->group);
goto err_unprepare_clocks;
}
err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
if (err)
goto err_put_group;
goto err_unprepare_clocks;
err = iommu_device_register(&iommu->iommu, &rk_iommu_ops, dev);
if (err)
@ -1325,8 +1292,6 @@ err_pm_disable:
pm_runtime_disable(dev);
err_remove_sysfs:
iommu_device_sysfs_remove(&iommu->iommu);
err_put_group:
iommu_group_put(iommu->group);
err_unprepare_clocks:
clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
return err;

424
drivers/iommu/s390-iommu.c
View File

@ -14,16 +14,300 @@
#include <linux/rcupdate.h>
#include <asm/pci_dma.h>
#include "dma-iommu.h"
static const struct iommu_ops s390_iommu_ops;
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
static u64 s390_iommu_aperture;
static u32 s390_iommu_aperture_factor = 1;
struct s390_domain {
struct iommu_domain domain;
struct list_head devices;
struct zpci_iommu_ctrs ctrs;
unsigned long *dma_table;
spinlock_t list_lock;
struct rcu_head rcu;
};
static inline unsigned int calc_rtx(dma_addr_t ptr)
{
return ((unsigned long)ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
}
static inline unsigned int calc_sx(dma_addr_t ptr)
{
return ((unsigned long)ptr >> ZPCI_ST_SHIFT) & ZPCI_INDEX_MASK;
}
static inline unsigned int calc_px(dma_addr_t ptr)
{
return ((unsigned long)ptr >> PAGE_SHIFT) & ZPCI_PT_MASK;
}
static inline void set_pt_pfaa(unsigned long *entry, phys_addr_t pfaa)
{
*entry &= ZPCI_PTE_FLAG_MASK;
*entry |= (pfaa & ZPCI_PTE_ADDR_MASK);
}
static inline void set_rt_sto(unsigned long *entry, phys_addr_t sto)
{
*entry &= ZPCI_RTE_FLAG_MASK;
*entry |= (sto & ZPCI_RTE_ADDR_MASK);
*entry |= ZPCI_TABLE_TYPE_RTX;
}
static inline void set_st_pto(unsigned long *entry, phys_addr_t pto)
{
*entry &= ZPCI_STE_FLAG_MASK;
*entry |= (pto & ZPCI_STE_ADDR_MASK);
*entry |= ZPCI_TABLE_TYPE_SX;
}
static inline void validate_rt_entry(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_VALID_MASK;
*entry &= ~ZPCI_TABLE_OFFSET_MASK;
*entry |= ZPCI_TABLE_VALID;
*entry |= ZPCI_TABLE_LEN_RTX;
}
static inline void validate_st_entry(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_VALID_MASK;
*entry |= ZPCI_TABLE_VALID;
}
static inline void invalidate_pt_entry(unsigned long *entry)
{
WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_INVALID);
*entry &= ~ZPCI_PTE_VALID_MASK;
*entry |= ZPCI_PTE_INVALID;
}
static inline void validate_pt_entry(unsigned long *entry)
{
WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID);
*entry &= ~ZPCI_PTE_VALID_MASK;
*entry |= ZPCI_PTE_VALID;
}
static inline void entry_set_protected(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_PROT_MASK;
*entry |= ZPCI_TABLE_PROTECTED;
}
static inline void entry_clr_protected(unsigned long *entry)
{
*entry &= ~ZPCI_TABLE_PROT_MASK;
*entry |= ZPCI_TABLE_UNPROTECTED;
}
static inline int reg_entry_isvalid(unsigned long entry)
{
return (entry & ZPCI_TABLE_VALID_MASK) == ZPCI_TABLE_VALID;
}
static inline int pt_entry_isvalid(unsigned long entry)
{
return (entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID;
}
static inline unsigned long *get_rt_sto(unsigned long entry)
{
if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RTX)
return phys_to_virt(entry & ZPCI_RTE_ADDR_MASK);
else
return NULL;
}
static inline unsigned long *get_st_pto(unsigned long entry)
{
if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_SX)
return phys_to_virt(entry & ZPCI_STE_ADDR_MASK);
else
return NULL;
}
static int __init dma_alloc_cpu_table_caches(void)
{
dma_region_table_cache = kmem_cache_create("PCI_DMA_region_tables",
ZPCI_TABLE_SIZE,
ZPCI_TABLE_ALIGN,
0, NULL);
if (!dma_region_table_cache)
return -ENOMEM;
dma_page_table_cache = kmem_cache_create("PCI_DMA_page_tables",
ZPCI_PT_SIZE,
ZPCI_PT_ALIGN,
0, NULL);
if (!dma_page_table_cache) {
kmem_cache_destroy(dma_region_table_cache);
return -ENOMEM;
}
return 0;
}
static unsigned long *dma_alloc_cpu_table(gfp_t gfp)
{
unsigned long *table, *entry;
table = kmem_cache_alloc(dma_region_table_cache, gfp);
if (!table)
return NULL;
for (entry = table; entry < table + ZPCI_TABLE_ENTRIES; entry++)
*entry = ZPCI_TABLE_INVALID;
return table;
}
static void dma_free_cpu_table(void *table)
{
kmem_cache_free(dma_region_table_cache, table);
}
static void dma_free_page_table(void *table)
{
kmem_cache_free(dma_page_table_cache, table);
}
static void dma_free_seg_table(unsigned long entry)
{
unsigned long *sto = get_rt_sto(entry);
int sx;
for (sx = 0; sx < ZPCI_TABLE_ENTRIES; sx++)
if (reg_entry_isvalid(sto[sx]))
dma_free_page_table(get_st_pto(sto[sx]));
dma_free_cpu_table(sto);
}
static void dma_cleanup_tables(unsigned long *table)
{
int rtx;
if (!table)
return;
for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++)
if (reg_entry_isvalid(table[rtx]))
dma_free_seg_table(table[rtx]);
dma_free_cpu_table(table);
}
static unsigned long *dma_alloc_page_table(gfp_t gfp)
{
unsigned long *table, *entry;
table = kmem_cache_alloc(dma_page_table_cache, gfp);
if (!table)
return NULL;
for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++)
*entry = ZPCI_PTE_INVALID;
return table;
}
static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp)
{
unsigned long old_rte, rte;
unsigned long *sto;
rte = READ_ONCE(*rtep);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
} else {
sto = dma_alloc_cpu_table(gfp);
if (!sto)
return NULL;
set_rt_sto(&rte, virt_to_phys(sto));
validate_rt_entry(&rte);
entry_clr_protected(&rte);
old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte);
if (old_rte != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_cpu_table(sto);
sto = get_rt_sto(old_rte);
}
}
return sto;
}
static unsigned long *dma_get_page_table_origin(unsigned long *step, gfp_t gfp)
{
unsigned long old_ste, ste;
unsigned long *pto;
ste = READ_ONCE(*step);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
} else {
pto = dma_alloc_page_table(gfp);
if (!pto)
return NULL;
set_st_pto(&ste, virt_to_phys(pto));
validate_st_entry(&ste);
entry_clr_protected(&ste);
old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste);
if (old_ste != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_page_table(pto);
pto = get_st_pto(old_ste);
}
}
return pto;
}
static unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr, gfp_t gfp)
{
unsigned long *sto, *pto;
unsigned int rtx, sx, px;
rtx = calc_rtx(dma_addr);
sto = dma_get_seg_table_origin(&rto[rtx], gfp);
if (!sto)
return NULL;
sx = calc_sx(dma_addr);
pto = dma_get_page_table_origin(&sto[sx], gfp);
if (!pto)
return NULL;
px = calc_px(dma_addr);
return &pto[px];
}
static void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags)
{
unsigned long pte;
pte = READ_ONCE(*ptep);
if (flags & ZPCI_PTE_INVALID) {
invalidate_pt_entry(&pte);
} else {
set_pt_pfaa(&pte, page_addr);
validate_pt_entry(&pte);
}
if (flags & ZPCI_TABLE_PROTECTED)
entry_set_protected(&pte);
else
entry_clr_protected(&pte);
xchg(ptep, pte);
}
static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
{
return container_of(dom, struct s390_domain, domain);
@ -31,21 +315,22 @@ static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
return true;
case IOMMU_CAP_DEFERRED_FLUSH:
return zdev->pft != PCI_FUNC_TYPE_ISM;
default:
return false;
}
}
static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
static struct iommu_domain *s390_domain_alloc_paging(struct device *dev)
{
struct s390_domain *s390_domain;
if (domain_type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
if (!s390_domain)
return NULL;
@ -84,14 +369,13 @@ static void s390_domain_free(struct iommu_domain *domain)
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
}
static void __s390_iommu_detach_device(struct zpci_dev *zdev)
static void s390_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct s390_domain *s390_domain = zdev->s390_domain;
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev = to_zpci_dev(dev);
unsigned long flags;
if (!s390_domain)
return;
spin_lock_irqsave(&s390_domain->list_lock, flags);
list_del_rcu(&zdev->iommu_list);
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
@ -118,9 +402,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
return -EINVAL;
if (zdev->s390_domain)
__s390_iommu_detach_device(zdev);
else if (zdev->dma_table)
zpci_dma_exit_device(zdev);
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(s390_domain->dma_table), &status);
@ -130,7 +412,6 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
*/
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
return -EIO;
zdev->dma_table = s390_domain->dma_table;
zdev->dma_table = s390_domain->dma_table;
zdev->s390_domain = s390_domain;
@ -142,14 +423,6 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
return 0;
}
static void s390_iommu_set_platform_dma(struct device *dev)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
__s390_iommu_detach_device(zdev);
zpci_dma_init_device(zdev);
}
static void s390_iommu_get_resv_regions(struct device *dev,
struct list_head *list)
{
@ -190,6 +463,9 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev)
if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1)
zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1;
if (zdev->tlb_refresh)
dev->iommu->shadow_on_flush = 1;
return &zdev->iommu_dev;
}
@ -202,7 +478,13 @@ static void s390_iommu_release_device(struct device *dev)
* to the device, but keep it attached to other devices in the group.
*/
if (zdev)
__s390_iommu_detach_device(zdev);
s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
}
static int zpci_refresh_all(struct zpci_dev *zdev)
{
return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
}
static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain)
@ -212,8 +494,8 @@ static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain)
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
atomic64_inc(&s390_domain->ctrs.global_rpcits);
zpci_refresh_all(zdev);
}
rcu_read_unlock();
}
@ -231,26 +513,40 @@ static void s390_iommu_iotlb_sync(struct iommu_domain *domain,
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
atomic64_inc(&s390_domain->ctrs.sync_rpcits);
zpci_refresh_trans((u64)zdev->fh << 32, gather->start,
size);
}
rcu_read_unlock();
}
static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
static int s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev;
int ret = 0;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
if (!zdev->tlb_refresh)
continue;
zpci_refresh_trans((u64)zdev->fh << 32,
iova, size);
atomic64_inc(&s390_domain->ctrs.sync_map_rpcits);
ret = zpci_refresh_trans((u64)zdev->fh << 32,
iova, size);
/*
* let the hypervisor discover invalidated entries
* allowing it to free IOVAs and unpin pages
*/
if (ret == -ENOMEM) {
ret = zpci_refresh_all(zdev);
if (ret)
break;
}
}
rcu_read_unlock();
return ret;
}
static int s390_iommu_validate_trans(struct s390_domain *s390_domain,
@ -330,16 +626,15 @@ static int s390_iommu_map_pages(struct iommu_domain *domain,
if (!IS_ALIGNED(iova | paddr, pgsize))
return -EINVAL;
if (!(prot & IOMMU_READ))
return -EINVAL;
if (!(prot & IOMMU_WRITE))
flags |= ZPCI_TABLE_PROTECTED;
rc = s390_iommu_validate_trans(s390_domain, paddr, iova,
pgcount, flags, gfp);
if (!rc)
pgcount, flags, gfp);
if (!rc) {
*mapped = size;
atomic64_add(pgcount, &s390_domain->ctrs.mapped_pages);
}
return rc;
}
@ -395,12 +690,26 @@ static size_t s390_iommu_unmap_pages(struct iommu_domain *domain,
return 0;
iommu_iotlb_gather_add_range(gather, iova, size);
atomic64_add(pgcount, &s390_domain->ctrs.unmapped_pages);
return size;
}
static void s390_iommu_probe_finalize(struct device *dev)
{
iommu_setup_dma_ops(dev, 0, U64_MAX);
}
struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev)
{
if (!zdev || !zdev->s390_domain)
return NULL;
return &zdev->s390_domain->ctrs;
}
int zpci_init_iommu(struct zpci_dev *zdev)
{
u64 aperture_size;
int rc = 0;
rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
@ -412,6 +721,12 @@ int zpci_init_iommu(struct zpci_dev *zdev)
if (rc)
goto out_sysfs;
zdev->start_dma = PAGE_ALIGN(zdev->start_dma);
aperture_size = min3(s390_iommu_aperture,
ZPCI_TABLE_SIZE_RT - zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
zdev->end_dma = zdev->start_dma + aperture_size - 1;
return 0;
out_sysfs:
@ -427,13 +742,52 @@ void zpci_destroy_iommu(struct zpci_dev *zdev)
iommu_device_sysfs_remove(&zdev->iommu_dev);
}
static int __init s390_iommu_setup(char *str)
{
if (!strcmp(str, "strict")) {
pr_warn("s390_iommu=strict deprecated; use iommu.strict=1 instead\n");
iommu_set_dma_strict();
}
return 1;
}
__setup("s390_iommu=", s390_iommu_setup);
static int __init s390_iommu_aperture_setup(char *str)
{
if (kstrtou32(str, 10, &s390_iommu_aperture_factor))
s390_iommu_aperture_factor = 1;
return 1;
}
__setup("s390_iommu_aperture=", s390_iommu_aperture_setup);
static int __init s390_iommu_init(void)
{
int rc;
iommu_dma_forcedac = true;
s390_iommu_aperture = (u64)virt_to_phys(high_memory);
if (!s390_iommu_aperture_factor)
s390_iommu_aperture = ULONG_MAX;
else
s390_iommu_aperture *= s390_iommu_aperture_factor;
rc = dma_alloc_cpu_table_caches();
if (rc)
return rc;
return rc;
}
subsys_initcall(s390_iommu_init);
static const struct iommu_ops s390_iommu_ops = {
.capable = s390_iommu_capable,
.domain_alloc = s390_domain_alloc,
.domain_alloc_paging = s390_domain_alloc_paging,
.probe_device = s390_iommu_probe_device,
.probe_finalize = s390_iommu_probe_finalize,
.release_device = s390_iommu_release_device,
.device_group = generic_device_group,
.set_platform_dma_ops = s390_iommu_set_platform_dma,
.pgsize_bitmap = SZ_4K,
.get_resv_regions = s390_iommu_get_resv_regions,
.default_domain_ops = &(const struct iommu_domain_ops) {

36
drivers/iommu/sprd-iommu.c
View File

@ -70,7 +70,6 @@ struct sprd_iommu_device {
void __iomem *base;
struct device *dev;
struct iommu_device iommu;
struct iommu_group *group;
struct clk *eb;
};
@ -134,13 +133,10 @@ sprd_iommu_pgt_size(struct iommu_domain *domain)
SPRD_IOMMU_PAGE_SHIFT) * sizeof(u32);
}
static struct iommu_domain *sprd_iommu_domain_alloc(unsigned int domain_type)
static struct iommu_domain *sprd_iommu_domain_alloc_paging(struct device *dev)
{
struct sprd_iommu_domain *dom;
if (domain_type != IOMMU_DOMAIN_DMA && domain_type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
dom = kzalloc(sizeof(*dom), GFP_KERNEL);
if (!dom)
return NULL;
@ -345,8 +341,8 @@ static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
return size;
}
static void sprd_iommu_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
static int sprd_iommu_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
unsigned int reg;
@ -358,6 +354,7 @@ static void sprd_iommu_sync_map(struct iommu_domain *domain,
/* clear IOMMU TLB buffer after page table updated */
sprd_iommu_write(dom->sdev, reg, 0xffffffff);
return 0;
}
static void sprd_iommu_sync(struct iommu_domain *domain,
@ -399,13 +396,6 @@ static struct iommu_device *sprd_iommu_probe_device(struct device *dev)
return &sdev->iommu;
}
static struct iommu_group *sprd_iommu_device_group(struct device *dev)
{
struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev);
return iommu_group_ref_get(sdev->group);
}
static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
{
struct platform_device *pdev;
@ -421,9 +411,9 @@ static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
static const struct iommu_ops sprd_iommu_ops = {
.domain_alloc = sprd_iommu_domain_alloc,
.domain_alloc_paging = sprd_iommu_domain_alloc_paging,
.probe_device = sprd_iommu_probe_device,
.device_group = sprd_iommu_device_group,
.device_group = generic_single_device_group,
.of_xlate = sprd_iommu_of_xlate,
.pgsize_bitmap = SPRD_IOMMU_PAGE_SIZE,
.owner = THIS_MODULE,
@ -496,16 +486,9 @@ static int sprd_iommu_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, sdev);
sdev->dev = dev;
/* All the client devices are in the same iommu-group */
sdev->group = iommu_group_alloc();
if (IS_ERR(sdev->group)) {
ret = PTR_ERR(sdev->group);
goto free_page;
}
ret = iommu_device_sysfs_add(&sdev->iommu, dev, NULL, dev_name(dev));
if (ret)
goto put_group;
goto free_page;
ret = iommu_device_register(&sdev->iommu, &sprd_iommu_ops, dev);
if (ret)
@ -530,8 +513,6 @@ unregister_iommu:
iommu_device_unregister(&sdev->iommu);
remove_sysfs:
iommu_device_sysfs_remove(&sdev->iommu);
put_group:
iommu_group_put(sdev->group);
free_page:
dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa);
return ret;
@ -543,9 +524,6 @@ static void sprd_iommu_remove(struct platform_device *pdev)
dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa);
iommu_group_put(sdev->group);
sdev->group = NULL;
platform_set_drvdata(pdev, NULL);
iommu_device_sysfs_remove(&sdev->iommu);
iommu_device_unregister(&sdev->iommu);

80
drivers/iommu/sun50i-iommu.c
View File

@ -107,7 +107,6 @@ struct sun50i_iommu {
struct clk *clk;
struct iommu_domain *domain;
struct iommu_group *group;
struct kmem_cache *pt_pool;
};
@ -402,8 +401,8 @@ static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}
static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
struct sun50i_iommu *iommu = sun50i_domain->iommu;
@ -412,6 +411,8 @@ static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
spin_lock_irqsave(&iommu->iommu_lock, flags);
sun50i_iommu_zap_range(iommu, iova, size);
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
return 0;
}
static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
@ -589,7 +590,8 @@ static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
}
static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t size, size_t count,
int prot, gfp_t gfp, size_t *mapped)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
struct sun50i_iommu *iommu = sun50i_domain->iommu;
@ -616,13 +618,14 @@ static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
*pte_addr = sun50i_mk_pte(paddr, prot);
sun50i_table_flush(sun50i_domain, pte_addr, 1);
*mapped = size;
out:
return ret;
}
static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t size, size_t count, struct iommu_iotlb_gather *gather)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
phys_addr_t pt_phys;
@ -667,14 +670,11 @@ static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
sun50i_iova_get_page_offset(iova);
}
static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
static struct iommu_domain *
sun50i_iommu_domain_alloc_paging(struct device *dev)
{
struct sun50i_iommu_domain *sun50i_domain;
if (type != IOMMU_DOMAIN_DMA &&
type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
if (!sun50i_domain)
return NULL;
@ -757,21 +757,32 @@ static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
iommu->domain = NULL;
}
static void sun50i_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
struct sun50i_iommu_domain *sun50i_domain;
dev_dbg(dev, "Detaching from IOMMU domain\n");
if (iommu->domain != domain)
return;
if (iommu->domain == identity_domain)
return 0;
sun50i_domain = to_sun50i_domain(iommu->domain);
if (refcount_dec_and_test(&sun50i_domain->refcnt))
sun50i_iommu_detach_domain(iommu, sun50i_domain);
return 0;
}
static struct iommu_domain_ops sun50i_iommu_identity_ops = {
.attach_dev = sun50i_iommu_identity_attach,
};
static struct iommu_domain sun50i_iommu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &sun50i_iommu_identity_ops,
};
static int sun50i_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
@ -789,8 +800,7 @@ static int sun50i_iommu_attach_device(struct iommu_domain *domain,
if (iommu->domain == domain)
return 0;
if (iommu->domain)
sun50i_iommu_detach_device(iommu->domain, dev);
sun50i_iommu_identity_attach(&sun50i_iommu_identity_domain, dev);
sun50i_iommu_attach_domain(iommu, sun50i_domain);
@ -808,13 +818,6 @@ static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
return &iommu->iommu;
}
static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
{
struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
return iommu_group_ref_get(iommu->group);
}
static int sun50i_iommu_of_xlate(struct device *dev,
struct of_phandle_args *args)
{
@ -827,9 +830,10 @@ static int sun50i_iommu_of_xlate(struct device *dev,
}
static const struct iommu_ops sun50i_iommu_ops = {
.identity_domain = &sun50i_iommu_identity_domain,
.pgsize_bitmap = SZ_4K,
.device_group = sun50i_iommu_device_group,
.domain_alloc = sun50i_iommu_domain_alloc,
.device_group = generic_single_device_group,
.domain_alloc_paging = sun50i_iommu_domain_alloc_paging,
.of_xlate = sun50i_iommu_of_xlate,
.probe_device = sun50i_iommu_probe_device,
.default_domain_ops = &(const struct iommu_domain_ops) {
@ -838,8 +842,8 @@ static const struct iommu_ops sun50i_iommu_ops = {
.iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
.iotlb_sync = sun50i_iommu_iotlb_sync,
.iova_to_phys = sun50i_iommu_iova_to_phys,
.map = sun50i_iommu_map,
.unmap = sun50i_iommu_unmap,
.map_pages = sun50i_iommu_map,
.unmap_pages = sun50i_iommu_unmap,
.free = sun50i_iommu_domain_free,
}
};
@ -985,6 +989,7 @@ static int sun50i_iommu_probe(struct platform_device *pdev)
if (!iommu)
return -ENOMEM;
spin_lock_init(&iommu->iommu_lock);
iommu->domain = &sun50i_iommu_identity_domain;
platform_set_drvdata(pdev, iommu);
iommu->dev = &pdev->dev;
@ -995,42 +1000,36 @@ static int sun50i_iommu_probe(struct platform_device *pdev)
if (!iommu->pt_pool)
return -ENOMEM;
iommu->group = iommu_group_alloc();
if (IS_ERR(iommu->group)) {
ret = PTR_ERR(iommu->group);
goto err_free_cache;
}
iommu->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(iommu->base)) {
ret = PTR_ERR(iommu->base);
goto err_free_group;
goto err_free_cache;
}
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
ret = irq;
goto err_free_group;
goto err_free_cache;
}
iommu->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(iommu->clk)) {
dev_err(&pdev->dev, "Couldn't get our clock.\n");
ret = PTR_ERR(iommu->clk);
goto err_free_group;
goto err_free_cache;
}
iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
if (IS_ERR(iommu->reset)) {
dev_err(&pdev->dev, "Couldn't get our reset line.\n");
ret = PTR_ERR(iommu->reset);
goto err_free_group;
goto err_free_cache;
}
ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
NULL, dev_name(&pdev->dev));
if (ret)
goto err_free_group;
goto err_free_cache;
ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
if (ret)
@ -1049,9 +1048,6 @@ err_unregister:
err_remove_sysfs:
iommu_device_sysfs_remove(&iommu->iommu);
err_free_group:
iommu_group_put(iommu->group);
err_free_cache:
kmem_cache_destroy(iommu->pt_pool);

371
drivers/iommu/tegra-gart.c
View File

@ -1,371 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* IOMMU API for Graphics Address Relocation Table on Tegra20
*
* Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved.
*
* Author: Hiroshi DOYU <hdoyu@nvidia.com>
*/
#define dev_fmt(fmt) "gart: " fmt
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/moduleparam.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <soc/tegra/mc.h>
#define GART_REG_BASE 0x24
#define GART_CONFIG (0x24 - GART_REG_BASE)
#define GART_ENTRY_ADDR (0x28 - GART_REG_BASE)
#define GART_ENTRY_DATA (0x2c - GART_REG_BASE)
#define GART_ENTRY_PHYS_ADDR_VALID BIT(31)
#define GART_PAGE_SHIFT 12
#define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
#define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT)
/* bitmap of the page sizes currently supported */
#define GART_IOMMU_PGSIZES (GART_PAGE_SIZE)
struct gart_device {
void __iomem *regs;
u32 *savedata;
unsigned long iovmm_base; /* offset to vmm_area start */
unsigned long iovmm_end; /* offset to vmm_area end */
spinlock_t pte_lock; /* for pagetable */
spinlock_t dom_lock; /* for active domain */
unsigned int active_devices; /* number of active devices */
struct iommu_domain *active_domain; /* current active domain */
struct iommu_device iommu; /* IOMMU Core handle */
struct device *dev;
};
static struct gart_device *gart_handle; /* unique for a system */
static bool gart_debug;
/*
* Any interaction between any block on PPSB and a block on APB or AHB
* must have these read-back to ensure the APB/AHB bus transaction is
* complete before initiating activity on the PPSB block.
*/
#define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG)
#define for_each_gart_pte(gart, iova) \
for (iova = gart->iovmm_base; \
iova < gart->iovmm_end; \
iova += GART_PAGE_SIZE)
static inline void gart_set_pte(struct gart_device *gart,
unsigned long iova, unsigned long pte)
{
writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
writel_relaxed(pte, gart->regs + GART_ENTRY_DATA);
}
static inline unsigned long gart_read_pte(struct gart_device *gart,
unsigned long iova)
{
unsigned long pte;
writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
pte = readl_relaxed(gart->regs + GART_ENTRY_DATA);
return pte;
}
static void do_gart_setup(struct gart_device *gart, const u32 *data)
{
unsigned long iova;
for_each_gart_pte(gart, iova)
gart_set_pte(gart, iova, data ? *(data++) : 0);
writel_relaxed(1, gart->regs + GART_CONFIG);
FLUSH_GART_REGS(gart);
}
static inline bool gart_iova_range_invalid(struct gart_device *gart,
unsigned long iova, size_t bytes)
{
return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE ||
iova + bytes > gart->iovmm_end);
}
static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova)
{
return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID);
}
static int gart_iommu_attach_dev(struct iommu_domain *domain,
struct device *dev)
{
struct gart_device *gart = gart_handle;
int ret = 0;
spin_lock(&gart->dom_lock);
if (gart->active_domain && gart->active_domain != domain) {
ret = -EINVAL;
} else if (dev_iommu_priv_get(dev) != domain) {
dev_iommu_priv_set(dev, domain);
gart->active_domain = domain;
gart->active_devices++;
}
spin_unlock(&gart->dom_lock);
return ret;
}
static void gart_iommu_set_platform_dma(struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct gart_device *gart = gart_handle;
spin_lock(&gart->dom_lock);
if (dev_iommu_priv_get(dev) == domain) {
dev_iommu_priv_set(dev, NULL);
if (--gart->active_devices == 0)
gart->active_domain = NULL;
}
spin_unlock(&gart->dom_lock);
}
static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
{
struct iommu_domain *domain;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (domain) {
domain->geometry.aperture_start = gart_handle->iovmm_base;
domain->geometry.aperture_end = gart_handle->iovmm_end - 1;
domain->geometry.force_aperture = true;
}
return domain;
}
static void gart_iommu_domain_free(struct iommu_domain *domain)
{
WARN_ON(gart_handle->active_domain == domain);
kfree(domain);
}
static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova,
unsigned long pa)
{
if (unlikely(gart_debug && gart_pte_valid(gart, iova))) {
dev_err(gart->dev, "Page entry is in-use\n");
return -EINVAL;
}
gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa);
return 0;
}
static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
{
struct gart_device *gart = gart_handle;
int ret;
if (gart_iova_range_invalid(gart, iova, bytes))
return -EINVAL;
spin_lock(&gart->pte_lock);
ret = __gart_iommu_map(gart, iova, (unsigned long)pa);
spin_unlock(&gart->pte_lock);
return ret;
}
static inline int __gart_iommu_unmap(struct gart_device *gart,
unsigned long iova)
{
if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) {
dev_err(gart->dev, "Page entry is invalid\n");
return -EINVAL;
}
gart_set_pte(gart, iova, 0);
return 0;
}
static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t bytes, struct iommu_iotlb_gather *gather)
{
struct gart_device *gart = gart_handle;
int err;
if (gart_iova_range_invalid(gart, iova, bytes))
return 0;
spin_lock(&gart->pte_lock);
err = __gart_iommu_unmap(gart, iova);
spin_unlock(&gart->pte_lock);
return err ? 0 : bytes;
}
static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct gart_device *gart = gart_handle;
unsigned long pte;
if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE))
return -EINVAL;
spin_lock(&gart->pte_lock);
pte = gart_read_pte(gart, iova);
spin_unlock(&gart->pte_lock);
return pte & GART_PAGE_MASK;
}
static struct iommu_device *gart_iommu_probe_device(struct device *dev)
{
if (!dev_iommu_fwspec_get(dev))
return ERR_PTR(-ENODEV);
return &gart_handle->iommu;
}
static int gart_iommu_of_xlate(struct device *dev,
struct of_phandle_args *args)
{
return 0;
}
static void gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
size_t size)
{
FLUSH_GART_REGS(gart_handle);
}
static void gart_iommu_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
size_t length = gather->end - gather->start + 1;
gart_iommu_sync_map(domain, gather->start, length);
}
static const struct iommu_ops gart_iommu_ops = {
.domain_alloc = gart_iommu_domain_alloc,
.probe_device = gart_iommu_probe_device,
.device_group = generic_device_group,
.set_platform_dma_ops = gart_iommu_set_platform_dma,
.pgsize_bitmap = GART_IOMMU_PGSIZES,
.of_xlate = gart_iommu_of_xlate,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = gart_iommu_attach_dev,
.map = gart_iommu_map,
.unmap = gart_iommu_unmap,
.iova_to_phys = gart_iommu_iova_to_phys,
.iotlb_sync_map = gart_iommu_sync_map,
.iotlb_sync = gart_iommu_sync,
.free = gart_iommu_domain_free,
}
};
int tegra_gart_suspend(struct gart_device *gart)
{
u32 *data = gart->savedata;
unsigned long iova;
/*
* All GART users shall be suspended at this point. Disable
* address translation to trap all GART accesses as invalid
* memory accesses.
*/
writel_relaxed(0, gart->regs + GART_CONFIG);
FLUSH_GART_REGS(gart);
for_each_gart_pte(gart, iova)
*(data++) = gart_read_pte(gart, iova);
return 0;
}
int tegra_gart_resume(struct gart_device *gart)
{
do_gart_setup(gart, gart->savedata);
return 0;
}
struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
{
struct gart_device *gart;
struct resource *res;
int err;
BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
/* the GART memory aperture is required */
res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1);
if (!res) {
dev_err(dev, "Memory aperture resource unavailable\n");
return ERR_PTR(-ENXIO);
}
gart = kzalloc(sizeof(*gart), GFP_KERNEL);
if (!gart)
return ERR_PTR(-ENOMEM);
gart_handle = gart;
gart->dev = dev;
gart->regs = mc->regs + GART_REG_BASE;
gart->iovmm_base = res->start;
gart->iovmm_end = res->end + 1;
spin_lock_init(&gart->pte_lock);
spin_lock_init(&gart->dom_lock);
do_gart_setup(gart, NULL);
err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
if (err)
goto free_gart;
err = iommu_device_register(&gart->iommu, &gart_iommu_ops, dev);
if (err)
goto remove_sysfs;
gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE *
sizeof(u32));
if (!gart->savedata) {
err = -ENOMEM;
goto unregister_iommu;
}
return gart;
unregister_iommu:
iommu_device_unregister(&gart->iommu);
remove_sysfs:
iommu_device_sysfs_remove(&gart->iommu);
free_gart:
kfree(gart);
return ERR_PTR(err);
}
module_param(gart_debug, bool, 0644);
MODULE_PARM_DESC(gart_debug, "Enable GART debugging");

58
drivers/iommu/tegra-smmu.c
View File

@ -272,13 +272,10 @@ static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
clear_bit(id, smmu->asids);
}
static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
static struct iommu_domain *tegra_smmu_domain_alloc_paging(struct device *dev)
{
struct tegra_smmu_as *as;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
as = kzalloc(sizeof(*as), GFP_KERNEL);
if (!as)
return NULL;
@ -511,23 +508,39 @@ disable:
return err;
}
static void tegra_smmu_set_platform_dma(struct device *dev)
static int tegra_smmu_identity_attach(struct iommu_domain *identity_domain,
struct device *dev)
{
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct tegra_smmu_as *as = to_smmu_as(domain);
struct tegra_smmu *smmu = as->smmu;
struct tegra_smmu_as *as;
struct tegra_smmu *smmu;
unsigned int index;
if (!fwspec)
return;
return -ENODEV;
if (domain == identity_domain || !domain)
return 0;
as = to_smmu_as(domain);
smmu = as->smmu;
for (index = 0; index < fwspec->num_ids; index++) {
tegra_smmu_disable(smmu, fwspec->ids[index], as->id);
tegra_smmu_as_unprepare(smmu, as);
}
return 0;
}
static struct iommu_domain_ops tegra_smmu_identity_ops = {
.attach_dev = tegra_smmu_identity_attach,
};
static struct iommu_domain tegra_smmu_identity_domain = {
.type = IOMMU_DOMAIN_IDENTITY,
.ops = &tegra_smmu_identity_ops,
};
static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
u32 value)
{
@ -751,7 +764,8 @@ __tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
}
static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t size, size_t count,
int prot, gfp_t gfp, size_t *mapped)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned long flags;
@ -761,11 +775,14 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
ret = __tegra_smmu_map(domain, iova, paddr, size, prot, gfp, &flags);
spin_unlock_irqrestore(&as->lock, flags);
if (!ret)
*mapped = size;
return ret;
}
static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t size, size_t count, struct iommu_iotlb_gather *gather)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
unsigned long flags;
@ -962,17 +979,28 @@ static int tegra_smmu_of_xlate(struct device *dev,
return iommu_fwspec_add_ids(dev, &id, 1);
}
static int tegra_smmu_def_domain_type(struct device *dev)
{
/*
* FIXME: For now we want to run all translation in IDENTITY mode, due
* to some device quirks. Better would be to just quirk the troubled
* devices.
*/
return IOMMU_DOMAIN_IDENTITY;
}
static const struct iommu_ops tegra_smmu_ops = {
.domain_alloc = tegra_smmu_domain_alloc,
.identity_domain = &tegra_smmu_identity_domain,
.def_domain_type = &tegra_smmu_def_domain_type,
.domain_alloc_paging = tegra_smmu_domain_alloc_paging,
.probe_device = tegra_smmu_probe_device,
.device_group = tegra_smmu_device_group,
.set_platform_dma_ops = tegra_smmu_set_platform_dma,
.of_xlate = tegra_smmu_of_xlate,
.pgsize_bitmap = SZ_4K,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = tegra_smmu_attach_dev,
.map = tegra_smmu_map,
.unmap = tegra_smmu_unmap,
.map_pages = tegra_smmu_map,
.unmap_pages = tegra_smmu_unmap,
.iova_to_phys = tegra_smmu_iova_to_phys,
.free = tegra_smmu_domain_free,
}
@ -1056,8 +1084,6 @@ DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients);
static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
{
smmu->debugfs = debugfs_create_dir("smmu", NULL);
if (!smmu->debugfs)
return;
debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
&tegra_smmu_swgroups_fops);

4
drivers/iommu/virtio-iommu.c
View File

@ -85,7 +85,7 @@ struct viommu_request {
void *writeback;
unsigned int write_offset;
unsigned int len;
char buf[];
char buf[] __counted_by(len);
};
#define VIOMMU_FAULT_RESV_MASK 0xffffff00
@ -230,7 +230,7 @@ static int __viommu_add_req(struct viommu_dev *viommu, void *buf, size_t len,
if (write_offset <= 0)
return -EINVAL;
req = kzalloc(sizeof(*req) + len, GFP_ATOMIC);
req = kzalloc(struct_size(req, buf, len), GFP_ATOMIC);
if (!req)
return -ENOMEM;

34
drivers/memory/tegra/mc.c
View File

@ -979,35 +979,6 @@ static int tegra_mc_probe(struct platform_device *pdev)
}
}
if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
mc->gart = tegra_gart_probe(&pdev->dev, mc);
if (IS_ERR(mc->gart)) {
dev_err(&pdev->dev, "failed to probe GART: %ld\n",
PTR_ERR(mc->gart));
mc->gart = NULL;
}
}
return 0;
}
static int __maybe_unused tegra_mc_suspend(struct device *dev)
{
struct tegra_mc *mc = dev_get_drvdata(dev);
if (mc->soc->ops && mc->soc->ops->suspend)
return mc->soc->ops->suspend(mc);
return 0;
}
static int __maybe_unused tegra_mc_resume(struct device *dev)
{
struct tegra_mc *mc = dev_get_drvdata(dev);
if (mc->soc->ops && mc->soc->ops->resume)
return mc->soc->ops->resume(mc);
return 0;
}
@ -1020,15 +991,10 @@ static void tegra_mc_sync_state(struct device *dev)
icc_sync_state(dev);
}
static const struct dev_pm_ops tegra_mc_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
};
static struct platform_driver tegra_mc_driver = {
.driver = {
.name = "tegra-mc",
.of_match_table = tegra_mc_of_match,
.pm = &tegra_mc_pm_ops,
.suppress_bind_attrs = true,
.sync_state = tegra_mc_sync_state,
},

28
drivers/memory/tegra/tegra20.c
View File

@ -688,32 +688,6 @@ static int tegra20_mc_probe(struct tegra_mc *mc)
return 0;
}
static int tegra20_mc_suspend(struct tegra_mc *mc)
{
int err;
if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
err = tegra_gart_suspend(mc->gart);
if (err < 0)
return err;
}
return 0;
}
static int tegra20_mc_resume(struct tegra_mc *mc)
{
int err;
if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
err = tegra_gart_resume(mc->gart);
if (err < 0)
return err;
}
return 0;
}
static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
{
struct tegra_mc *mc = data;
@ -789,8 +763,6 @@ static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
static const struct tegra_mc_ops tegra20_mc_ops = {
.probe = tegra20_mc_probe,
.suspend = tegra20_mc_suspend,
.resume = tegra20_mc_resume,
.handle_irq = tegra20_mc_handle_irq,
};

120
include/linux/amd-iommu.h
View File

@ -33,126 +33,6 @@ struct pci_dev;
extern int amd_iommu_detect(void);
/**
* amd_iommu_init_device() - Init device for use with IOMMUv2 driver
* @pdev: The PCI device to initialize
* @pasids: Number of PASIDs to support for this device
*
* This function does all setup for the device pdev so that it can be
* used with IOMMUv2.
* Returns 0 on success or negative value on error.
*/
extern int amd_iommu_init_device(struct pci_dev *pdev, int pasids);
/**
* amd_iommu_free_device() - Free all IOMMUv2 related device resources
* and disable IOMMUv2 usage for this device
* @pdev: The PCI device to disable IOMMUv2 usage for'
*/
extern void amd_iommu_free_device(struct pci_dev *pdev);
/**
* amd_iommu_bind_pasid() - Bind a given task to a PASID on a device
* @pdev: The PCI device to bind the task to
* @pasid: The PASID on the device the task should be bound to
* @task: the task to bind
*
* The function returns 0 on success or a negative value on error.
*/
extern int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
struct task_struct *task);
/**
* amd_iommu_unbind_pasid() - Unbind a PASID from its task on
* a device
* @pdev: The device of the PASID
* @pasid: The PASID to unbind
*
* When this function returns the device is no longer using the PASID
* and the PASID is no longer bound to its task.
*/
extern void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid);
/**
* amd_iommu_set_invalid_ppr_cb() - Register a call-back for failed
* PRI requests
* @pdev: The PCI device the call-back should be registered for
* @cb: The call-back function
*
* The IOMMUv2 driver invokes this call-back when it is unable to
* successfully handle a PRI request. The device driver can then decide
* which PRI response the device should see. Possible return values for
* the call-back are:
*
* - AMD_IOMMU_INV_PRI_RSP_SUCCESS - Send SUCCESS back to the device
* - AMD_IOMMU_INV_PRI_RSP_INVALID - Send INVALID back to the device
* - AMD_IOMMU_INV_PRI_RSP_FAIL - Send Failure back to the device,
* the device is required to disable
* PRI when it receives this response
*
* The function returns 0 on success or negative value on error.
*/
#define AMD_IOMMU_INV_PRI_RSP_SUCCESS 0
#define AMD_IOMMU_INV_PRI_RSP_INVALID 1
#define AMD_IOMMU_INV_PRI_RSP_FAIL 2
typedef int (*amd_iommu_invalid_ppr_cb)(struct pci_dev *pdev,
u32 pasid,
unsigned long address,
u16);
extern int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
amd_iommu_invalid_ppr_cb cb);
#define PPR_FAULT_EXEC (1 << 1)
#define PPR_FAULT_READ (1 << 2)
#define PPR_FAULT_WRITE (1 << 5)
#define PPR_FAULT_USER (1 << 6)
#define PPR_FAULT_RSVD (1 << 7)
#define PPR_FAULT_GN (1 << 8)
/**
* amd_iommu_device_info() - Get information about IOMMUv2 support of a
* PCI device
* @pdev: PCI device to query information from
* @info: A pointer to an amd_iommu_device_info structure which will contain
* the information about the PCI device
*
* Returns 0 on success, negative value on error
*/
#define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
#define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
#define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
#define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8 /* Device may request execution
on memory pages */
#define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10 /* Device may request
super-user privileges */
struct amd_iommu_device_info {
int max_pasids;
u32 flags;
};
extern int amd_iommu_device_info(struct pci_dev *pdev,
struct amd_iommu_device_info *info);
/**
* amd_iommu_set_invalidate_ctx_cb() - Register a call-back for invalidating
* a pasid context. This call-back is
* invoked when the IOMMUv2 driver needs to
* invalidate a PASID context, for example
* because the task that is bound to that
* context is about to exit.
*
* @pdev: The PCI device the call-back should be registered for
* @cb: The call-back function
*/
typedef void (*amd_iommu_invalidate_ctx)(struct pci_dev *pdev, u32 pasid);
extern int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
amd_iommu_invalidate_ctx cb);
#else /* CONFIG_AMD_IOMMU */
static inline int amd_iommu_detect(void) { return -ENODEV; }

38
include/linux/iommu.h
View File

@ -66,6 +66,7 @@ struct iommu_domain_geometry {
#define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */
#define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */
#define __IOMMU_DOMAIN_PLATFORM (1U << 5)
#define __IOMMU_DOMAIN_NESTED (1U << 6) /* User-managed address space nested
on a stage-2 translation */
@ -86,6 +87,8 @@ struct iommu_domain_geometry {
* invalidation.
* IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses
* represented by mm_struct's.
* IOMMU_DOMAIN_PLATFORM - Legacy domain for drivers that do their own
* dma_api stuff. Do not use in new drivers.
*/
#define IOMMU_DOMAIN_BLOCKED (0U)
#define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
@ -96,6 +99,7 @@ struct iommu_domain_geometry {
__IOMMU_DOMAIN_DMA_API | \
__IOMMU_DOMAIN_DMA_FQ)
#define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA)
#define IOMMU_DOMAIN_PLATFORM (__IOMMU_DOMAIN_PLATFORM)
#define IOMMU_DOMAIN_NESTED (__IOMMU_DOMAIN_NESTED)
struct iommu_domain {
@ -340,13 +344,12 @@ static inline int __iommu_copy_struct_from_user(
* NULL while the @user_data can be optionally provided, the
* new domain must support __IOMMU_DOMAIN_PAGING.
* Upon failure, ERR_PTR must be returned.
* @domain_alloc_paging: Allocate an iommu_domain that can be used for
* UNMANAGED, DMA, and DMA_FQ domain types.
* @probe_device: Add device to iommu driver handling
* @release_device: Remove device from iommu driver handling
* @probe_finalize: Do final setup work after the device is added to an IOMMU
* group and attached to the groups domain
* @set_platform_dma_ops: Returning control back to the platform DMA ops. This op
* is to support old IOMMU drivers, new drivers should use
* default domains, and the common IOMMU DMA ops.
* @device_group: find iommu group for a particular device
* @get_resv_regions: Request list of reserved regions for a device
* @of_xlate: add OF master IDs to iommu grouping
@ -365,6 +368,13 @@ static inline int __iommu_copy_struct_from_user(
* will be blocked by the hardware.
* @pgsize_bitmap: bitmap of all possible supported page sizes
* @owner: Driver module providing these ops
* @identity_domain: An always available, always attachable identity
* translation.
* @blocked_domain: An always available, always attachable blocking
* translation.
* @default_domain: If not NULL this will always be set as the default domain.
* This should be an IDENTITY/BLOCKED/PLATFORM domain.
* Do not use in new drivers.
*/
struct iommu_ops {
bool (*capable)(struct device *dev, enum iommu_cap);
@ -375,11 +385,11 @@ struct iommu_ops {
struct iommu_domain *(*domain_alloc_user)(
struct device *dev, u32 flags, struct iommu_domain *parent,
const struct iommu_user_data *user_data);
struct iommu_domain *(*domain_alloc_paging)(struct device *dev);
struct iommu_device *(*probe_device)(struct device *dev);
void (*release_device)(struct device *dev);
void (*probe_finalize)(struct device *dev);
void (*set_platform_dma_ops)(struct device *dev);
struct iommu_group *(*device_group)(struct device *dev);
/* Request/Free a list of reserved regions for a device */
@ -402,6 +412,9 @@ struct iommu_ops {
const struct iommu_domain_ops *default_domain_ops;
unsigned long pgsize_bitmap;
struct module *owner;
struct iommu_domain *identity_domain;
struct iommu_domain *blocked_domain;
struct iommu_domain *default_domain;
};
/**
@ -420,10 +433,8 @@ struct iommu_ops {
* * ENODEV - device specific errors, not able to be attached
* * <others> - treated as ENODEV by the caller. Use is discouraged
* @set_dev_pasid: set an iommu domain to a pasid of device
* @map: map a physically contiguous memory region to an iommu domain
* @map_pages: map a physically contiguous set of pages of the same size to
* an iommu domain.
* @unmap: unmap a physically contiguous memory region from an iommu domain
* @unmap_pages: unmap a number of pages of the same size from an iommu domain
* @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
* @iotlb_sync_map: Sync mappings created recently using @map to the hardware
@ -442,20 +453,16 @@ struct iommu_domain_ops {
int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
ioasid_t pasid);
int (*map)(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped);
size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *iotlb_gather);
size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova,
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *iotlb_gather);
void (*flush_iotlb_all)(struct iommu_domain *domain);
void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
size_t size);
int (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
size_t size);
void (*iotlb_sync)(struct iommu_domain *domain,
struct iommu_iotlb_gather *iotlb_gather);
@ -476,6 +483,7 @@ struct iommu_domain_ops {
* @list: Used by the iommu-core to keep a list of registered iommus
* @ops: iommu-ops for talking to this iommu
* @dev: struct device for sysfs handling
* @singleton_group: Used internally for drivers that have only one group
* @max_pasids: number of supported PASIDs
*/
struct iommu_device {
@ -483,6 +491,7 @@ struct iommu_device {
const struct iommu_ops *ops;
struct fwnode_handle *fwnode;
struct device *dev;
struct iommu_group *singleton_group;
u32 max_pasids;
};
@ -526,6 +535,7 @@ struct iommu_fault_param {
* @attach_deferred: the dma domain attachment is deferred
* @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs
* @require_direct: device requires IOMMU_RESV_DIRECT regions
* @shadow_on_flush: IOTLB flushes are used to sync shadow tables
*
* TODO: migrate other per device data pointers under iommu_dev_data, e.g.
* struct iommu_group *iommu_group;
@ -541,6 +551,7 @@ struct dev_iommu {
u32 attach_deferred:1;
u32 pci_32bit_workaround:1;
u32 require_direct:1;
u32 shadow_on_flush:1;
};
int iommu_device_register(struct iommu_device *iommu,
@ -768,6 +779,7 @@ extern struct iommu_group *pci_device_group(struct device *dev);
extern struct iommu_group *generic_device_group(struct device *dev);
/* FSL-MC device grouping function */
struct iommu_group *fsl_mc_device_group(struct device *dev);
extern struct iommu_group *generic_single_device_group(struct device *dev);
/**
* struct iommu_fwspec - per-device IOMMU instance data
@ -1253,7 +1265,7 @@ static inline void iommu_free_global_pasid(ioasid_t pasid) {}
* Creates a mapping at @iova for the buffer described by a scatterlist
* stored in the given sg_table object in the provided IOMMU domain.
*/
static inline size_t iommu_map_sgtable(struct iommu_domain *domain,
static inline ssize_t iommu_map_sgtable(struct iommu_domain *domain,
unsigned long iova, struct sg_table *sgt, int prot)
{
return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot,

26
include/soc/tegra/mc.h
View File

@ -96,7 +96,6 @@ struct tegra_smmu_soc {
struct tegra_mc;
struct tegra_smmu;
struct gart_device;
#ifdef CONFIG_TEGRA_IOMMU_SMMU
struct tegra_smmu *tegra_smmu_probe(struct device *dev,
@ -116,28 +115,6 @@ static inline void tegra_smmu_remove(struct tegra_smmu *smmu)
}
#endif
#ifdef CONFIG_TEGRA_IOMMU_GART
struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc);
int tegra_gart_suspend(struct gart_device *gart);
int tegra_gart_resume(struct gart_device *gart);
#else
static inline struct gart_device *
tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
{
return ERR_PTR(-ENODEV);
}
static inline int tegra_gart_suspend(struct gart_device *gart)
{
return -ENODEV;
}
static inline int tegra_gart_resume(struct gart_device *gart)
{
return -ENODEV;
}
#endif
struct tegra_mc_reset {
const char *name;
unsigned long id;
@ -185,8 +162,6 @@ struct tegra_mc_ops {
*/
int (*probe)(struct tegra_mc *mc);
void (*remove)(struct tegra_mc *mc);
int (*suspend)(struct tegra_mc *mc);
int (*resume)(struct tegra_mc *mc);
irqreturn_t (*handle_irq)(int irq, void *data);
int (*probe_device)(struct tegra_mc *mc, struct device *dev);
};
@ -225,7 +200,6 @@ struct tegra_mc {
struct tegra_bpmp *bpmp;
struct device *dev;
struct tegra_smmu *smmu;
struct gart_device *gart;
void __iomem *regs;
void __iomem *bcast_ch_regs;
void __iomem **ch_regs;