From 9355a08186e52b7c120adea91c984923b54efa10 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 14:24:08 +0200 Subject: [PATCH 01/14] x86/amd-iommu: Make fetch_pte aware of dynamic mapping levels This patch changes the fetch_pte function in the AMD IOMMU driver to support dynamic mapping levels. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 9 +++++++++ arch/x86/kernel/amd_iommu.c | 24 +++++++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 0c878caaa0a2..7fce4ef77bdd 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -146,12 +146,21 @@ #define PAGE_MODE_1_LEVEL 0x01 #define PAGE_MODE_2_LEVEL 0x02 #define PAGE_MODE_3_LEVEL 0x03 +#define PAGE_MODE_4_LEVEL 0x04 +#define PAGE_MODE_5_LEVEL 0x05 +#define PAGE_MODE_6_LEVEL 0x06 #define IOMMU_PDE_NL_0 0x000ULL #define IOMMU_PDE_NL_1 0x200ULL #define IOMMU_PDE_NL_2 0x400ULL #define IOMMU_PDE_NL_3 0x600ULL +#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) +#define PM_LEVEL_SIZE(x) (((x) < 6) ? \ + ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ + (0xffffffffffffffffULL)) +#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) + #define IOMMU_PTE_L2_INDEX(address) (((address) >> 30) & 0x1ffULL) #define IOMMU_PTE_L1_INDEX(address) (((address) >> 21) & 0x1ffULL) #define IOMMU_PTE_L0_INDEX(address) (((address) >> 12) & 0x1ffULL) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 6c99f5037801..29bcd358b6ce 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -61,6 +61,8 @@ static u64* alloc_pte(struct protection_domain *dom, static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, unsigned long start_page, unsigned int pages); +static u64 *fetch_pte(struct protection_domain *domain, + unsigned long address); #ifndef BUS_NOTIFY_UNBOUND_DRIVER #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005 @@ -670,24 +672,24 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, * This function checks if there is a PTE for a given dma address. If * there is one, it returns the pointer to it. */ -static u64* fetch_pte(struct protection_domain *domain, +static u64 *fetch_pte(struct protection_domain *domain, unsigned long address) { + int level; u64 *pte; - pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(address)]; + level = domain->mode - 1; + pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; - if (!IOMMU_PTE_PRESENT(*pte)) - return NULL; + while (level > 0) { + if (!IOMMU_PTE_PRESENT(*pte)) + return NULL; - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L1_INDEX(address)]; + level -= 1; - if (!IOMMU_PTE_PRESENT(*pte)) - return NULL; - - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L0_INDEX(address)]; + pte = IOMMU_PTE_PAGE(*pte); + pte = &pte[PM_LEVEL_INDEX(level, address)]; + } return pte; } From 38a76eeeafb251bf67d143a34b37a8105cff302e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 17:02:47 +0200 Subject: [PATCH 02/14] x86/amd-iommu: Use fetch_pte in iommu_unmap_page Instead of reimplementing existing logic use fetch_pte to walk the page table in iommu_unmap_page. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 29bcd358b6ce..5e5279899994 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -546,23 +546,10 @@ static int iommu_map_page(struct protection_domain *dom, static void iommu_unmap_page(struct protection_domain *dom, unsigned long bus_addr) { - u64 *pte; + u64 *pte = fetch_pte(dom, bus_addr); - pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)]; - - if (!IOMMU_PTE_PRESENT(*pte)) - return; - - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)]; - - if (!IOMMU_PTE_PRESENT(*pte)) - return; - - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)]; - - *pte = 0; + if (pte) + *pte = 0; } /* From a6d41a4027b758a9473197a78fab45afb31003aa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 17:08:55 +0200 Subject: [PATCH 03/14] x86/amd-iommu: Use fetch_pte in amd_iommu_iova_to_phys Don't reimplement the page table walker in this function. Use the generic one. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 5e5279899994..4a54366d422d 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -2140,21 +2140,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, phys_addr_t paddr; u64 *pte; - pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(iova)]; + pte = fetch_pte(domain, iova); - if (!IOMMU_PTE_PRESENT(*pte)) - return 0; - - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L1_INDEX(iova)]; - - if (!IOMMU_PTE_PRESENT(*pte)) - return 0; - - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L0_INDEX(iova)]; - - if (!IOMMU_PTE_PRESENT(*pte)) + if (!pte || !IOMMU_PTE_PRESENT(*pte)) return 0; paddr = *pte & IOMMU_PAGE_MASK; From 6a0dbcbe4e612fbc9d73cd4dde8ebef19295058a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 15:41:59 +0200 Subject: [PATCH 04/14] x86/amd-iommu: Add a gneric version of amd_iommu_flush_all_devices This patch adds a generic variant of amd_iommu_flush_all_devices function which flushes only the DTEs for a given protection domain. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 4a54366d422d..5265dd17eb5c 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -481,13 +481,14 @@ void amd_iommu_flush_all_domains(void) } } -void amd_iommu_flush_all_devices(void) +static void flush_devices_by_domain(struct protection_domain *domain) { struct amd_iommu *iommu; int i; for (i = 0; i <= amd_iommu_last_bdf; ++i) { - if (amd_iommu_pd_table[i] == NULL) + if ((domain == NULL && amd_iommu_pd_table[i] == NULL) || + (amd_iommu_pd_table[i] != domain)) continue; iommu = amd_iommu_rlookup_table[i]; @@ -499,6 +500,11 @@ void amd_iommu_flush_all_devices(void) } } +void amd_iommu_flush_all_devices(void) +{ + flush_devices_by_domain(NULL); +} + /**************************************************************************** * * The functions below are used the create the page table mappings for From 407d733e30a97daf5ea6f9eb5f9ebbd42a0a9ef2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 16:07:00 +0200 Subject: [PATCH 05/14] x86/amd-iommu: Introduce set_dte_entry function This function factors out some logic of attach_device to a seperate function. This new function will be used to update device table entries when necessary. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 5265dd17eb5c..0fab1f1d135e 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1058,18 +1058,10 @@ static struct protection_domain *domain_for_device(u16 devid) return dom; } -/* - * If a device is not yet associated with a domain, this function does - * assigns it visible for the hardware - */ -static void attach_device(struct amd_iommu *iommu, - struct protection_domain *domain, - u16 devid) +static void set_dte_entry(u16 devid, struct protection_domain *domain) { - unsigned long flags; u64 pte_root = virt_to_phys(domain->pt_root); - - domain->dev_cnt += 1; + unsigned long flags; pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) << DEV_ENTRY_MODE_SHIFT; @@ -1082,6 +1074,21 @@ static void attach_device(struct amd_iommu *iommu, amd_iommu_pd_table[devid] = domain; write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); +} + +/* + * If a device is not yet associated with a domain, this function does + * assigns it visible for the hardware + */ +static void attach_device(struct amd_iommu *iommu, + struct protection_domain *domain, + u16 devid) +{ + /* set the DTE entry */ + set_dte_entry(devid, domain); + + /* increase reference counter */ + domain->dev_cnt += 1; /* * We might boot into a crash-kernel here. The crashed kernel From 04bfdd8406099fca2e6b8844748c4d6c5eba8c8d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 16:00:23 +0200 Subject: [PATCH 06/14] x86/amd-iommu: Flush domains if address space size was increased Thist patch introduces the update_domain function which propagates the larger address space of a protection domain to the device table and flushes all relevant DTEs and the domain TLB. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 32 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 7fce4ef77bdd..97f3d09d3be5 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -235,6 +235,7 @@ struct protection_domain { int mode; /* paging mode (0-6 levels) */ u64 *pt_root; /* page table root pointer */ unsigned long flags; /* flags to find out type of domain */ + bool updated; /* complete domain flush required */ unsigned dev_cnt; /* devices assigned to this domain */ void *priv; /* private data */ }; diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 0fab1f1d135e..5eab6a84b9cc 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -63,6 +63,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, unsigned int pages); static u64 *fetch_pte(struct protection_domain *domain, unsigned long address); +static void update_domain(struct protection_domain *domain); #ifndef BUS_NOTIFY_UNBOUND_DRIVER #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005 @@ -546,6 +547,8 @@ static int iommu_map_page(struct protection_domain *dom, *pte = __pte; + update_domain(dom); + return 0; } @@ -762,9 +765,13 @@ static int alloc_new_range(struct amd_iommu *iommu, dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1); } + update_domain(&dma_dom->domain); + return 0; out_free: + update_domain(&dma_dom->domain); + free_page((unsigned long)dma_dom->aperture[index]->bitmap); kfree(dma_dom->aperture[index]); @@ -1294,6 +1301,29 @@ static int get_device_resources(struct device *dev, return 1; } +static void update_device_table(struct protection_domain *domain) +{ + int i; + + for (i = 0; i <= amd_iommu_last_bdf; ++i) { + if (amd_iommu_pd_table[i] != domain) + continue; + set_dte_entry(i, domain); + } +} + +static void update_domain(struct protection_domain *domain) +{ + if (!domain->updated) + return; + + update_device_table(domain); + flush_devices_by_domain(domain); + iommu_flush_domain(domain->id); + + domain->updated = false; +} + /* * If the pte_page is not yet allocated this function is called */ @@ -1351,6 +1381,8 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, } else pte += IOMMU_PTE_L0_INDEX(address); + update_domain(&dom->domain); + return pte; } From 50020fb6324465e478d6c8cdbf3c695f0a60358d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 15:38:40 +0200 Subject: [PATCH 07/14] x86/amd-iommu: Introduce increase_address_space function This function will be used to increase the address space size of a protection domain. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 4 ++++ arch/x86/kernel/amd_iommu.c | 27 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 97f3d09d3be5..1b4b3d6c9f04 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -160,6 +160,10 @@ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ (0xffffffffffffffffULL)) #define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) +#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL) +#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \ + IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) + #define IOMMU_PTE_L2_INDEX(address) (((address) >> 30) & 0x1ffULL) #define IOMMU_PTE_L1_INDEX(address) (((address) >> 21) & 0x1ffULL) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 5eab6a84b9cc..fc97b51f0287 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1324,6 +1324,33 @@ static void update_domain(struct protection_domain *domain) domain->updated = false; } +/* + * This function is used to add another level to an IO page table. Adding + * another level increases the size of the address space by 9 bits to a size up + * to 64 bits. + */ +static bool increase_address_space(struct protection_domain *domain, + gfp_t gfp) +{ + u64 *pte; + + if (domain->mode == PAGE_MODE_6_LEVEL) + /* address space already 64 bit large */ + return false; + + pte = (void *)get_zeroed_page(gfp); + if (!pte) + return false; + + *pte = PM_LEVEL_PDE(domain->mode, + virt_to_phys(domain->pt_root)); + domain->pt_root = pte; + domain->mode += 1; + domain->updated = true; + + return true; +} + /* * If the pte_page is not yet allocated this function is called */ From 8bc3e127421bf3b735edbde05135892c12c5f615 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 16:48:40 +0200 Subject: [PATCH 08/14] x86/amd-iommu: Change alloc_pte to support 64 bit address space This patch changes the alloc_pte function to be able to map pages into the whole 64 bit address space supported by AMD IOMMU hardware from the old limit of 2**39 bytes. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 52 +++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index fc97b51f0287..3be2b61fc31f 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -55,7 +55,7 @@ struct iommu_cmd { static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, struct unity_map_entry *e); static struct dma_ops_domain *find_protection_domain(u16 devid); -static u64* alloc_pte(struct protection_domain *dom, +static u64 *alloc_pte(struct protection_domain *domain, unsigned long address, u64 **pte_page, gfp_t gfp); static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, @@ -1351,40 +1351,36 @@ static bool increase_address_space(struct protection_domain *domain, return true; } -/* - * If the pte_page is not yet allocated this function is called - */ -static u64* alloc_pte(struct protection_domain *dom, +static u64 *alloc_pte(struct protection_domain *domain, unsigned long address, u64 **pte_page, gfp_t gfp) { u64 *pte, *page; + int level; - pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)]; + while (address > PM_LEVEL_SIZE(domain->mode)) + increase_address_space(domain, gfp); - if (!IOMMU_PTE_PRESENT(*pte)) { - page = (u64 *)get_zeroed_page(gfp); - if (!page) - return NULL; - *pte = IOMMU_L2_PDE(virt_to_phys(page)); + level = domain->mode - 1; + pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; + + while (level > 0) { + if (!IOMMU_PTE_PRESENT(*pte)) { + page = (u64 *)get_zeroed_page(gfp); + if (!page) + return NULL; + *pte = PM_LEVEL_PDE(level, virt_to_phys(page)); + } + + level -= 1; + + pte = IOMMU_PTE_PAGE(*pte); + + if (pte_page && level == 0) + *pte_page = pte; + + pte = &pte[PM_LEVEL_INDEX(level, address)]; } - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[IOMMU_PTE_L1_INDEX(address)]; - - if (!IOMMU_PTE_PRESENT(*pte)) { - page = (u64 *)get_zeroed_page(gfp); - if (!page) - return NULL; - *pte = IOMMU_L1_PDE(virt_to_phys(page)); - } - - pte = IOMMU_PTE_PAGE(*pte); - - if (pte_page) - *pte_page = pte; - - pte = &pte[IOMMU_PTE_L0_INDEX(address)]; - return pte; } From 8c8c143cdc95ebe50fd962917556e25e8912997b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 17:30:00 +0200 Subject: [PATCH 09/14] x86/amd-iommu: Remove last usages of IOMMU_PTE_L0_INDEX This change allows to remove these old macros later. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 3be2b61fc31f..ebc1c8443926 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1402,7 +1402,7 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC); aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page; } else - pte += IOMMU_PTE_L0_INDEX(address); + pte += PM_LEVEL_INDEX(0, address); update_domain(&dom->domain); @@ -1466,7 +1466,7 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu, if (!pte) return; - pte += IOMMU_PTE_L0_INDEX(address); + pte += PM_LEVEL_INDEX(0, address); WARN_ON(!*pte); From bad1cac28a707c69301a4d0612da9ccbecd51953 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 16:52:23 +0200 Subject: [PATCH 10/14] x86/amd-iommu: Remove bus_addr check in iommu_map_page The driver now supports full 64 bit device address spaces. So this check is not longer required. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index ebc1c8443926..6ffb3e637657 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -530,8 +530,7 @@ static int iommu_map_page(struct protection_domain *dom, bus_addr = PAGE_ALIGN(bus_addr); phys_addr = PAGE_ALIGN(phys_addr); - /* only support 512GB address spaces for now */ - if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK)) + if (!(prot & IOMMU_PROT_MASK)) return -EINVAL; pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL); From 8f7a017ce05ed4522809448e169daa44fe6edeb1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 16:55:24 +0200 Subject: [PATCH 11/14] x86/amd-iommu: Use 2-level page tables for dma_ops domains The driver now supports a dynamic number of levels for IO page tables. This allows to reduce the number of levels for dma_ops domains by one because a dma_ops domain has usually an address space size between 128MB and 4G. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 6ffb3e637657..addf6588c366 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1010,7 +1010,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu) dma_dom->domain.id = domain_id_alloc(); if (dma_dom->domain.id == 0) goto free_dma_dom; - dma_dom->domain.mode = PAGE_MODE_3_LEVEL; + dma_dom->domain.mode = PAGE_MODE_2_LEVEL; dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); dma_dom->domain.flags = PD_DMA_OPS_MASK; dma_dom->domain.priv = dma_dom; From 674d798a80cb6ea1defa01899099f40d9124423c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 2 Sep 2009 17:26:09 +0200 Subject: [PATCH 12/14] x86/amd-iommu: Remove old page table handling macros These macros are not longer required. So remove them. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 1b4b3d6c9f04..d66430de5f7c 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -150,11 +150,6 @@ #define PAGE_MODE_5_LEVEL 0x05 #define PAGE_MODE_6_LEVEL 0x06 -#define IOMMU_PDE_NL_0 0x000ULL -#define IOMMU_PDE_NL_1 0x200ULL -#define IOMMU_PDE_NL_2 0x400ULL -#define IOMMU_PDE_NL_3 0x600ULL - #define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) #define PM_LEVEL_SIZE(x) (((x) < 6) ? \ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ @@ -164,15 +159,6 @@ #define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) - -#define IOMMU_PTE_L2_INDEX(address) (((address) >> 30) & 0x1ffULL) -#define IOMMU_PTE_L1_INDEX(address) (((address) >> 21) & 0x1ffULL) -#define IOMMU_PTE_L0_INDEX(address) (((address) >> 12) & 0x1ffULL) - -#define IOMMU_MAP_SIZE_L1 (1ULL << 21) -#define IOMMU_MAP_SIZE_L2 (1ULL << 30) -#define IOMMU_MAP_SIZE_L3 (1ULL << 39) - #define IOMMU_PTE_P (1ULL << 0) #define IOMMU_PTE_TV (1ULL << 1) #define IOMMU_PTE_U (1ULL << 59) @@ -180,11 +166,6 @@ #define IOMMU_PTE_IR (1ULL << 61) #define IOMMU_PTE_IW (1ULL << 62) -#define IOMMU_L1_PDE(address) \ - ((address) | IOMMU_PDE_NL_1 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) -#define IOMMU_L2_PDE(address) \ - ((address) | IOMMU_PDE_NL_2 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) - #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) #define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P) #define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK)) From a6b256b41357c33ccb2d105a4457e22bdc83e021 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 3 Sep 2009 12:21:31 +0200 Subject: [PATCH 13/14] x86/amd-iommu: Support higher level PTEs in iommu_page_unmap This patch changes fetch_pte and iommu_page_unmap to support different page sizes too. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index d66430de5f7c..351ca39ece05 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -158,6 +158,7 @@ #define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL) #define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) +#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL) #define IOMMU_PTE_P (1ULL << 0) #define IOMMU_PTE_TV (1ULL << 1) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index addf6588c366..002cf9cab9e9 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -62,7 +62,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, unsigned long start_page, unsigned int pages); static u64 *fetch_pte(struct protection_domain *domain, - unsigned long address); + unsigned long address, int map_size); static void update_domain(struct protection_domain *domain); #ifndef BUS_NOTIFY_UNBOUND_DRIVER @@ -552,9 +552,9 @@ static int iommu_map_page(struct protection_domain *dom, } static void iommu_unmap_page(struct protection_domain *dom, - unsigned long bus_addr) + unsigned long bus_addr, int map_size) { - u64 *pte = fetch_pte(dom, bus_addr); + u64 *pte = fetch_pte(dom, bus_addr, map_size); if (pte) *pte = 0; @@ -668,7 +668,7 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, * there is one, it returns the pointer to it. */ static u64 *fetch_pte(struct protection_domain *domain, - unsigned long address) + unsigned long address, int map_size) { int level; u64 *pte; @@ -676,7 +676,7 @@ static u64 *fetch_pte(struct protection_domain *domain, level = domain->mode - 1; pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; - while (level > 0) { + while (level > map_size) { if (!IOMMU_PTE_PRESENT(*pte)) return NULL; @@ -684,6 +684,11 @@ static u64 *fetch_pte(struct protection_domain *domain, pte = IOMMU_PTE_PAGE(*pte); pte = &pte[PM_LEVEL_INDEX(level, address)]; + + if ((PM_PTE_LEVEL(*pte) == 0) && level != map_size) { + pte = NULL; + break; + } } return pte; @@ -757,7 +762,7 @@ static int alloc_new_range(struct amd_iommu *iommu, for (i = dma_dom->aperture[index]->offset; i < dma_dom->aperture_size; i += PAGE_SIZE) { - u64 *pte = fetch_pte(&dma_dom->domain, i); + u64 *pte = fetch_pte(&dma_dom->domain, i, PM_MAP_4k); if (!pte || !IOMMU_PTE_PRESENT(*pte)) continue; @@ -2192,7 +2197,7 @@ static void amd_iommu_unmap_range(struct iommu_domain *dom, iova &= PAGE_MASK; for (i = 0; i < npages; ++i) { - iommu_unmap_page(domain, iova); + iommu_unmap_page(domain, iova, PM_MAP_4k); iova += PAGE_SIZE; } @@ -2207,7 +2212,7 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, phys_addr_t paddr; u64 *pte; - pte = fetch_pte(domain, iova); + pte = fetch_pte(domain, iova, PM_MAP_4k); if (!pte || !IOMMU_PTE_PRESENT(*pte)) return 0; From abdc5eb3d69279039ba6cb89719913d08013ab14 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 3 Sep 2009 11:33:51 +0200 Subject: [PATCH 14/14] x86/amd-iommu: Change iommu_map_page to support multiple page sizes This patch adds a map_size parameter to the iommu_map_page function which makes it generic enough to handle multiple page sizes. This also requires a change to alloc_pte which is also done in this patch. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 6 +++++ arch/x86/kernel/amd_iommu.c | 31 +++++++++++++++++--------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 351ca39ece05..9e8bb9746dcf 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -160,6 +160,12 @@ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) #define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL) +#define PM_MAP_4k 0 +#define PM_ADDR_MASK 0x000ffffffffff000ULL +#define PM_MAP_MASK(lvl) (PM_ADDR_MASK & \ + (~((1ULL << (12 + ((lvl) * 9))) - 1))) +#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr)) + #define IOMMU_PTE_P (1ULL << 0) #define IOMMU_PTE_TV (1ULL << 1) #define IOMMU_PTE_U (1ULL << 59) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 002cf9cab9e9..45be9499c973 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -56,8 +56,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, struct unity_map_entry *e); static struct dma_ops_domain *find_protection_domain(u16 devid); static u64 *alloc_pte(struct protection_domain *domain, - unsigned long address, u64 - **pte_page, gfp_t gfp); + unsigned long address, int end_lvl, + u64 **pte_page, gfp_t gfp); static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, unsigned long start_page, unsigned int pages); @@ -523,17 +523,21 @@ void amd_iommu_flush_all_devices(void) static int iommu_map_page(struct protection_domain *dom, unsigned long bus_addr, unsigned long phys_addr, - int prot) + int prot, + int map_size) { u64 __pte, *pte; bus_addr = PAGE_ALIGN(bus_addr); phys_addr = PAGE_ALIGN(phys_addr); + BUG_ON(!PM_ALIGNED(map_size, bus_addr)); + BUG_ON(!PM_ALIGNED(map_size, phys_addr)); + if (!(prot & IOMMU_PROT_MASK)) return -EINVAL; - pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL); + pte = alloc_pte(dom, bus_addr, map_size, NULL, GFP_KERNEL); if (IOMMU_PTE_PRESENT(*pte)) return -EBUSY; @@ -612,7 +616,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, for (addr = e->address_start; addr < e->address_end; addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot); + ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, + PM_MAP_4k); if (ret) return ret; /* @@ -729,7 +734,7 @@ static int alloc_new_range(struct amd_iommu *iommu, u64 *pte, *pte_page; for (i = 0; i < num_ptes; ++i) { - pte = alloc_pte(&dma_dom->domain, address, + pte = alloc_pte(&dma_dom->domain, address, PM_MAP_4k, &pte_page, gfp); if (!pte) goto out_free; @@ -1356,7 +1361,10 @@ static bool increase_address_space(struct protection_domain *domain, } static u64 *alloc_pte(struct protection_domain *domain, - unsigned long address, u64 **pte_page, gfp_t gfp) + unsigned long address, + int end_lvl, + u64 **pte_page, + gfp_t gfp) { u64 *pte, *page; int level; @@ -1367,7 +1375,7 @@ static u64 *alloc_pte(struct protection_domain *domain, level = domain->mode - 1; pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; - while (level > 0) { + while (level > end_lvl) { if (!IOMMU_PTE_PRESENT(*pte)) { page = (u64 *)get_zeroed_page(gfp); if (!page) @@ -1379,7 +1387,7 @@ static u64 *alloc_pte(struct protection_domain *domain, pte = IOMMU_PTE_PAGE(*pte); - if (pte_page && level == 0) + if (pte_page && level == end_lvl) *pte_page = pte; pte = &pte[PM_LEVEL_INDEX(level, address)]; @@ -1403,7 +1411,8 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; if (!pte) { - pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC); + pte = alloc_pte(&dom->domain, address, PM_MAP_4k, &pte_page, + GFP_ATOMIC); aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page; } else pte += PM_LEVEL_INDEX(0, address); @@ -2176,7 +2185,7 @@ static int amd_iommu_map_range(struct iommu_domain *dom, paddr &= PAGE_MASK; for (i = 0; i < npages; ++i) { - ret = iommu_map_page(domain, iova, paddr, prot); + ret = iommu_map_page(domain, iova, paddr, prot, PM_MAP_4k); if (ret) return ret;