From 2130a790ca49763f724ec45cf93b9dd765e2023e Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 2 Jun 2022 15:05:26 +0200 Subject: [PATCH 01/11] kernel: add platform_has() infrastructure Add a simple infrastructure for setting, resetting and querying platform feature flags. Flags can be either global or architecture specific. Signed-off-by: Juergen Gross Reviewed-by: Oleksandr Tyshchenko Tested-by: Oleksandr Tyshchenko # Arm64 only Reviewed-by: Christoph Hellwig Acked-by: Borislav Petkov Signed-off-by: Juergen Gross --- MAINTAINERS | 8 ++++++++ include/asm-generic/Kbuild | 1 + include/asm-generic/platform-feature.h | 8 ++++++++ include/linux/platform-feature.h | 15 ++++++++++++++ kernel/Makefile | 2 +- kernel/platform-feature.c | 27 ++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 include/asm-generic/platform-feature.h create mode 100644 include/linux/platform-feature.h create mode 100644 kernel/platform-feature.c diff --git a/MAINTAINERS b/MAINTAINERS index a6d3bd9d2a8d..06678abc22ca 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15824,6 +15824,14 @@ S: Maintained F: Documentation/devicetree/bindings/iio/chemical/plantower,pms7003.yaml F: drivers/iio/chemical/pms7003.c +PLATFORM FEATURE INFRASTRUCTURE +M: Juergen Gross +S: Maintained +F: arch/*/include/asm/platform-feature.h +F: include/asm-generic/platform-feature.h +F: include/linux/platform-feature.h +F: kernel/platform-feature.c + PLDMFW LIBRARY M: Jacob Keller S: Maintained diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 302506bbc2a4..8e47d483b524 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -44,6 +44,7 @@ mandatory-y += msi.h mandatory-y += pci.h mandatory-y += percpu.h mandatory-y += pgalloc.h +mandatory-y += platform-feature.h mandatory-y += preempt.h mandatory-y += rwonce.h mandatory-y += sections.h diff --git a/include/asm-generic/platform-feature.h b/include/asm-generic/platform-feature.h new file mode 100644 index 000000000000..4b0af3d51588 --- /dev/null +++ b/include/asm-generic/platform-feature.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_PLATFORM_FEATURE_H +#define _ASM_GENERIC_PLATFORM_FEATURE_H + +/* Number of arch specific feature flags. */ +#define PLATFORM_ARCH_FEAT_N 0 + +#endif /* _ASM_GENERIC_PLATFORM_FEATURE_H */ diff --git a/include/linux/platform-feature.h b/include/linux/platform-feature.h new file mode 100644 index 000000000000..6ed859928b97 --- /dev/null +++ b/include/linux/platform-feature.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _PLATFORM_FEATURE_H +#define _PLATFORM_FEATURE_H + +#include +#include + +/* The platform features are starting with the architecture specific ones. */ +#define PLATFORM_FEAT_N (0 + PLATFORM_ARCH_FEAT_N) + +void platform_set(unsigned int feature); +void platform_clear(unsigned int feature); +bool platform_has(unsigned int feature); + +#endif /* _PLATFORM_FEATURE_H */ diff --git a/kernel/Makefile b/kernel/Makefile index 318789c728d3..a7e1f49ab2b3 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -7,7 +7,7 @@ obj-y = fork.o exec_domain.o panic.o \ cpu.o exit.o softirq.o resource.o \ sysctl.o capability.o ptrace.o user.o \ signal.o sys.o umh.o workqueue.o pid.o task_work.o \ - extable.o params.o \ + extable.o params.o platform-feature.o \ kthread.o sys_ni.o nsproxy.o \ notifier.o ksysfs.o cred.o reboot.o \ async.o range.o smpboot.o ucount.o regset.o diff --git a/kernel/platform-feature.c b/kernel/platform-feature.c new file mode 100644 index 000000000000..cb6a6c3e4fed --- /dev/null +++ b/kernel/platform-feature.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +#define PLATFORM_FEAT_ARRAY_SZ BITS_TO_LONGS(PLATFORM_FEAT_N) +static unsigned long __read_mostly platform_features[PLATFORM_FEAT_ARRAY_SZ]; + +void platform_set(unsigned int feature) +{ + set_bit(feature, platform_features); +} +EXPORT_SYMBOL_GPL(platform_set); + +void platform_clear(unsigned int feature) +{ + clear_bit(feature, platform_features); +} +EXPORT_SYMBOL_GPL(platform_clear); + +bool platform_has(unsigned int feature) +{ + return test_bit(feature, platform_features); +} +EXPORT_SYMBOL_GPL(platform_has); From 3f9dfbebdc48cebfbda738f6f3d1dbf6d7232f90 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Mon, 6 Jun 2022 08:09:16 +0200 Subject: [PATCH 02/11] virtio: replace arch_has_restricted_virtio_memory_access() Instead of using arch_has_restricted_virtio_memory_access() together with CONFIG_ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS, replace those with platform_has() and a new platform feature PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS. Signed-off-by: Juergen Gross Reviewed-by: Oleksandr Tyshchenko Tested-by: Oleksandr Tyshchenko # Arm64 only Reviewed-by: Christoph Hellwig Acked-by: Borislav Petkov --- arch/s390/Kconfig | 1 - arch/s390/mm/init.c | 13 +++---------- arch/x86/Kconfig | 1 - arch/x86/mm/mem_encrypt.c | 7 ------- arch/x86/mm/mem_encrypt_amd.c | 4 ++++ drivers/virtio/Kconfig | 6 ------ drivers/virtio/virtio.c | 5 ++--- include/linux/platform-feature.h | 6 +++++- include/linux/virtio_config.h | 9 --------- 9 files changed, 14 insertions(+), 38 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index b1a88f6cc349..bf253628c136 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -768,7 +768,6 @@ menu "Virtualization" config PROTECTED_VIRTUALIZATION_GUEST def_bool n prompt "Protected virtualization guest support" - select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS help Select this option, if you want to be able to run this kernel as a protected virtualization KVM guest. diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 6fb6bf64326f..6a0ac00d5a42 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -168,22 +169,14 @@ bool force_dma_unencrypted(struct device *dev) return is_prot_virt_guest(); } -#ifdef CONFIG_ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS - -int arch_has_restricted_virtio_memory_access(void) -{ - return is_prot_virt_guest(); -} -EXPORT_SYMBOL(arch_has_restricted_virtio_memory_access); - -#endif - /* protected virtualization */ static void pv_init(void) { if (!is_prot_virt_guest()) return; + platform_set(PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS); + /* make sure bounce buffers are shared */ swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE); swiotlb_update_mem_attributes(); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9783ebc4e021..be0b95e51df6 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1542,7 +1542,6 @@ config X86_CPA_STATISTICS config X86_MEM_ENCRYPT select ARCH_HAS_FORCE_DMA_UNENCRYPTED select DYNAMIC_PHYSICAL_MASK - select ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS def_bool n config AMD_MEM_ENCRYPT diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index 11350e2fd736..9f27e14e185f 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -12,7 +12,6 @@ #include #include #include -#include /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */ bool force_dma_unencrypted(struct device *dev) @@ -87,9 +86,3 @@ void __init mem_encrypt_init(void) print_mem_encrypt_feature_info(); } - -int arch_has_restricted_virtio_memory_access(void) -{ - return cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT); -} -EXPORT_SYMBOL_GPL(arch_has_restricted_virtio_memory_access); diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index e8f7953fda83..f6d038e2cd8e 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -242,6 +243,9 @@ void __init sev_setup_arch(void) size = total_mem * 6 / 100; size = clamp_val(size, IO_TLB_DEFAULT_SIZE, SZ_1G); swiotlb_adjust_size(size); + + /* Set restricted memory access for virtio. */ + platform_set(PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS); } static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot) diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index b5adf6abd241..a6dc8b5846fe 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -6,12 +6,6 @@ config VIRTIO bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_RPMSG or CONFIG_S390_GUEST. -config ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS - bool - help - This option is selected if the architecture may need to enforce - VIRTIO_F_ACCESS_PLATFORM - config VIRTIO_PCI_LIB tristate help diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index ef04a96942bf..6bace84ae37e 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -5,6 +5,7 @@ #include #include #include +#include #include /* Unique numbering for virtio devices. */ @@ -170,12 +171,10 @@ EXPORT_SYMBOL_GPL(virtio_add_status); static int virtio_features_ok(struct virtio_device *dev) { unsigned int status; - int ret; might_sleep(); - ret = arch_has_restricted_virtio_memory_access(); - if (ret) { + if (platform_has(PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS)) { if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) { dev_warn(&dev->dev, "device must provide VIRTIO_F_VERSION_1\n"); diff --git a/include/linux/platform-feature.h b/include/linux/platform-feature.h index 6ed859928b97..b2f48be999fa 100644 --- a/include/linux/platform-feature.h +++ b/include/linux/platform-feature.h @@ -6,7 +6,11 @@ #include /* The platform features are starting with the architecture specific ones. */ -#define PLATFORM_FEAT_N (0 + PLATFORM_ARCH_FEAT_N) + +/* Used to enable platform specific DMA handling for virtio devices. */ +#define PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS (0 + PLATFORM_ARCH_FEAT_N) + +#define PLATFORM_FEAT_N (1 + PLATFORM_ARCH_FEAT_N) void platform_set(unsigned int feature); void platform_clear(unsigned int feature); diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 9a36051ceb76..49c7c32815f1 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -604,13 +604,4 @@ static inline void virtio_cwrite64(struct virtio_device *vdev, _r; \ }) -#ifdef CONFIG_ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS -int arch_has_restricted_virtio_memory_access(void); -#else -static inline int arch_has_restricted_virtio_memory_access(void) -{ - return 0; -} -#endif /* CONFIG_ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS */ - #endif /* _LINUX_VIRTIO_CONFIG_H */ From 9bf22421dc8a69cade3c994771637e9693ff0216 Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Thu, 2 Jun 2022 22:23:46 +0300 Subject: [PATCH 03/11] arm/xen: Introduce xen_setup_dma_ops() This patch introduces new helper and places it in new header. The helper's purpose is to assign any Xen specific DMA ops in a single place. For now, we deal with xen-swiotlb DMA ops only. The one of the subsequent commits in current series will add xen-grant DMA ops case. Also re-use the xen_swiotlb_detect() check on Arm32. Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini [For arm64] Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/1654197833-25362-2-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- arch/arm/include/asm/xen/xen-ops.h | 2 ++ arch/arm/mm/dma-mapping.c | 7 ++----- arch/arm64/include/asm/xen/xen-ops.h | 2 ++ arch/arm64/mm/dma-mapping.c | 7 ++----- include/xen/arm/xen-ops.h | 15 +++++++++++++++ 5 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 arch/arm/include/asm/xen/xen-ops.h create mode 100644 arch/arm64/include/asm/xen/xen-ops.h create mode 100644 include/xen/arm/xen-ops.h diff --git a/arch/arm/include/asm/xen/xen-ops.h b/arch/arm/include/asm/xen/xen-ops.h new file mode 100644 index 000000000000..7ebb7eb0bd93 --- /dev/null +++ b/arch/arm/include/asm/xen/xen-ops.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 82ffac621854..059cce018570 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include "dma.h" #include "mm.h" @@ -2287,10 +2287,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, set_dma_ops(dev, dma_ops); -#ifdef CONFIG_XEN - if (xen_initial_domain()) - dev->dma_ops = &xen_swiotlb_dma_ops; -#endif + xen_setup_dma_ops(dev); dev->archdata.dma_ops_setup = true; } diff --git a/arch/arm64/include/asm/xen/xen-ops.h b/arch/arm64/include/asm/xen/xen-ops.h new file mode 100644 index 000000000000..7ebb7eb0bd93 --- /dev/null +++ b/arch/arm64/include/asm/xen/xen-ops.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 6719f9efea09..6099c81b9322 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -9,9 +9,9 @@ #include #include #include -#include #include +#include void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir) @@ -52,8 +52,5 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, if (iommu) iommu_setup_dma_ops(dev, dma_base, dma_base + size - 1); -#ifdef CONFIG_XEN - if (xen_swiotlb_detect()) - dev->dma_ops = &xen_swiotlb_dma_ops; -#endif + xen_setup_dma_ops(dev); } diff --git a/include/xen/arm/xen-ops.h b/include/xen/arm/xen-ops.h new file mode 100644 index 000000000000..288deb1c3ac9 --- /dev/null +++ b/include/xen/arm/xen-ops.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ARM_XEN_OPS_H +#define _ASM_ARM_XEN_OPS_H + +#include + +static inline void xen_setup_dma_ops(struct device *dev) +{ +#ifdef CONFIG_XEN + if (xen_swiotlb_detect()) + dev->dma_ops = &xen_swiotlb_dma_ops; +#endif +} + +#endif /* _ASM_ARM_XEN_OPS_H */ From 02a9e681a3f7998074f39ec265080bf934871530 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 2 Jun 2022 22:23:47 +0300 Subject: [PATCH 04/11] xen/grants: support allocating consecutive grants For support of virtio via grant mappings in rare cases larger mappings using consecutive grants are needed. Support those by adding a bitmap of free grants. As consecutive grants will be needed only in very rare cases (e.g. when configuring a virtio device with a multi-page ring), optimize for the normal case of non-consecutive allocations. Signed-off-by: Juergen Gross Reviewed-by: Boris Ostrovsky Link: https://lore.kernel.org/r/1654197833-25362-3-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/grant-table.c | 255 ++++++++++++++++++++++++++++++++------ include/xen/grant_table.h | 4 + 2 files changed, 221 insertions(+), 38 deletions(-) diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 7a18292540bc..738029de3c67 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -33,6 +33,7 @@ #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -70,9 +71,32 @@ static grant_ref_t **gnttab_list; static unsigned int nr_grant_frames; + +/* + * Handling of free grants: + * + * Free grants are in a simple list anchored in gnttab_free_head. They are + * linked by grant ref, the last element contains GNTTAB_LIST_END. The number + * of free entries is stored in gnttab_free_count. + * Additionally there is a bitmap of free entries anchored in + * gnttab_free_bitmap. This is being used for simplifying allocation of + * multiple consecutive grants, which is needed e.g. for support of virtio. + * gnttab_last_free is used to add free entries of new frames at the end of + * the free list. + * gnttab_free_tail_ptr specifies the variable which references the start + * of consecutive free grants ending with gnttab_last_free. This pointer is + * updated in a rather defensive way, in order to avoid performance hits in + * hot paths. + * All those variables are protected by gnttab_list_lock. + */ static int gnttab_free_count; -static grant_ref_t gnttab_free_head; +static unsigned int gnttab_size; +static grant_ref_t gnttab_free_head = GNTTAB_LIST_END; +static grant_ref_t gnttab_last_free = GNTTAB_LIST_END; +static grant_ref_t *gnttab_free_tail_ptr; +static unsigned long *gnttab_free_bitmap; static DEFINE_SPINLOCK(gnttab_list_lock); + struct grant_frames xen_auto_xlat_grant_frames; static unsigned int xen_gnttab_version; module_param_named(version, xen_gnttab_version, uint, 0); @@ -168,16 +192,116 @@ static int get_free_entries(unsigned count) ref = head = gnttab_free_head; gnttab_free_count -= count; - while (count-- > 1) - head = gnttab_entry(head); + while (count--) { + bitmap_clear(gnttab_free_bitmap, head, 1); + if (gnttab_free_tail_ptr == __gnttab_entry(head)) + gnttab_free_tail_ptr = &gnttab_free_head; + if (count) + head = gnttab_entry(head); + } gnttab_free_head = gnttab_entry(head); gnttab_entry(head) = GNTTAB_LIST_END; + if (!gnttab_free_count) { + gnttab_last_free = GNTTAB_LIST_END; + gnttab_free_tail_ptr = NULL; + } + spin_unlock_irqrestore(&gnttab_list_lock, flags); return ref; } +static int get_seq_entry_count(void) +{ + if (gnttab_last_free == GNTTAB_LIST_END || !gnttab_free_tail_ptr || + *gnttab_free_tail_ptr == GNTTAB_LIST_END) + return 0; + + return gnttab_last_free - *gnttab_free_tail_ptr + 1; +} + +/* Rebuilds the free grant list and tries to find count consecutive entries. */ +static int get_free_seq(unsigned int count) +{ + int ret = -ENOSPC; + unsigned int from, to; + grant_ref_t *last; + + gnttab_free_tail_ptr = &gnttab_free_head; + last = &gnttab_free_head; + + for (from = find_first_bit(gnttab_free_bitmap, gnttab_size); + from < gnttab_size; + from = find_next_bit(gnttab_free_bitmap, gnttab_size, to + 1)) { + to = find_next_zero_bit(gnttab_free_bitmap, gnttab_size, + from + 1); + if (ret < 0 && to - from >= count) { + ret = from; + bitmap_clear(gnttab_free_bitmap, ret, count); + from += count; + gnttab_free_count -= count; + if (from == to) + continue; + } + + /* + * Recreate the free list in order to have it properly sorted. + * This is needed to make sure that the free tail has the maximum + * possible size. + */ + while (from < to) { + *last = from; + last = __gnttab_entry(from); + gnttab_last_free = from; + from++; + } + if (to < gnttab_size) + gnttab_free_tail_ptr = __gnttab_entry(to - 1); + } + + *last = GNTTAB_LIST_END; + if (gnttab_last_free != gnttab_size - 1) + gnttab_free_tail_ptr = NULL; + + return ret; +} + +static int get_free_entries_seq(unsigned int count) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&gnttab_list_lock, flags); + + if (gnttab_free_count < count) { + ret = gnttab_expand(count - gnttab_free_count); + if (ret < 0) + goto out; + } + + if (get_seq_entry_count() < count) { + ret = get_free_seq(count); + if (ret >= 0) + goto out; + ret = gnttab_expand(count - get_seq_entry_count()); + if (ret < 0) + goto out; + } + + ret = *gnttab_free_tail_ptr; + *gnttab_free_tail_ptr = gnttab_entry(ret + count - 1); + gnttab_free_count -= count; + if (!gnttab_free_count) + gnttab_free_tail_ptr = NULL; + bitmap_clear(gnttab_free_bitmap, ret, count); + + out: + spin_unlock_irqrestore(&gnttab_list_lock, flags); + + return ret; +} + static void do_free_callbacks(void) { struct gnttab_free_callback *callback, *next; @@ -204,21 +328,51 @@ static inline void check_free_callbacks(void) do_free_callbacks(); } +static void put_free_entry_locked(grant_ref_t ref) +{ + if (unlikely(ref < GNTTAB_NR_RESERVED_ENTRIES)) + return; + + gnttab_entry(ref) = gnttab_free_head; + gnttab_free_head = ref; + if (!gnttab_free_count) + gnttab_last_free = ref; + if (gnttab_free_tail_ptr == &gnttab_free_head) + gnttab_free_tail_ptr = __gnttab_entry(ref); + gnttab_free_count++; + bitmap_set(gnttab_free_bitmap, ref, 1); +} + static void put_free_entry(grant_ref_t ref) { unsigned long flags; - if (unlikely(ref < GNTTAB_NR_RESERVED_ENTRIES)) - return; - spin_lock_irqsave(&gnttab_list_lock, flags); - gnttab_entry(ref) = gnttab_free_head; - gnttab_free_head = ref; - gnttab_free_count++; + put_free_entry_locked(ref); check_free_callbacks(); spin_unlock_irqrestore(&gnttab_list_lock, flags); } +static void gnttab_set_free(unsigned int start, unsigned int n) +{ + unsigned int i; + + for (i = start; i < start + n - 1; i++) + gnttab_entry(i) = i + 1; + + gnttab_entry(i) = GNTTAB_LIST_END; + if (!gnttab_free_count) { + gnttab_free_head = start; + gnttab_free_tail_ptr = &gnttab_free_head; + } else { + gnttab_entry(gnttab_last_free) = start; + } + gnttab_free_count += n; + gnttab_last_free = i; + + bitmap_set(gnttab_free_bitmap, start, n); +} + /* * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2. * Introducing a valid entry into the grant table: @@ -450,23 +604,31 @@ void gnttab_free_grant_references(grant_ref_t head) { grant_ref_t ref; unsigned long flags; - int count = 1; - if (head == GNTTAB_LIST_END) - return; + spin_lock_irqsave(&gnttab_list_lock, flags); - ref = head; - while (gnttab_entry(ref) != GNTTAB_LIST_END) { - ref = gnttab_entry(ref); - count++; + while (head != GNTTAB_LIST_END) { + ref = gnttab_entry(head); + put_free_entry_locked(head); + head = ref; } - gnttab_entry(ref) = gnttab_free_head; - gnttab_free_head = head; - gnttab_free_count += count; check_free_callbacks(); spin_unlock_irqrestore(&gnttab_list_lock, flags); } EXPORT_SYMBOL_GPL(gnttab_free_grant_references); +void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count) +{ + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&gnttab_list_lock, flags); + for (i = count; i > 0; i--) + put_free_entry_locked(head + i - 1); + check_free_callbacks(); + spin_unlock_irqrestore(&gnttab_list_lock, flags); +} +EXPORT_SYMBOL_GPL(gnttab_free_grant_reference_seq); + int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) { int h = get_free_entries(count); @@ -480,6 +642,24 @@ int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) } EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); +int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first) +{ + int h; + + if (count == 1) + h = get_free_entries(1); + else + h = get_free_entries_seq(count); + + if (h < 0) + return -ENOSPC; + + *first = h; + + return 0; +} +EXPORT_SYMBOL_GPL(gnttab_alloc_grant_reference_seq); + int gnttab_empty_grant_references(const grant_ref_t *private_head) { return (*private_head == GNTTAB_LIST_END); @@ -572,16 +752,13 @@ static int grow_gnttab_list(unsigned int more_frames) goto grow_nomem; } + gnttab_set_free(gnttab_size, extra_entries); - for (i = grefs_per_frame * nr_grant_frames; - i < grefs_per_frame * new_nr_grant_frames - 1; i++) - gnttab_entry(i) = i + 1; - - gnttab_entry(i) = gnttab_free_head; - gnttab_free_head = grefs_per_frame * nr_grant_frames; - gnttab_free_count += extra_entries; + if (!gnttab_free_tail_ptr) + gnttab_free_tail_ptr = __gnttab_entry(gnttab_size); nr_grant_frames = new_nr_grant_frames; + gnttab_size += extra_entries; check_free_callbacks(); @@ -1424,20 +1601,20 @@ static int gnttab_expand(unsigned int req_entries) int gnttab_init(void) { int i; - unsigned long max_nr_grant_frames; + unsigned long max_nr_grant_frames, max_nr_grefs; unsigned int max_nr_glist_frames, nr_glist_frames; - unsigned int nr_init_grefs; int ret; gnttab_request_version(); max_nr_grant_frames = gnttab_max_grant_frames(); + max_nr_grefs = max_nr_grant_frames * + gnttab_interface->grefs_per_grant_frame; nr_grant_frames = 1; /* Determine the maximum number of frames required for the * grant reference free list on the current hypervisor. */ - max_nr_glist_frames = (max_nr_grant_frames * - gnttab_interface->grefs_per_grant_frame / RPP); + max_nr_glist_frames = max_nr_grefs / RPP; gnttab_list = kmalloc_array(max_nr_glist_frames, sizeof(grant_ref_t *), @@ -1454,6 +1631,12 @@ int gnttab_init(void) } } + gnttab_free_bitmap = bitmap_zalloc(max_nr_grefs, GFP_KERNEL); + if (!gnttab_free_bitmap) { + ret = -ENOMEM; + goto ini_nomem; + } + ret = arch_gnttab_init(max_nr_grant_frames, nr_status_frames(max_nr_grant_frames)); if (ret < 0) @@ -1464,15 +1647,10 @@ int gnttab_init(void) goto ini_nomem; } - nr_init_grefs = nr_grant_frames * - gnttab_interface->grefs_per_grant_frame; + gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame; - for (i = GNTTAB_NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) - gnttab_entry(i) = i + 1; - - gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; - gnttab_free_count = nr_init_grefs - GNTTAB_NR_RESERVED_ENTRIES; - gnttab_free_head = GNTTAB_NR_RESERVED_ENTRIES; + gnttab_set_free(GNTTAB_NR_RESERVED_ENTRIES, + gnttab_size - GNTTAB_NR_RESERVED_ENTRIES); printk("Grant table initialized\n"); return 0; @@ -1481,6 +1659,7 @@ int gnttab_init(void) for (i--; i >= 0; i--) free_page((unsigned long)gnttab_list[i]); kfree(gnttab_list); + bitmap_free(gnttab_free_bitmap); return ret; } EXPORT_SYMBOL_GPL(gnttab_init); diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h index 527c9907f99c..e279be353e3f 100644 --- a/include/xen/grant_table.h +++ b/include/xen/grant_table.h @@ -127,10 +127,14 @@ int gnttab_try_end_foreign_access(grant_ref_t ref); */ int gnttab_alloc_grant_references(u16 count, grant_ref_t *pprivate_head); +int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first); + void gnttab_free_grant_reference(grant_ref_t ref); void gnttab_free_grant_references(grant_ref_t head); +void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count); + int gnttab_empty_grant_references(const grant_ref_t *pprivate_head); int gnttab_claim_grant_reference(grant_ref_t *pprivate_head); From d6aca3504c7ded5f4f46957e3685b9344d9743dd Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 2 Jun 2022 22:23:48 +0300 Subject: [PATCH 05/11] xen/grant-dma-ops: Add option to restrict memory access under Xen Introduce Xen grant DMA-mapping layer which contains special DMA-mapping routines for providing grant references as DMA addresses to be used by frontends (e.g. virtio) in Xen guests. Add the needed functionality by providing a special set of DMA ops handling the needed grant operations for the I/O pages. The subsequent commit will introduce the use case for xen-grant DMA ops layer to enable using virtio devices in Xen guests in a safe manner. Signed-off-by: Juergen Gross Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Link: https://lore.kernel.org/r/1654197833-25362-4-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/Kconfig | 4 + drivers/xen/Makefile | 1 + drivers/xen/grant-dma-ops.c | 312 ++++++++++++++++++++++++++++++++++++ include/xen/xen-ops.h | 8 + 4 files changed, 325 insertions(+) create mode 100644 drivers/xen/grant-dma-ops.c diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 120d32f164ac..313a9127fd5c 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -335,4 +335,8 @@ config XEN_UNPOPULATED_ALLOC having to balloon out RAM regions in order to obtain physical memory space to create such mappings. +config XEN_GRANT_DMA_OPS + bool + select DMA_OPS + endmenu diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile index 5aae66e638a7..1a23cb0dc188 100644 --- a/drivers/xen/Makefile +++ b/drivers/xen/Makefile @@ -39,3 +39,4 @@ xen-gntalloc-y := gntalloc.o xen-privcmd-y := privcmd.o privcmd-buf.o obj-$(CONFIG_XEN_FRONT_PGDIR_SHBUF) += xen-front-pgdir-shbuf.o obj-$(CONFIG_XEN_UNPOPULATED_ALLOC) += unpopulated-alloc.o +obj-$(CONFIG_XEN_GRANT_DMA_OPS) += grant-dma-ops.o diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c new file mode 100644 index 000000000000..aaff35cda517 --- /dev/null +++ b/drivers/xen/grant-dma-ops.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Xen grant DMA-mapping layer - contains special DMA-mapping routines + * for providing grant references as DMA addresses to be used by frontends + * (e.g. virtio) in Xen guests + * + * Copyright (c) 2021, Juergen Gross + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct xen_grant_dma_data { + /* The ID of backend domain */ + domid_t backend_domid; + /* Is device behaving sane? */ + bool broken; +}; + +static DEFINE_XARRAY(xen_grant_dma_devices); + +#define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63) + +static inline dma_addr_t grant_to_dma(grant_ref_t grant) +{ + return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << PAGE_SHIFT); +} + +static inline grant_ref_t dma_to_grant(dma_addr_t dma) +{ + return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> PAGE_SHIFT); +} + +static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev) +{ + struct xen_grant_dma_data *data; + + xa_lock(&xen_grant_dma_devices); + data = xa_load(&xen_grant_dma_devices, (unsigned long)dev); + xa_unlock(&xen_grant_dma_devices); + + return data; +} + +/* + * DMA ops for Xen frontends (e.g. virtio). + * + * Used to act as a kind of software IOMMU for Xen guests by using grants as + * DMA addresses. + * Such a DMA address is formed by using the grant reference as a frame + * number and setting the highest address bit (this bit is for the backend + * to be able to distinguish it from e.g. a mmio address). + * + * Note that for now we hard wire dom0 to be the backend domain. In order + * to support any domain as backend we'd need to add a way to communicate + * the domid of this backend, e.g. via Xenstore, via the PCI-device's + * config space or DT/ACPI. + */ +static void *xen_grant_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, + unsigned long attrs) +{ + struct xen_grant_dma_data *data; + unsigned int i, n_pages = PFN_UP(size); + unsigned long pfn; + grant_ref_t grant; + void *ret; + + data = find_xen_grant_dma_data(dev); + if (!data) + return NULL; + + if (unlikely(data->broken)) + return NULL; + + ret = alloc_pages_exact(n_pages * PAGE_SIZE, gfp); + if (!ret) + return NULL; + + pfn = virt_to_pfn(ret); + + if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) { + free_pages_exact(ret, n_pages * PAGE_SIZE); + return NULL; + } + + for (i = 0; i < n_pages; i++) { + gnttab_grant_foreign_access_ref(grant + i, data->backend_domid, + pfn_to_gfn(pfn + i), 0); + } + + *dma_handle = grant_to_dma(grant); + + return ret; +} + +static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_handle, unsigned long attrs) +{ + struct xen_grant_dma_data *data; + unsigned int i, n_pages = PFN_UP(size); + grant_ref_t grant; + + data = find_xen_grant_dma_data(dev); + if (!data) + return; + + if (unlikely(data->broken)) + return; + + grant = dma_to_grant(dma_handle); + + for (i = 0; i < n_pages; i++) { + if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { + dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n"); + data->broken = true; + return; + } + } + + gnttab_free_grant_reference_seq(grant, n_pages); + + free_pages_exact(vaddr, n_pages * PAGE_SIZE); +} + +static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size, + dma_addr_t *dma_handle, + enum dma_data_direction dir, + gfp_t gfp) +{ + void *vaddr; + + vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0); + if (!vaddr) + return NULL; + + return virt_to_page(vaddr); +} + +static void xen_grant_dma_free_pages(struct device *dev, size_t size, + struct page *vaddr, dma_addr_t dma_handle, + enum dma_data_direction dir) +{ + xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0); +} + +static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct xen_grant_dma_data *data; + unsigned int i, n_pages = PFN_UP(size); + grant_ref_t grant; + dma_addr_t dma_handle; + + if (WARN_ON(dir == DMA_NONE)) + return DMA_MAPPING_ERROR; + + data = find_xen_grant_dma_data(dev); + if (!data) + return DMA_MAPPING_ERROR; + + if (unlikely(data->broken)) + return DMA_MAPPING_ERROR; + + if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) + return DMA_MAPPING_ERROR; + + for (i = 0; i < n_pages; i++) { + gnttab_grant_foreign_access_ref(grant + i, data->backend_domid, + xen_page_to_gfn(page) + i, dir == DMA_TO_DEVICE); + } + + dma_handle = grant_to_dma(grant) + offset; + + return dma_handle; +} + +static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct xen_grant_dma_data *data; + unsigned int i, n_pages = PFN_UP(size); + grant_ref_t grant; + + if (WARN_ON(dir == DMA_NONE)) + return; + + data = find_xen_grant_dma_data(dev); + if (!data) + return; + + if (unlikely(data->broken)) + return; + + grant = dma_to_grant(dma_handle); + + for (i = 0; i < n_pages; i++) { + if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) { + dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n"); + data->broken = true; + return; + } + } + + gnttab_free_grant_reference_seq(grant, n_pages); +} + +static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + unsigned long attrs) +{ + struct scatterlist *s; + unsigned int i; + + if (WARN_ON(dir == DMA_NONE)) + return; + + for_each_sg(sg, s, nents, i) + xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir, + attrs); +} + +static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + unsigned long attrs) +{ + struct scatterlist *s; + unsigned int i; + + if (WARN_ON(dir == DMA_NONE)) + return -EINVAL; + + for_each_sg(sg, s, nents, i) { + s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset, + s->length, dir, attrs); + if (s->dma_address == DMA_MAPPING_ERROR) + goto out; + + sg_dma_len(s) = s->length; + } + + return nents; + +out: + xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); + sg_dma_len(sg) = 0; + + return -EIO; +} + +static int xen_grant_dma_supported(struct device *dev, u64 mask) +{ + return mask == DMA_BIT_MASK(64); +} + +static const struct dma_map_ops xen_grant_dma_ops = { + .alloc = xen_grant_dma_alloc, + .free = xen_grant_dma_free, + .alloc_pages = xen_grant_dma_alloc_pages, + .free_pages = xen_grant_dma_free_pages, + .mmap = dma_common_mmap, + .get_sgtable = dma_common_get_sgtable, + .map_page = xen_grant_dma_map_page, + .unmap_page = xen_grant_dma_unmap_page, + .map_sg = xen_grant_dma_map_sg, + .unmap_sg = xen_grant_dma_unmap_sg, + .dma_supported = xen_grant_dma_supported, +}; + +void xen_grant_setup_dma_ops(struct device *dev) +{ + struct xen_grant_dma_data *data; + + data = find_xen_grant_dma_data(dev); + if (data) { + dev_err(dev, "Xen grant DMA data is already created\n"); + return; + } + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + goto err; + + /* XXX The dom0 is hardcoded as the backend domain for now */ + data->backend_domid = 0; + + if (xa_err(xa_store(&xen_grant_dma_devices, (unsigned long)dev, data, + GFP_KERNEL))) { + dev_err(dev, "Cannot store Xen grant DMA data\n"); + goto err; + } + + dev->dma_ops = &xen_grant_dma_ops; + + return; + +err: + dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n"); +} + +MODULE_DESCRIPTION("Xen grant DMA-mapping layer"); +MODULE_AUTHOR("Juergen Gross "); +MODULE_LICENSE("GPL"); diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index c7c1b46ff4cd..afd586d717a4 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -214,4 +214,12 @@ static inline void xen_preemptible_hcall_end(void) { } #endif /* CONFIG_XEN_PV && !CONFIG_PREEMPTION */ +#ifdef CONFIG_XEN_GRANT_DMA_OPS +void xen_grant_setup_dma_ops(struct device *dev); +#else +static inline void xen_grant_setup_dma_ops(struct device *dev) +{ +} +#endif /* CONFIG_XEN_GRANT_DMA_OPS */ + #endif /* INCLUDE_XEN_OPS_H */ From fa1f57421e0b1c57843902c89728f823abc32f02 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 2 Jun 2022 22:23:49 +0300 Subject: [PATCH 06/11] xen/virtio: Enable restricted memory access using Xen grant mappings In order to support virtio in Xen guests add a config option XEN_VIRTIO enabling the user to specify whether in all Xen guests virtio should be able to access memory via Xen grant mappings only on the host side. Also set PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS feature from the guest initialization code on Arm and x86 if CONFIG_XEN_VIRTIO is enabled. Signed-off-by: Juergen Gross Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Link: https://lore.kernel.org/r/1654197833-25362-5-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- arch/arm/xen/enlighten.c | 2 ++ arch/x86/xen/enlighten_hvm.c | 2 ++ arch/x86/xen/enlighten_pv.c | 2 ++ drivers/xen/Kconfig | 11 +++++++++++ include/xen/xen.h | 8 ++++++++ 5 files changed, 25 insertions(+) diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 07eb69f9e7df..1f9c3ba32833 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -443,6 +443,8 @@ static int __init xen_guest_init(void) if (!xen_domain()) return 0; + xen_set_restricted_virtio_memory_access(); + if (!acpi_disabled) xen_acpi_guest_init(); else diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 517a9d8d8f94..8b71b1dd7639 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -195,6 +195,8 @@ static void __init xen_hvm_guest_init(void) if (xen_pv_domain()) return; + xen_set_restricted_virtio_memory_access(); + init_hvm_pv_info(); reserve_shared_info(); diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index f33a4421e7cd..e3297b15701c 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -109,6 +109,8 @@ static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc); static void __init xen_pv_init_platform(void) { + xen_set_restricted_virtio_memory_access(); + populate_extra_pte(fix_to_virt(FIX_PARAVIRT_BOOTMAP)); set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 313a9127fd5c..a7bd8ce7a1d2 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -339,4 +339,15 @@ config XEN_GRANT_DMA_OPS bool select DMA_OPS +config XEN_VIRTIO + bool "Xen virtio support" + depends on VIRTIO + select XEN_GRANT_DMA_OPS + help + Enable virtio support for running as Xen guest. Depending on the + guest type this will require special support on the backend side + (qemu or kernel, depending on the virtio device types used). + + If in doubt, say n. + endmenu diff --git a/include/xen/xen.h b/include/xen/xen.h index a99bab817523..0780a81e140d 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -52,6 +52,14 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, extern u64 xen_saved_max_mem_size; #endif +#include + +static inline void xen_set_restricted_virtio_memory_access(void) +{ + if (IS_ENABLED(CONFIG_XEN_VIRTIO) && xen_domain()) + platform_set(PLATFORM_VIRTIO_RESTRICTED_MEM_ACCESS); +} + #ifdef CONFIG_XEN_UNPOPULATED_ALLOC int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); From add413a1d966d8458571a2571881903d234a9f8e Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Thu, 2 Jun 2022 22:23:50 +0300 Subject: [PATCH 07/11] dt-bindings: Add xen,grant-dma IOMMU description for xen-grant DMA ops The main purpose of this binding is to communicate Xen specific information using generic IOMMU device tree bindings (which is a good fit here) rather than introducing a custom property. Introduce Xen specific IOMMU for the virtualized device (e.g. virtio) to be used by Xen grant DMA-mapping layer in the subsequent commit. The reference to Xen specific IOMMU node using "iommus" property indicates that Xen grant mappings need to be enabled for the device, and it specifies the ID of the domain where the corresponding backend resides. The domid (domain ID) is used as an argument to the Xen grant mapping APIs. This is needed for the option to restrict memory access using Xen grant mappings to work which primary goal is to enable using virtio devices in Xen guests. Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/1654197833-25362-6-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- .../bindings/iommu/xen,grant-dma.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Documentation/devicetree/bindings/iommu/xen,grant-dma.yaml diff --git a/Documentation/devicetree/bindings/iommu/xen,grant-dma.yaml b/Documentation/devicetree/bindings/iommu/xen,grant-dma.yaml new file mode 100644 index 000000000000..be1539d234f9 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/xen,grant-dma.yaml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/xen,grant-dma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xen specific IOMMU for virtualized devices (e.g. virtio) + +maintainers: + - Stefano Stabellini + +description: + The Xen IOMMU represents the Xen grant table interface. Grant mappings + are to be used with devices connected to the Xen IOMMU using the "iommus" + property, which also specifies the ID of the backend domain. + The binding is required to restrict memory access using Xen grant mappings. + +properties: + compatible: + const: xen,grant-dma + + '#iommu-cells': + const: 1 + description: + The single cell is the domid (domain ID) of the domain where the backend + is running. + +required: + - compatible + - "#iommu-cells" + +additionalProperties: false + +examples: + - | + iommu { + compatible = "xen,grant-dma"; + #iommu-cells = <1>; + }; From 1ca55d50e50c74747a7b8846dac306fbe5ac4cf5 Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Thu, 2 Jun 2022 22:23:51 +0300 Subject: [PATCH 08/11] xen/grant-dma-iommu: Introduce stub IOMMU driver In order to reuse generic IOMMU device tree bindings by Xen grant DMA-mapping layer we need to add this stub driver from a fw_devlink perspective (grant-dma-ops cannot be converted into the proper IOMMU driver). Otherwise, just reusing IOMMU bindings (without having a corresponding driver) leads to the deferred probe timeout afterwards, because the IOMMU device never becomes available. This stub driver does nothing except registering empty iommu_ops, the upper layer "of_iommu" will treat this as NO_IOMMU condition and won't return -EPROBE_DEFER. As this driver is quite different from the most hardware IOMMU implementations and only needed in Xen guests, place it in drivers/xen directory. The subsequent commit will make use of it. Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Link: https://lore.kernel.org/r/1654197833-25362-7-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/Kconfig | 4 ++ drivers/xen/Makefile | 1 + drivers/xen/grant-dma-iommu.c | 78 +++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 drivers/xen/grant-dma-iommu.c diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index a7bd8ce7a1d2..35d20d90390b 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -335,6 +335,10 @@ config XEN_UNPOPULATED_ALLOC having to balloon out RAM regions in order to obtain physical memory space to create such mappings. +config XEN_GRANT_DMA_IOMMU + bool + select IOMMU_API + config XEN_GRANT_DMA_OPS bool select DMA_OPS diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile index 1a23cb0dc188..c0503f1c7d5b 100644 --- a/drivers/xen/Makefile +++ b/drivers/xen/Makefile @@ -40,3 +40,4 @@ xen-privcmd-y := privcmd.o privcmd-buf.o obj-$(CONFIG_XEN_FRONT_PGDIR_SHBUF) += xen-front-pgdir-shbuf.o obj-$(CONFIG_XEN_UNPOPULATED_ALLOC) += unpopulated-alloc.o obj-$(CONFIG_XEN_GRANT_DMA_OPS) += grant-dma-ops.o +obj-$(CONFIG_XEN_GRANT_DMA_IOMMU) += grant-dma-iommu.o diff --git a/drivers/xen/grant-dma-iommu.c b/drivers/xen/grant-dma-iommu.c new file mode 100644 index 000000000000..16b8bc0c0b33 --- /dev/null +++ b/drivers/xen/grant-dma-iommu.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Stub IOMMU driver which does nothing. + * The main purpose of it being present is to reuse generic IOMMU device tree + * bindings by Xen grant DMA-mapping layer. + * + * Copyright (C) 2022 EPAM Systems Inc. + */ + +#include +#include +#include + +struct grant_dma_iommu_device { + struct device *dev; + struct iommu_device iommu; +}; + +/* Nothing is really needed here */ +static const struct iommu_ops grant_dma_iommu_ops; + +static const struct of_device_id grant_dma_iommu_of_match[] = { + { .compatible = "xen,grant-dma" }, + { }, +}; + +static int grant_dma_iommu_probe(struct platform_device *pdev) +{ + struct grant_dma_iommu_device *mmu; + int ret; + + mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); + if (!mmu) + return -ENOMEM; + + mmu->dev = &pdev->dev; + + ret = iommu_device_register(&mmu->iommu, &grant_dma_iommu_ops, &pdev->dev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mmu); + + return 0; +} + +static int grant_dma_iommu_remove(struct platform_device *pdev) +{ + struct grant_dma_iommu_device *mmu = platform_get_drvdata(pdev); + + platform_set_drvdata(pdev, NULL); + iommu_device_unregister(&mmu->iommu); + + return 0; +} + +static struct platform_driver grant_dma_iommu_driver = { + .driver = { + .name = "grant-dma-iommu", + .of_match_table = grant_dma_iommu_of_match, + }, + .probe = grant_dma_iommu_probe, + .remove = grant_dma_iommu_remove, +}; + +static int __init grant_dma_iommu_init(void) +{ + struct device_node *iommu_np; + + iommu_np = of_find_matching_node(NULL, grant_dma_iommu_of_match); + if (!iommu_np) + return 0; + + of_node_put(iommu_np); + + return platform_driver_register(&grant_dma_iommu_driver); +} +subsys_initcall(grant_dma_iommu_init); From 625ab90ecdf7770bda7ae21c4d5c938aa9b43bb4 Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Thu, 2 Jun 2022 22:23:52 +0300 Subject: [PATCH 09/11] xen/grant-dma-ops: Retrieve the ID of backend's domain for DT devices Use the presence of "iommus" property pointed to the IOMMU node with recently introduced "xen,grant-dma" compatible as a clear indicator of enabling Xen grant mappings scheme for that device and read the ID of Xen domain where the corresponding backend is running. The domid (domain ID) is used as an argument to the Xen grant mapping APIs. To avoid the deferred probe timeout which takes place after reusing generic IOMMU device tree bindings (because the IOMMU device never becomes available) enable recently introduced stub IOMMU driver by selecting XEN_GRANT_DMA_IOMMU. Also introduce xen_is_grant_dma_device() to check whether xen-grant DMA ops need to be set for a passed device. Remove the hardcoded domid 0 in xen_grant_setup_dma_ops(). Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Link: https://lore.kernel.org/r/1654197833-25362-8-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/Kconfig | 1 + drivers/xen/grant-dma-ops.c | 48 +++++++++++++++++++++++++++++++------ include/xen/xen-ops.h | 5 ++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 35d20d90390b..bfd5f4f706bc 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -347,6 +347,7 @@ config XEN_VIRTIO bool "Xen virtio support" depends on VIRTIO select XEN_GRANT_DMA_OPS + select XEN_GRANT_DMA_IOMMU if OF help Enable virtio support for running as Xen guest. Depending on the guest type this will require special support on the backend side diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index aaff35cda517..fc0142484001 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -56,11 +56,6 @@ static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev) * Such a DMA address is formed by using the grant reference as a frame * number and setting the highest address bit (this bit is for the backend * to be able to distinguish it from e.g. a mmio address). - * - * Note that for now we hard wire dom0 to be the backend domain. In order - * to support any domain as backend we'd need to add a way to communicate - * the domid of this backend, e.g. via Xenstore, via the PCI-device's - * config space or DT/ACPI. */ static void *xen_grant_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, @@ -276,9 +271,26 @@ static const struct dma_map_ops xen_grant_dma_ops = { .dma_supported = xen_grant_dma_supported, }; +bool xen_is_grant_dma_device(struct device *dev) +{ + struct device_node *iommu_np; + bool has_iommu; + + /* XXX Handle only DT devices for now */ + if (!dev->of_node) + return false; + + iommu_np = of_parse_phandle(dev->of_node, "iommus", 0); + has_iommu = iommu_np && of_device_is_compatible(iommu_np, "xen,grant-dma"); + of_node_put(iommu_np); + + return has_iommu; +} + void xen_grant_setup_dma_ops(struct device *dev) { struct xen_grant_dma_data *data; + struct of_phandle_args iommu_spec; data = find_xen_grant_dma_data(dev); if (data) { @@ -286,12 +298,34 @@ void xen_grant_setup_dma_ops(struct device *dev) return; } + /* XXX ACPI device unsupported for now */ + if (!dev->of_node) + goto err; + + if (of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells", + 0, &iommu_spec)) { + dev_err(dev, "Cannot parse iommus property\n"); + goto err; + } + + if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") || + iommu_spec.args_count != 1) { + dev_err(dev, "Incompatible IOMMU node\n"); + of_node_put(iommu_spec.np); + goto err; + } + + of_node_put(iommu_spec.np); + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) goto err; - /* XXX The dom0 is hardcoded as the backend domain for now */ - data->backend_domid = 0; + /* + * The endpoint ID here means the ID of the domain where the corresponding + * backend is running + */ + data->backend_domid = iommu_spec.args[0]; if (xa_err(xa_store(&xen_grant_dma_devices, (unsigned long)dev, data, GFP_KERNEL))) { diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index afd586d717a4..80546960f8b7 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -216,10 +216,15 @@ static inline void xen_preemptible_hcall_end(void) { } #ifdef CONFIG_XEN_GRANT_DMA_OPS void xen_grant_setup_dma_ops(struct device *dev); +bool xen_is_grant_dma_device(struct device *dev); #else static inline void xen_grant_setup_dma_ops(struct device *dev) { } +static inline bool xen_is_grant_dma_device(struct device *dev) +{ + return false; +} #endif /* CONFIG_XEN_GRANT_DMA_OPS */ #endif /* INCLUDE_XEN_OPS_H */ From fea981610c25173e6e5d63ccd4fce49739663ab0 Mon Sep 17 00:00:00 2001 From: Oleksandr Tyshchenko Date: Thu, 2 Jun 2022 22:23:53 +0300 Subject: [PATCH 10/11] arm/xen: Assign xen-grant DMA ops for xen-grant DMA devices By assigning xen-grant DMA ops we will restrict memory access for passed device using Xen grant mappings. This is needed for using any virtualized device (e.g. virtio) in Xen guests in a safe manner. Please note, for the virtio devices the XEN_VIRTIO config should be enabled (it forces ARCH_HAS_RESTRICTED_VIRTIO_MEMORY_ACCESS). Signed-off-by: Oleksandr Tyshchenko Reviewed-by: Stefano Stabellini Link: https://lore.kernel.org/r/1654197833-25362-9-git-send-email-olekstysh@gmail.com Signed-off-by: Juergen Gross --- include/xen/arm/xen-ops.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/xen/arm/xen-ops.h b/include/xen/arm/xen-ops.h index 288deb1c3ac9..b0766a660338 100644 --- a/include/xen/arm/xen-ops.h +++ b/include/xen/arm/xen-ops.h @@ -3,11 +3,14 @@ #define _ASM_ARM_XEN_OPS_H #include +#include static inline void xen_setup_dma_ops(struct device *dev) { #ifdef CONFIG_XEN - if (xen_swiotlb_detect()) + if (xen_is_grant_dma_device(dev)) + xen_grant_setup_dma_ops(dev); + else if (xen_swiotlb_detect()) dev->dma_ops = &xen_swiotlb_dma_ops; #endif } From dbac14a5a05ff8e1ce7c0da0e1f520ce39ec62ea Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 6 Jun 2022 13:59:20 +0900 Subject: [PATCH 11/11] xen: unexport __init-annotated xen_xlate_map_ballooned_pages() EXPORT_SYMBOL and __init is a bad combination because the .init.text section is freed up after the initialization. Hence, modules cannot use symbols annotated __init. The access to a freed symbol may end up with kernel panic. modpost used to detect it, but it has been broken for a decade. Recently, I fixed modpost so it started to warn it again, then this showed up in linux-next builds. There are two ways to fix it: - Remove __init - Remove EXPORT_SYMBOL I chose the latter for this case because none of the in-tree call-sites (arch/arm/xen/enlighten.c, arch/x86/xen/grant-table.c) is compiled as modular. Fixes: 243848fc018c ("xen/grant-table: Move xlated_setup_gnttab_pages to common place") Reported-by: Stephen Rothwell Signed-off-by: Masahiro Yamada Reviewed-by: Oleksandr Tyshchenko Acked-by: Stefano Stabellini Link: https://lore.kernel.org/r/20220606045920.4161881-1-masahiroy@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/xlate_mmu.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/xen/xlate_mmu.c b/drivers/xen/xlate_mmu.c index 34742c6e189e..f17c4c03db30 100644 --- a/drivers/xen/xlate_mmu.c +++ b/drivers/xen/xlate_mmu.c @@ -261,7 +261,6 @@ int __init xen_xlate_map_ballooned_pages(xen_pfn_t **gfns, void **virt, return 0; } -EXPORT_SYMBOL_GPL(xen_xlate_map_ballooned_pages); struct remap_pfn { struct mm_struct *mm;