mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-05 18:39:59 +00:00
KVM/riscv changes for 6.5
- Redirect AMO load/store misaligned traps to KVM guest - Trap-n-emulate AIA in-kernel irqchip for KVM guest - Svnapot support for KVM Guest -----BEGIN PGP SIGNATURE----- iQIzBAABCgAdFiEEZdn75s5e6LHDQ+f/rUjsVaLHLAcFAmSUUnoACgkQrUjsVaLH LAchmw//aKApCr6hOUehE0WCrG2QJedco0nghwwBlRLM2FSzACKsinRDcq3ey/jx OsJ03LpOM9I3ucpy7SBirqJq06dxWtFITVtWGKA3CKVncH9rPdsSmiDc7LbiDUt9 HxAZa9katL1c/DyFWiBKa520FZ4qVySJ0i+gCHF6DhrDdySbUqJTgqqMzRSpz8Ei RGDtn/156X5gtCuAs4CMIT1N/r1oyhd1s9y9/Cmy7PYMJcqpBZQu9n+5c9ss/Xz4 CJ+kiMBZIH61oeUwVSp/sz3aWc50OkX8zcoGtbsEcZvqVrdPl98aaJliVb6MiUXT PGxQqRxgFGHOlOpzpdBdiUVWT0NJCZGVXNS3/zHh37fccHVE0rJUClJAtz9PzCKq XfUSlx2tx99tnbO8zyqqT8aRwfV0jKcYXTcy4spBj6tEelubhXkuirck60Ny1Ywc ced6mXAfpHe/Dl6vn91OwE4Xk+EDXBpE+8YZVIms6lc3URzub8Y7dIrB+XZzHSKj JRV4mtsLtLDX6oLKvqJVZu+OIY5n3h9KWbuA4+LknQOVWCac24+kyI2sSGLOUu/f B806qzhmZeNRaM2IRsmUT6h6uaWcdCfnd06rrKsAhyc9ZlCpFvE1NYkAwM7rSvWb L6WiSv+1T8W1b6LP2WUDzfZNe79Vv5ZMu0V4VBPonuF2mvi1r6g= =nK7C -----END PGP SIGNATURE----- Merge tag 'kvm-riscv-6.5-1' of https://github.com/kvm-riscv/linux into HEAD KVM/riscv changes for 6.5 - Redirect AMO load/store misaligned traps to KVM guest - Trap-n-emulate AIA in-kernel irqchip for KVM guest - Svnapot support for KVM Guest
This commit is contained in:
commit
b5396271ea
20 changed files with 3100 additions and 61 deletions
|
@ -82,7 +82,9 @@
|
|||
#define EXC_INST_ACCESS 1
|
||||
#define EXC_INST_ILLEGAL 2
|
||||
#define EXC_BREAKPOINT 3
|
||||
#define EXC_LOAD_MISALIGNED 4
|
||||
#define EXC_LOAD_ACCESS 5
|
||||
#define EXC_STORE_MISALIGNED 6
|
||||
#define EXC_STORE_ACCESS 7
|
||||
#define EXC_SYSCALL 8
|
||||
#define EXC_HYPERVISOR_SYSCALL 9
|
||||
|
|
|
@ -20,6 +20,33 @@ struct kvm_aia {
|
|||
|
||||
/* In-kernel irqchip initialized */
|
||||
bool initialized;
|
||||
|
||||
/* Virtualization mode (Emulation, HW Accelerated, or Auto) */
|
||||
u32 mode;
|
||||
|
||||
/* Number of MSIs */
|
||||
u32 nr_ids;
|
||||
|
||||
/* Number of wired IRQs */
|
||||
u32 nr_sources;
|
||||
|
||||
/* Number of group bits in IMSIC address */
|
||||
u32 nr_group_bits;
|
||||
|
||||
/* Position of group bits in IMSIC address */
|
||||
u32 nr_group_shift;
|
||||
|
||||
/* Number of hart bits in IMSIC address */
|
||||
u32 nr_hart_bits;
|
||||
|
||||
/* Number of guest bits in IMSIC address */
|
||||
u32 nr_guest_bits;
|
||||
|
||||
/* Guest physical address of APLIC */
|
||||
gpa_t aplic_addr;
|
||||
|
||||
/* Internal state of APLIC */
|
||||
void *aplic_state;
|
||||
};
|
||||
|
||||
struct kvm_vcpu_aia_csr {
|
||||
|
@ -38,25 +65,53 @@ struct kvm_vcpu_aia {
|
|||
|
||||
/* CPU AIA CSR context upon Guest VCPU reset */
|
||||
struct kvm_vcpu_aia_csr guest_reset_csr;
|
||||
|
||||
/* Guest physical address of IMSIC for this VCPU */
|
||||
gpa_t imsic_addr;
|
||||
|
||||
/* HART index of IMSIC extacted from guest physical address */
|
||||
u32 hart_index;
|
||||
|
||||
/* Internal state of IMSIC for this VCPU */
|
||||
void *imsic_state;
|
||||
};
|
||||
|
||||
#define KVM_RISCV_AIA_UNDEF_ADDR (-1)
|
||||
|
||||
#define kvm_riscv_aia_initialized(k) ((k)->arch.aia.initialized)
|
||||
|
||||
#define irqchip_in_kernel(k) ((k)->arch.aia.in_kernel)
|
||||
|
||||
extern unsigned int kvm_riscv_aia_nr_hgei;
|
||||
extern unsigned int kvm_riscv_aia_max_ids;
|
||||
DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
|
||||
#define kvm_riscv_aia_available() \
|
||||
static_branch_unlikely(&kvm_riscv_aia_available)
|
||||
|
||||
extern struct kvm_device_ops kvm_riscv_aia_device_ops;
|
||||
|
||||
void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu);
|
||||
int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu);
|
||||
|
||||
#define KVM_RISCV_AIA_IMSIC_TOPEI (ISELECT_MASK + 1)
|
||||
static inline int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu,
|
||||
unsigned long isel,
|
||||
unsigned long *val,
|
||||
unsigned long new_val,
|
||||
unsigned long wr_mask)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel,
|
||||
unsigned long *val, unsigned long new_val,
|
||||
unsigned long wr_mask);
|
||||
int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
|
||||
bool write, unsigned long *val);
|
||||
int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type);
|
||||
void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu);
|
||||
int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu,
|
||||
u32 guest_index, u32 offset, u32 iid);
|
||||
int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu);
|
||||
void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu);
|
||||
|
||||
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v);
|
||||
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v);
|
||||
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type);
|
||||
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level);
|
||||
int kvm_riscv_aia_aplic_init(struct kvm *kvm);
|
||||
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm);
|
||||
|
||||
#ifdef CONFIG_32BIT
|
||||
void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu);
|
||||
|
@ -93,31 +148,23 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
|
|||
{ .base = CSR_SIREG, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_ireg }, \
|
||||
{ .base = CSR_STOPEI, .count = 1, .func = kvm_riscv_vcpu_aia_rmw_topei },
|
||||
|
||||
static inline int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu);
|
||||
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu);
|
||||
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu);
|
||||
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu);
|
||||
|
||||
static inline void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
}
|
||||
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
|
||||
u32 guest_index, u32 iid);
|
||||
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
|
||||
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level);
|
||||
|
||||
static inline int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
void kvm_riscv_aia_init_vm(struct kvm *kvm);
|
||||
void kvm_riscv_aia_destroy_vm(struct kvm *kvm);
|
||||
|
||||
static inline void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void kvm_riscv_aia_init_vm(struct kvm *kvm)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
|
||||
{
|
||||
}
|
||||
int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
|
||||
void __iomem **hgei_va, phys_addr_t *hgei_pa);
|
||||
void kvm_riscv_aia_free_hgei(int cpu, int hgei);
|
||||
void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable);
|
||||
|
||||
void kvm_riscv_aia_enable(void);
|
||||
void kvm_riscv_aia_disable(void);
|
||||
|
|
58
arch/riscv/include/asm/kvm_aia_aplic.h
Normal file
58
arch/riscv/include/asm/kvm_aia_aplic.h
Normal file
|
@ -0,0 +1,58 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
|
||||
* Copyright (C) 2022 Ventana Micro Systems Inc.
|
||||
*/
|
||||
#ifndef __KVM_RISCV_AIA_IMSIC_H
|
||||
#define __KVM_RISCV_AIA_IMSIC_H
|
||||
|
||||
#include <linux/bitops.h>
|
||||
|
||||
#define APLIC_MAX_IDC BIT(14)
|
||||
#define APLIC_MAX_SOURCE 1024
|
||||
|
||||
#define APLIC_DOMAINCFG 0x0000
|
||||
#define APLIC_DOMAINCFG_RDONLY 0x80000000
|
||||
#define APLIC_DOMAINCFG_IE BIT(8)
|
||||
#define APLIC_DOMAINCFG_DM BIT(2)
|
||||
#define APLIC_DOMAINCFG_BE BIT(0)
|
||||
|
||||
#define APLIC_SOURCECFG_BASE 0x0004
|
||||
#define APLIC_SOURCECFG_D BIT(10)
|
||||
#define APLIC_SOURCECFG_CHILDIDX_MASK 0x000003ff
|
||||
#define APLIC_SOURCECFG_SM_MASK 0x00000007
|
||||
#define APLIC_SOURCECFG_SM_INACTIVE 0x0
|
||||
#define APLIC_SOURCECFG_SM_DETACH 0x1
|
||||
#define APLIC_SOURCECFG_SM_EDGE_RISE 0x4
|
||||
#define APLIC_SOURCECFG_SM_EDGE_FALL 0x5
|
||||
#define APLIC_SOURCECFG_SM_LEVEL_HIGH 0x6
|
||||
#define APLIC_SOURCECFG_SM_LEVEL_LOW 0x7
|
||||
|
||||
#define APLIC_IRQBITS_PER_REG 32
|
||||
|
||||
#define APLIC_SETIP_BASE 0x1c00
|
||||
#define APLIC_SETIPNUM 0x1cdc
|
||||
|
||||
#define APLIC_CLRIP_BASE 0x1d00
|
||||
#define APLIC_CLRIPNUM 0x1ddc
|
||||
|
||||
#define APLIC_SETIE_BASE 0x1e00
|
||||
#define APLIC_SETIENUM 0x1edc
|
||||
|
||||
#define APLIC_CLRIE_BASE 0x1f00
|
||||
#define APLIC_CLRIENUM 0x1fdc
|
||||
|
||||
#define APLIC_SETIPNUM_LE 0x2000
|
||||
#define APLIC_SETIPNUM_BE 0x2004
|
||||
|
||||
#define APLIC_GENMSI 0x3000
|
||||
|
||||
#define APLIC_TARGET_BASE 0x3004
|
||||
#define APLIC_TARGET_HART_IDX_SHIFT 18
|
||||
#define APLIC_TARGET_HART_IDX_MASK 0x3fff
|
||||
#define APLIC_TARGET_GUEST_IDX_SHIFT 12
|
||||
#define APLIC_TARGET_GUEST_IDX_MASK 0x3f
|
||||
#define APLIC_TARGET_IPRIO_MASK 0xff
|
||||
#define APLIC_TARGET_EIID_MASK 0x7ff
|
||||
|
||||
#endif
|
38
arch/riscv/include/asm/kvm_aia_imsic.h
Normal file
38
arch/riscv/include/asm/kvm_aia_imsic.h
Normal file
|
@ -0,0 +1,38 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
|
||||
* Copyright (C) 2022 Ventana Micro Systems Inc.
|
||||
*/
|
||||
#ifndef __KVM_RISCV_AIA_IMSIC_H
|
||||
#define __KVM_RISCV_AIA_IMSIC_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <asm/csr.h>
|
||||
|
||||
#define IMSIC_MMIO_PAGE_SHIFT 12
|
||||
#define IMSIC_MMIO_PAGE_SZ (1UL << IMSIC_MMIO_PAGE_SHIFT)
|
||||
#define IMSIC_MMIO_PAGE_LE 0x00
|
||||
#define IMSIC_MMIO_PAGE_BE 0x04
|
||||
|
||||
#define IMSIC_MIN_ID 63
|
||||
#define IMSIC_MAX_ID 2048
|
||||
|
||||
#define IMSIC_EIDELIVERY 0x70
|
||||
|
||||
#define IMSIC_EITHRESHOLD 0x72
|
||||
|
||||
#define IMSIC_EIP0 0x80
|
||||
#define IMSIC_EIP63 0xbf
|
||||
#define IMSIC_EIPx_BITS 32
|
||||
|
||||
#define IMSIC_EIE0 0xc0
|
||||
#define IMSIC_EIE63 0xff
|
||||
#define IMSIC_EIEx_BITS 32
|
||||
|
||||
#define IMSIC_FIRST IMSIC_EIDELIVERY
|
||||
#define IMSIC_LAST IMSIC_EIE63
|
||||
|
||||
#define IMSIC_MMIO_SETIPNUM_LE 0x00
|
||||
#define IMSIC_MMIO_SETIPNUM_BE 0x04
|
||||
|
||||
#endif
|
|
@ -27,6 +27,8 @@
|
|||
|
||||
#define KVM_VCPU_MAX_FEATURES 0
|
||||
|
||||
#define KVM_IRQCHIP_NUM_PINS 1024
|
||||
|
||||
#define KVM_REQ_SLEEP \
|
||||
KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
|
||||
#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(1)
|
||||
|
@ -318,6 +320,8 @@ int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
|
|||
bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
|
||||
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
|
||||
|
||||
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines);
|
||||
|
||||
void __kvm_riscv_unpriv_trap(void);
|
||||
|
||||
unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu,
|
||||
|
|
|
@ -14,9 +14,15 @@
|
|||
#define KVM_SBI_VERSION_MAJOR 1
|
||||
#define KVM_SBI_VERSION_MINOR 0
|
||||
|
||||
enum kvm_riscv_sbi_ext_status {
|
||||
KVM_RISCV_SBI_EXT_UNINITIALIZED,
|
||||
KVM_RISCV_SBI_EXT_AVAILABLE,
|
||||
KVM_RISCV_SBI_EXT_UNAVAILABLE,
|
||||
};
|
||||
|
||||
struct kvm_vcpu_sbi_context {
|
||||
int return_handled;
|
||||
bool extension_disabled[KVM_RISCV_SBI_EXT_MAX];
|
||||
enum kvm_riscv_sbi_ext_status ext_status[KVM_RISCV_SBI_EXT_MAX];
|
||||
};
|
||||
|
||||
struct kvm_vcpu_sbi_return {
|
||||
|
@ -66,4 +72,7 @@ extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm;
|
|||
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental;
|
||||
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor;
|
||||
|
||||
#ifdef CONFIG_RISCV_PMU_SBI
|
||||
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu;
|
||||
#endif
|
||||
#endif /* __RISCV_KVM_VCPU_SBI_H__ */
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
#include <asm/bitsperlong.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
#define __KVM_HAVE_IRQ_LINE
|
||||
#define __KVM_HAVE_READONLY_MEM
|
||||
|
||||
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
|
||||
|
@ -121,6 +122,7 @@ enum KVM_RISCV_ISA_EXT_ID {
|
|||
KVM_RISCV_ISA_EXT_ZICBOZ,
|
||||
KVM_RISCV_ISA_EXT_ZBB,
|
||||
KVM_RISCV_ISA_EXT_SSAIA,
|
||||
KVM_RISCV_ISA_EXT_SVNAPOT,
|
||||
KVM_RISCV_ISA_EXT_MAX,
|
||||
};
|
||||
|
||||
|
@ -203,6 +205,77 @@ enum KVM_RISCV_SBI_EXT_ID {
|
|||
#define KVM_REG_RISCV_SBI_MULTI_REG_LAST \
|
||||
KVM_REG_RISCV_SBI_MULTI_REG(KVM_RISCV_SBI_EXT_MAX - 1)
|
||||
|
||||
/* Device Control API: RISC-V AIA */
|
||||
#define KVM_DEV_RISCV_APLIC_ALIGN 0x1000
|
||||
#define KVM_DEV_RISCV_APLIC_SIZE 0x4000
|
||||
#define KVM_DEV_RISCV_APLIC_MAX_HARTS 0x4000
|
||||
#define KVM_DEV_RISCV_IMSIC_ALIGN 0x1000
|
||||
#define KVM_DEV_RISCV_IMSIC_SIZE 0x1000
|
||||
|
||||
#define KVM_DEV_RISCV_AIA_GRP_CONFIG 0
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_MODE 0
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_IDS 1
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_SRCS 2
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS 3
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT 4
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_HART_BITS 5
|
||||
#define KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS 6
|
||||
|
||||
/*
|
||||
* Modes of RISC-V AIA device:
|
||||
* 1) EMUL (aka Emulation): Trap-n-emulate IMSIC
|
||||
* 2) HWACCEL (aka HW Acceleration): Virtualize IMSIC using IMSIC guest files
|
||||
* 3) AUTO (aka Automatic): Virtualize IMSIC using IMSIC guest files whenever
|
||||
* available otherwise fallback to trap-n-emulation
|
||||
*/
|
||||
#define KVM_DEV_RISCV_AIA_MODE_EMUL 0
|
||||
#define KVM_DEV_RISCV_AIA_MODE_HWACCEL 1
|
||||
#define KVM_DEV_RISCV_AIA_MODE_AUTO 2
|
||||
|
||||
#define KVM_DEV_RISCV_AIA_IDS_MIN 63
|
||||
#define KVM_DEV_RISCV_AIA_IDS_MAX 2048
|
||||
#define KVM_DEV_RISCV_AIA_SRCS_MAX 1024
|
||||
#define KVM_DEV_RISCV_AIA_GROUP_BITS_MAX 8
|
||||
#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN 24
|
||||
#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX 56
|
||||
#define KVM_DEV_RISCV_AIA_HART_BITS_MAX 16
|
||||
#define KVM_DEV_RISCV_AIA_GUEST_BITS_MAX 8
|
||||
|
||||
#define KVM_DEV_RISCV_AIA_GRP_ADDR 1
|
||||
#define KVM_DEV_RISCV_AIA_ADDR_APLIC 0
|
||||
#define KVM_DEV_RISCV_AIA_ADDR_IMSIC(__vcpu) (1 + (__vcpu))
|
||||
#define KVM_DEV_RISCV_AIA_ADDR_MAX \
|
||||
(1 + KVM_DEV_RISCV_APLIC_MAX_HARTS)
|
||||
|
||||
#define KVM_DEV_RISCV_AIA_GRP_CTRL 2
|
||||
#define KVM_DEV_RISCV_AIA_CTRL_INIT 0
|
||||
|
||||
/*
|
||||
* The device attribute type contains the memory mapped offset of the
|
||||
* APLIC register (range 0x0000-0x3FFF) and it must be 4-byte aligned.
|
||||
*/
|
||||
#define KVM_DEV_RISCV_AIA_GRP_APLIC 3
|
||||
|
||||
/*
|
||||
* The lower 12-bits of the device attribute type contains the iselect
|
||||
* value of the IMSIC register (range 0x70-0xFF) whereas the higher order
|
||||
* bits contains the VCPU id.
|
||||
*/
|
||||
#define KVM_DEV_RISCV_AIA_GRP_IMSIC 4
|
||||
#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS 12
|
||||
#define KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK \
|
||||
((1U << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) - 1)
|
||||
#define KVM_DEV_RISCV_AIA_IMSIC_MKATTR(__vcpu, __isel) \
|
||||
(((__vcpu) << KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS) | \
|
||||
((__isel) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK))
|
||||
#define KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(__attr) \
|
||||
((__attr) & KVM_DEV_RISCV_AIA_IMSIC_ISEL_MASK)
|
||||
#define KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(__attr) \
|
||||
((__attr) >> KVM_DEV_RISCV_AIA_IMSIC_ISEL_BITS)
|
||||
|
||||
/* One single KVM irqchip, ie. the AIA */
|
||||
#define KVM_NR_IRQCHIPS 1
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_KVM_RISCV_H */
|
||||
|
|
|
@ -21,6 +21,10 @@ config KVM
|
|||
tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)"
|
||||
depends on RISCV_SBI && MMU
|
||||
select HAVE_KVM_EVENTFD
|
||||
select HAVE_KVM_IRQCHIP
|
||||
select HAVE_KVM_IRQFD
|
||||
select HAVE_KVM_IRQ_ROUTING
|
||||
select HAVE_KVM_MSI
|
||||
select HAVE_KVM_VCPU_ASYNC_IOCTL
|
||||
select KVM_GENERIC_DIRTYLOG_READ_PROTECT
|
||||
select KVM_GENERIC_HARDWARE_ENABLING
|
||||
|
|
|
@ -27,3 +27,6 @@ kvm-y += vcpu_sbi_hsm.o
|
|||
kvm-y += vcpu_timer.o
|
||||
kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o vcpu_sbi_pmu.o
|
||||
kvm-y += aia.o
|
||||
kvm-y += aia_device.o
|
||||
kvm-y += aia_aplic.o
|
||||
kvm-y += aia_imsic.o
|
||||
|
|
|
@ -8,11 +8,49 @@
|
|||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/irq.h>
|
||||
#include <linux/irqdomain.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/hwcap.h>
|
||||
#include <asm/kvm_aia_imsic.h>
|
||||
|
||||
struct aia_hgei_control {
|
||||
raw_spinlock_t lock;
|
||||
unsigned long free_bitmap;
|
||||
struct kvm_vcpu *owners[BITS_PER_LONG];
|
||||
};
|
||||
static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
|
||||
static int hgei_parent_irq;
|
||||
|
||||
unsigned int kvm_riscv_aia_nr_hgei;
|
||||
unsigned int kvm_riscv_aia_max_ids;
|
||||
DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
|
||||
|
||||
static int aia_find_hgei(struct kvm_vcpu *owner)
|
||||
{
|
||||
int i, hgei;
|
||||
unsigned long flags;
|
||||
struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
|
||||
hgei = -1;
|
||||
for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) {
|
||||
if (hgctrl->owners[i] == owner) {
|
||||
hgei = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
|
||||
put_cpu_ptr(&aia_hgei);
|
||||
return hgei;
|
||||
}
|
||||
|
||||
static void aia_set_hvictl(bool ext_irq_pending)
|
||||
{
|
||||
unsigned long hvictl;
|
||||
|
@ -56,6 +94,7 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
|
|||
|
||||
bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
|
||||
{
|
||||
int hgei;
|
||||
unsigned long seip;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
|
@ -74,6 +113,10 @@ bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
|
|||
if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
|
||||
return false;
|
||||
|
||||
hgei = aia_find_hgei(vcpu);
|
||||
if (hgei > 0)
|
||||
return !!(csr_read(CSR_HGEIP) & BIT(hgei));
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -323,8 +366,6 @@ static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel,
|
|||
return KVM_INSN_CONTINUE_NEXT_SEPC;
|
||||
}
|
||||
|
||||
#define IMSIC_FIRST 0x70
|
||||
#define IMSIC_LAST 0xff
|
||||
int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
|
||||
unsigned long *val, unsigned long new_val,
|
||||
unsigned long wr_mask)
|
||||
|
@ -348,6 +389,143 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
|
|||
return KVM_INSN_EXIT_TO_USER_SPACE;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
|
||||
void __iomem **hgei_va, phys_addr_t *hgei_pa)
|
||||
{
|
||||
int ret = -ENOENT;
|
||||
unsigned long flags;
|
||||
struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
|
||||
|
||||
if (!kvm_riscv_aia_available() || !hgctrl)
|
||||
return -ENODEV;
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
|
||||
if (hgctrl->free_bitmap) {
|
||||
ret = __ffs(hgctrl->free_bitmap);
|
||||
hgctrl->free_bitmap &= ~BIT(ret);
|
||||
hgctrl->owners[ret] = owner;
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
|
||||
/* TODO: To be updated later by AIA IMSIC HW guest file support */
|
||||
if (hgei_va)
|
||||
*hgei_va = NULL;
|
||||
if (hgei_pa)
|
||||
*hgei_pa = 0;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_free_hgei(int cpu, int hgei)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
|
||||
|
||||
if (!kvm_riscv_aia_available() || !hgctrl)
|
||||
return;
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
|
||||
if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
|
||||
if (!(hgctrl->free_bitmap & BIT(hgei))) {
|
||||
hgctrl->free_bitmap |= BIT(hgei);
|
||||
hgctrl->owners[hgei] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable)
|
||||
{
|
||||
int hgei;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
return;
|
||||
|
||||
hgei = aia_find_hgei(owner);
|
||||
if (hgei > 0) {
|
||||
if (enable)
|
||||
csr_set(CSR_HGEIE, BIT(hgei));
|
||||
else
|
||||
csr_clear(CSR_HGEIE, BIT(hgei));
|
||||
}
|
||||
}
|
||||
|
||||
static irqreturn_t hgei_interrupt(int irq, void *dev_id)
|
||||
{
|
||||
int i;
|
||||
unsigned long hgei_mask, flags;
|
||||
struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
|
||||
|
||||
hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
|
||||
csr_clear(CSR_HGEIE, hgei_mask);
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
|
||||
for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
|
||||
if (hgctrl->owners[i])
|
||||
kvm_vcpu_kick(hgctrl->owners[i]);
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
|
||||
put_cpu_ptr(&aia_hgei);
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
static int aia_hgei_init(void)
|
||||
{
|
||||
int cpu, rc;
|
||||
struct irq_domain *domain;
|
||||
struct aia_hgei_control *hgctrl;
|
||||
|
||||
/* Initialize per-CPU guest external interrupt line management */
|
||||
for_each_possible_cpu(cpu) {
|
||||
hgctrl = per_cpu_ptr(&aia_hgei, cpu);
|
||||
raw_spin_lock_init(&hgctrl->lock);
|
||||
if (kvm_riscv_aia_nr_hgei) {
|
||||
hgctrl->free_bitmap =
|
||||
BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
|
||||
hgctrl->free_bitmap &= ~BIT(0);
|
||||
} else
|
||||
hgctrl->free_bitmap = 0;
|
||||
}
|
||||
|
||||
/* Find INTC irq domain */
|
||||
domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
|
||||
DOMAIN_BUS_ANY);
|
||||
if (!domain) {
|
||||
kvm_err("unable to find INTC domain\n");
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* Map per-CPU SGEI interrupt from INTC domain */
|
||||
hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
|
||||
if (!hgei_parent_irq) {
|
||||
kvm_err("unable to map SGEI IRQ\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/* Request per-CPU SGEI interrupt */
|
||||
rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
|
||||
"riscv-kvm", &aia_hgei);
|
||||
if (rc) {
|
||||
kvm_err("failed to request SGEI IRQ\n");
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void aia_hgei_exit(void)
|
||||
{
|
||||
/* Free per-CPU SGEI interrupt */
|
||||
free_percpu_irq(hgei_parent_irq, &aia_hgei);
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_enable(void)
|
||||
{
|
||||
if (!kvm_riscv_aia_available())
|
||||
|
@ -362,21 +540,105 @@ void kvm_riscv_aia_enable(void)
|
|||
csr_write(CSR_HVIPRIO1H, 0x0);
|
||||
csr_write(CSR_HVIPRIO2H, 0x0);
|
||||
#endif
|
||||
|
||||
/* Enable per-CPU SGEI interrupt */
|
||||
enable_percpu_irq(hgei_parent_irq,
|
||||
irq_get_trigger_type(hgei_parent_irq));
|
||||
csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_disable(void)
|
||||
{
|
||||
int i;
|
||||
unsigned long flags;
|
||||
struct kvm_vcpu *vcpu;
|
||||
struct aia_hgei_control *hgctrl;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
return;
|
||||
hgctrl = get_cpu_ptr(&aia_hgei);
|
||||
|
||||
/* Disable per-CPU SGEI interrupt */
|
||||
csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
|
||||
disable_percpu_irq(hgei_parent_irq);
|
||||
|
||||
aia_set_hvictl(false);
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
|
||||
for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
|
||||
vcpu = hgctrl->owners[i];
|
||||
if (!vcpu)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* We release hgctrl->lock before notifying IMSIC
|
||||
* so that we don't have lock ordering issues.
|
||||
*/
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
|
||||
/* Notify IMSIC */
|
||||
kvm_riscv_vcpu_aia_imsic_release(vcpu);
|
||||
|
||||
/*
|
||||
* Wakeup VCPU if it was blocked so that it can
|
||||
* run on other HARTs
|
||||
*/
|
||||
if (csr_read(CSR_HGEIE) & BIT(i)) {
|
||||
csr_clear(CSR_HGEIE, BIT(i));
|
||||
kvm_vcpu_kick(vcpu);
|
||||
}
|
||||
|
||||
raw_spin_lock_irqsave(&hgctrl->lock, flags);
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
|
||||
|
||||
put_cpu_ptr(&aia_hgei);
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_init(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (!riscv_isa_extension_available(NULL, SxAIA))
|
||||
return -ENODEV;
|
||||
|
||||
/* Figure-out number of bits in HGEIE */
|
||||
csr_write(CSR_HGEIE, -1UL);
|
||||
kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
|
||||
csr_write(CSR_HGEIE, 0);
|
||||
if (kvm_riscv_aia_nr_hgei)
|
||||
kvm_riscv_aia_nr_hgei--;
|
||||
|
||||
/*
|
||||
* Number of usable HGEI lines should be minimum of per-HART
|
||||
* IMSIC guest files and number of bits in HGEIE
|
||||
*
|
||||
* TODO: To be updated later by AIA IMSIC HW guest file support
|
||||
*/
|
||||
kvm_riscv_aia_nr_hgei = 0;
|
||||
|
||||
/*
|
||||
* Find number of guest MSI IDs
|
||||
*
|
||||
* TODO: To be updated later by AIA IMSIC HW guest file support
|
||||
*/
|
||||
kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
|
||||
|
||||
/* Initialize guest external interrupt line management */
|
||||
rc = aia_hgei_init();
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
/* Register device operations */
|
||||
rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
|
||||
KVM_DEV_TYPE_RISCV_AIA);
|
||||
if (rc) {
|
||||
aia_hgei_exit();
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Enable KVM AIA support */
|
||||
static_branch_enable(&kvm_riscv_aia_available);
|
||||
|
||||
|
@ -385,4 +647,12 @@ int kvm_riscv_aia_init(void)
|
|||
|
||||
void kvm_riscv_aia_exit(void)
|
||||
{
|
||||
if (!kvm_riscv_aia_available())
|
||||
return;
|
||||
|
||||
/* Unregister device operations */
|
||||
kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
|
||||
|
||||
/* Cleanup the HGEI state */
|
||||
aia_hgei_exit();
|
||||
}
|
||||
|
|
619
arch/riscv/kvm/aia_aplic.c
Normal file
619
arch/riscv/kvm/aia_aplic.c
Normal file
|
@ -0,0 +1,619 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
|
||||
* Copyright (C) 2022 Ventana Micro Systems Inc.
|
||||
*
|
||||
* Authors:
|
||||
* Anup Patel <apatel@ventanamicro.com>
|
||||
*/
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/math.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/swab.h>
|
||||
#include <kvm/iodev.h>
|
||||
#include <asm/kvm_aia_aplic.h>
|
||||
|
||||
struct aplic_irq {
|
||||
raw_spinlock_t lock;
|
||||
u32 sourcecfg;
|
||||
u32 state;
|
||||
#define APLIC_IRQ_STATE_PENDING BIT(0)
|
||||
#define APLIC_IRQ_STATE_ENABLED BIT(1)
|
||||
#define APLIC_IRQ_STATE_ENPEND (APLIC_IRQ_STATE_PENDING | \
|
||||
APLIC_IRQ_STATE_ENABLED)
|
||||
#define APLIC_IRQ_STATE_INPUT BIT(8)
|
||||
u32 target;
|
||||
};
|
||||
|
||||
struct aplic {
|
||||
struct kvm_io_device iodev;
|
||||
|
||||
u32 domaincfg;
|
||||
u32 genmsi;
|
||||
|
||||
u32 nr_irqs;
|
||||
u32 nr_words;
|
||||
struct aplic_irq *irqs;
|
||||
};
|
||||
|
||||
static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq)
|
||||
{
|
||||
u32 ret;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return 0;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
ret = irqd->sourcecfg;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
if (val & APLIC_SOURCECFG_D)
|
||||
val = 0;
|
||||
else
|
||||
val &= APLIC_SOURCECFG_SM_MASK;
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
irqd->sourcecfg = val;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
}
|
||||
|
||||
static u32 aplic_read_target(struct aplic *aplic, u32 irq)
|
||||
{
|
||||
u32 ret;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return 0;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
ret = irqd->target;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
val &= APLIC_TARGET_EIID_MASK |
|
||||
(APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) |
|
||||
(APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT);
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
irqd->target = val;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
}
|
||||
|
||||
static bool aplic_read_pending(struct aplic *aplic, u32 irq)
|
||||
{
|
||||
bool ret;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return false;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending)
|
||||
{
|
||||
unsigned long flags, sm;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
|
||||
sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK;
|
||||
if (!pending &&
|
||||
((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) ||
|
||||
(sm == APLIC_SOURCECFG_SM_LEVEL_LOW)))
|
||||
goto skip_write_pending;
|
||||
|
||||
if (pending)
|
||||
irqd->state |= APLIC_IRQ_STATE_PENDING;
|
||||
else
|
||||
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
|
||||
|
||||
skip_write_pending:
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
}
|
||||
|
||||
static bool aplic_read_enabled(struct aplic *aplic, u32 irq)
|
||||
{
|
||||
bool ret;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return false;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
if (enabled)
|
||||
irqd->state |= APLIC_IRQ_STATE_ENABLED;
|
||||
else
|
||||
irqd->state &= ~APLIC_IRQ_STATE_ENABLED;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
}
|
||||
|
||||
static bool aplic_read_input(struct aplic *aplic, u32 irq)
|
||||
{
|
||||
bool ret;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
return false;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
ret = (irqd->state & APLIC_IRQ_STATE_INPUT) ? true : false;
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
|
||||
{
|
||||
u32 hart_idx, guest_idx, eiid;
|
||||
|
||||
hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT;
|
||||
hart_idx &= APLIC_TARGET_HART_IDX_MASK;
|
||||
guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT;
|
||||
guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
|
||||
eiid = target & APLIC_TARGET_EIID_MASK;
|
||||
kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
|
||||
}
|
||||
|
||||
static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
|
||||
{
|
||||
bool inject;
|
||||
u32 irq, target;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
struct aplic *aplic = kvm->arch.aia.aplic_state;
|
||||
|
||||
if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE))
|
||||
return;
|
||||
|
||||
for (irq = first; irq <= last; irq++) {
|
||||
if (!irq || aplic->nr_irqs <= irq)
|
||||
continue;
|
||||
irqd = &aplic->irqs[irq];
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
|
||||
inject = false;
|
||||
target = irqd->target;
|
||||
if ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
|
||||
APLIC_IRQ_STATE_ENPEND) {
|
||||
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
|
||||
inject = true;
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
if (inject)
|
||||
aplic_inject_msi(kvm, irq, target);
|
||||
}
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
|
||||
{
|
||||
u32 target;
|
||||
bool inject = false, ie;
|
||||
unsigned long flags;
|
||||
struct aplic_irq *irqd;
|
||||
struct aplic *aplic = kvm->arch.aia.aplic_state;
|
||||
|
||||
if (!aplic || !source || (aplic->nr_irqs <= source))
|
||||
return -ENODEV;
|
||||
irqd = &aplic->irqs[source];
|
||||
ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false;
|
||||
|
||||
raw_spin_lock_irqsave(&irqd->lock, flags);
|
||||
|
||||
if (irqd->sourcecfg & APLIC_SOURCECFG_D)
|
||||
goto skip_unlock;
|
||||
|
||||
switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) {
|
||||
case APLIC_SOURCECFG_SM_EDGE_RISE:
|
||||
if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) &&
|
||||
!(irqd->state & APLIC_IRQ_STATE_PENDING))
|
||||
irqd->state |= APLIC_IRQ_STATE_PENDING;
|
||||
break;
|
||||
case APLIC_SOURCECFG_SM_EDGE_FALL:
|
||||
if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) &&
|
||||
!(irqd->state & APLIC_IRQ_STATE_PENDING))
|
||||
irqd->state |= APLIC_IRQ_STATE_PENDING;
|
||||
break;
|
||||
case APLIC_SOURCECFG_SM_LEVEL_HIGH:
|
||||
if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
|
||||
irqd->state |= APLIC_IRQ_STATE_PENDING;
|
||||
break;
|
||||
case APLIC_SOURCECFG_SM_LEVEL_LOW:
|
||||
if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
|
||||
irqd->state |= APLIC_IRQ_STATE_PENDING;
|
||||
break;
|
||||
}
|
||||
|
||||
if (level)
|
||||
irqd->state |= APLIC_IRQ_STATE_INPUT;
|
||||
else
|
||||
irqd->state &= ~APLIC_IRQ_STATE_INPUT;
|
||||
|
||||
target = irqd->target;
|
||||
if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
|
||||
APLIC_IRQ_STATE_ENPEND)) {
|
||||
irqd->state &= ~APLIC_IRQ_STATE_PENDING;
|
||||
inject = true;
|
||||
}
|
||||
|
||||
skip_unlock:
|
||||
raw_spin_unlock_irqrestore(&irqd->lock, flags);
|
||||
|
||||
if (inject)
|
||||
aplic_inject_msi(kvm, source, target);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static u32 aplic_read_input_word(struct aplic *aplic, u32 word)
|
||||
{
|
||||
u32 i, ret = 0;
|
||||
|
||||
for (i = 0; i < 32; i++)
|
||||
ret |= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static u32 aplic_read_pending_word(struct aplic *aplic, u32 word)
|
||||
{
|
||||
u32 i, ret = 0;
|
||||
|
||||
for (i = 0; i < 32; i++)
|
||||
ret |= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_pending_word(struct aplic *aplic, u32 word,
|
||||
u32 val, bool pending)
|
||||
{
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
if (val & BIT(i))
|
||||
aplic_write_pending(aplic, word * 32 + i, pending);
|
||||
}
|
||||
}
|
||||
|
||||
static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word)
|
||||
{
|
||||
u32 i, ret = 0;
|
||||
|
||||
for (i = 0; i < 32; i++)
|
||||
ret |= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aplic_write_enabled_word(struct aplic *aplic, u32 word,
|
||||
u32 val, bool enabled)
|
||||
{
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < 32; i++) {
|
||||
if (val & BIT(i))
|
||||
aplic_write_enabled(aplic, word * 32 + i, enabled);
|
||||
}
|
||||
}
|
||||
|
||||
static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32)
|
||||
{
|
||||
u32 i;
|
||||
struct aplic *aplic = kvm->arch.aia.aplic_state;
|
||||
|
||||
if ((off & 0x3) != 0)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (off == APLIC_DOMAINCFG) {
|
||||
*val32 = APLIC_DOMAINCFG_RDONLY |
|
||||
aplic->domaincfg | APLIC_DOMAINCFG_DM;
|
||||
} else if ((off >= APLIC_SOURCECFG_BASE) &&
|
||||
(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
|
||||
i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
|
||||
*val32 = aplic_read_sourcecfg(aplic, i);
|
||||
} else if ((off >= APLIC_SETIP_BASE) &&
|
||||
(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_SETIP_BASE) >> 2;
|
||||
*val32 = aplic_read_pending_word(aplic, i);
|
||||
} else if (off == APLIC_SETIPNUM) {
|
||||
*val32 = 0;
|
||||
} else if ((off >= APLIC_CLRIP_BASE) &&
|
||||
(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_CLRIP_BASE) >> 2;
|
||||
*val32 = aplic_read_input_word(aplic, i);
|
||||
} else if (off == APLIC_CLRIPNUM) {
|
||||
*val32 = 0;
|
||||
} else if ((off >= APLIC_SETIE_BASE) &&
|
||||
(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_SETIE_BASE) >> 2;
|
||||
*val32 = aplic_read_enabled_word(aplic, i);
|
||||
} else if (off == APLIC_SETIENUM) {
|
||||
*val32 = 0;
|
||||
} else if ((off >= APLIC_CLRIE_BASE) &&
|
||||
(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
|
||||
*val32 = 0;
|
||||
} else if (off == APLIC_CLRIENUM) {
|
||||
*val32 = 0;
|
||||
} else if (off == APLIC_SETIPNUM_LE) {
|
||||
*val32 = 0;
|
||||
} else if (off == APLIC_SETIPNUM_BE) {
|
||||
*val32 = 0;
|
||||
} else if (off == APLIC_GENMSI) {
|
||||
*val32 = aplic->genmsi;
|
||||
} else if ((off >= APLIC_TARGET_BASE) &&
|
||||
(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
|
||||
i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
|
||||
*val32 = aplic_read_target(aplic, i);
|
||||
} else
|
||||
return -ENODEV;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aplic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
|
||||
gpa_t addr, int len, void *val)
|
||||
{
|
||||
if (len != 4)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
return aplic_mmio_read_offset(vcpu->kvm,
|
||||
addr - vcpu->kvm->arch.aia.aplic_addr,
|
||||
val);
|
||||
}
|
||||
|
||||
static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
|
||||
{
|
||||
u32 i;
|
||||
struct aplic *aplic = kvm->arch.aia.aplic_state;
|
||||
|
||||
if ((off & 0x3) != 0)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (off == APLIC_DOMAINCFG) {
|
||||
/* Only IE bit writeable */
|
||||
aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE;
|
||||
} else if ((off >= APLIC_SOURCECFG_BASE) &&
|
||||
(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
|
||||
i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
|
||||
aplic_write_sourcecfg(aplic, i, val32);
|
||||
} else if ((off >= APLIC_SETIP_BASE) &&
|
||||
(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_SETIP_BASE) >> 2;
|
||||
aplic_write_pending_word(aplic, i, val32, true);
|
||||
} else if (off == APLIC_SETIPNUM) {
|
||||
aplic_write_pending(aplic, val32, true);
|
||||
} else if ((off >= APLIC_CLRIP_BASE) &&
|
||||
(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_CLRIP_BASE) >> 2;
|
||||
aplic_write_pending_word(aplic, i, val32, false);
|
||||
} else if (off == APLIC_CLRIPNUM) {
|
||||
aplic_write_pending(aplic, val32, false);
|
||||
} else if ((off >= APLIC_SETIE_BASE) &&
|
||||
(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_SETIE_BASE) >> 2;
|
||||
aplic_write_enabled_word(aplic, i, val32, true);
|
||||
} else if (off == APLIC_SETIENUM) {
|
||||
aplic_write_enabled(aplic, val32, true);
|
||||
} else if ((off >= APLIC_CLRIE_BASE) &&
|
||||
(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
|
||||
i = (off - APLIC_CLRIE_BASE) >> 2;
|
||||
aplic_write_enabled_word(aplic, i, val32, false);
|
||||
} else if (off == APLIC_CLRIENUM) {
|
||||
aplic_write_enabled(aplic, val32, false);
|
||||
} else if (off == APLIC_SETIPNUM_LE) {
|
||||
aplic_write_pending(aplic, val32, true);
|
||||
} else if (off == APLIC_SETIPNUM_BE) {
|
||||
aplic_write_pending(aplic, __swab32(val32), true);
|
||||
} else if (off == APLIC_GENMSI) {
|
||||
aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK <<
|
||||
APLIC_TARGET_GUEST_IDX_SHIFT);
|
||||
kvm_riscv_aia_inject_msi_by_id(kvm,
|
||||
val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0,
|
||||
val32 & APLIC_TARGET_EIID_MASK);
|
||||
} else if ((off >= APLIC_TARGET_BASE) &&
|
||||
(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
|
||||
i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
|
||||
aplic_write_target(aplic, i, val32);
|
||||
} else
|
||||
return -ENODEV;
|
||||
|
||||
aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aplic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
|
||||
gpa_t addr, int len, const void *val)
|
||||
{
|
||||
if (len != 4)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
return aplic_mmio_write_offset(vcpu->kvm,
|
||||
addr - vcpu->kvm->arch.aia.aplic_addr,
|
||||
*((const u32 *)val));
|
||||
}
|
||||
|
||||
static struct kvm_io_device_ops aplic_iodoev_ops = {
|
||||
.read = aplic_mmio_read,
|
||||
.write = aplic_mmio_write,
|
||||
};
|
||||
|
||||
int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (!kvm->arch.aia.aplic_state)
|
||||
return -ENODEV;
|
||||
|
||||
rc = aplic_mmio_write_offset(kvm, type, v);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (!kvm->arch.aia.aplic_state)
|
||||
return -ENODEV;
|
||||
|
||||
rc = aplic_mmio_read_offset(kvm, type, v);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
|
||||
{
|
||||
int rc;
|
||||
u32 val;
|
||||
|
||||
if (!kvm->arch.aia.aplic_state)
|
||||
return -ENODEV;
|
||||
|
||||
rc = aplic_mmio_read_offset(kvm, type, &val);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_aplic_init(struct kvm *kvm)
|
||||
{
|
||||
int i, ret = 0;
|
||||
struct aplic *aplic;
|
||||
|
||||
/* Do nothing if we have zero sources */
|
||||
if (!kvm->arch.aia.nr_sources)
|
||||
return 0;
|
||||
|
||||
/* Allocate APLIC global state */
|
||||
aplic = kzalloc(sizeof(*aplic), GFP_KERNEL);
|
||||
if (!aplic)
|
||||
return -ENOMEM;
|
||||
kvm->arch.aia.aplic_state = aplic;
|
||||
|
||||
/* Setup APLIC IRQs */
|
||||
aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
|
||||
aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32);
|
||||
aplic->irqs = kcalloc(aplic->nr_irqs,
|
||||
sizeof(*aplic->irqs), GFP_KERNEL);
|
||||
if (!aplic->irqs) {
|
||||
ret = -ENOMEM;
|
||||
goto fail_free_aplic;
|
||||
}
|
||||
for (i = 0; i < aplic->nr_irqs; i++)
|
||||
raw_spin_lock_init(&aplic->irqs[i].lock);
|
||||
|
||||
/* Setup IO device */
|
||||
kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops);
|
||||
mutex_lock(&kvm->slots_lock);
|
||||
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
|
||||
kvm->arch.aia.aplic_addr,
|
||||
KVM_DEV_RISCV_APLIC_SIZE,
|
||||
&aplic->iodev);
|
||||
mutex_unlock(&kvm->slots_lock);
|
||||
if (ret)
|
||||
goto fail_free_aplic_irqs;
|
||||
|
||||
/* Setup default IRQ routing */
|
||||
ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
|
||||
if (ret)
|
||||
goto fail_unreg_iodev;
|
||||
|
||||
return 0;
|
||||
|
||||
fail_unreg_iodev:
|
||||
mutex_lock(&kvm->slots_lock);
|
||||
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
|
||||
mutex_unlock(&kvm->slots_lock);
|
||||
fail_free_aplic_irqs:
|
||||
kfree(aplic->irqs);
|
||||
fail_free_aplic:
|
||||
kvm->arch.aia.aplic_state = NULL;
|
||||
kfree(aplic);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
|
||||
{
|
||||
struct aplic *aplic = kvm->arch.aia.aplic_state;
|
||||
|
||||
if (!aplic)
|
||||
return;
|
||||
|
||||
mutex_lock(&kvm->slots_lock);
|
||||
kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
|
||||
mutex_unlock(&kvm->slots_lock);
|
||||
|
||||
kfree(aplic->irqs);
|
||||
|
||||
kvm->arch.aia.aplic_state = NULL;
|
||||
kfree(aplic);
|
||||
}
|
673
arch/riscv/kvm/aia_device.c
Normal file
673
arch/riscv/kvm/aia_device.c
Normal file
|
@ -0,0 +1,673 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (C) 2021 Western Digital Corporation or its affiliates.
|
||||
* Copyright (C) 2022 Ventana Micro Systems Inc.
|
||||
*
|
||||
* Authors:
|
||||
* Anup Patel <apatel@ventanamicro.com>
|
||||
*/
|
||||
|
||||
#include <linux/bits.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <asm/kvm_aia_imsic.h>
|
||||
|
||||
static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
|
||||
{
|
||||
struct kvm_vcpu *tmp_vcpu;
|
||||
|
||||
for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
|
||||
tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
|
||||
mutex_unlock(&tmp_vcpu->mutex);
|
||||
}
|
||||
}
|
||||
|
||||
static void unlock_all_vcpus(struct kvm *kvm)
|
||||
{
|
||||
unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
|
||||
}
|
||||
|
||||
static bool lock_all_vcpus(struct kvm *kvm)
|
||||
{
|
||||
struct kvm_vcpu *tmp_vcpu;
|
||||
unsigned long c;
|
||||
|
||||
kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
|
||||
if (!mutex_trylock(&tmp_vcpu->mutex)) {
|
||||
unlock_vcpus(kvm, c - 1);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int aia_create(struct kvm_device *dev, u32 type)
|
||||
{
|
||||
int ret;
|
||||
unsigned long i;
|
||||
struct kvm *kvm = dev->kvm;
|
||||
struct kvm_vcpu *vcpu;
|
||||
|
||||
if (irqchip_in_kernel(kvm))
|
||||
return -EEXIST;
|
||||
|
||||
ret = -EBUSY;
|
||||
if (!lock_all_vcpus(kvm))
|
||||
return ret;
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
if (vcpu->arch.ran_atleast_once)
|
||||
goto out_unlock;
|
||||
}
|
||||
ret = 0;
|
||||
|
||||
kvm->arch.aia.in_kernel = true;
|
||||
|
||||
out_unlock:
|
||||
unlock_all_vcpus(kvm);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void aia_destroy(struct kvm_device *dev)
|
||||
{
|
||||
kfree(dev);
|
||||
}
|
||||
|
||||
static int aia_config(struct kvm *kvm, unsigned long type,
|
||||
u32 *nr, bool write)
|
||||
{
|
||||
struct kvm_aia *aia = &kvm->arch.aia;
|
||||
|
||||
/* Writes can only be done before irqchip is initialized */
|
||||
if (write && kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
switch (type) {
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_MODE:
|
||||
if (write) {
|
||||
switch (*nr) {
|
||||
case KVM_DEV_RISCV_AIA_MODE_EMUL:
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
|
||||
case KVM_DEV_RISCV_AIA_MODE_AUTO:
|
||||
/*
|
||||
* HW Acceleration and Auto modes only
|
||||
* supported on host with non-zero guest
|
||||
* external interrupts (i.e. non-zero
|
||||
* VS-level IMSIC pages).
|
||||
*/
|
||||
if (!kvm_riscv_aia_nr_hgei)
|
||||
return -EINVAL;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
aia->mode = *nr;
|
||||
} else
|
||||
*nr = aia->mode;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_IDS:
|
||||
if (write) {
|
||||
if ((*nr < KVM_DEV_RISCV_AIA_IDS_MIN) ||
|
||||
(*nr >= KVM_DEV_RISCV_AIA_IDS_MAX) ||
|
||||
((*nr & KVM_DEV_RISCV_AIA_IDS_MIN) !=
|
||||
KVM_DEV_RISCV_AIA_IDS_MIN) ||
|
||||
(kvm_riscv_aia_max_ids <= *nr))
|
||||
return -EINVAL;
|
||||
aia->nr_ids = *nr;
|
||||
} else
|
||||
*nr = aia->nr_ids;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
|
||||
if (write) {
|
||||
if ((*nr >= KVM_DEV_RISCV_AIA_SRCS_MAX) ||
|
||||
(*nr >= kvm_riscv_aia_max_ids))
|
||||
return -EINVAL;
|
||||
aia->nr_sources = *nr;
|
||||
} else
|
||||
*nr = aia->nr_sources;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
|
||||
if (write) {
|
||||
if (*nr >= KVM_DEV_RISCV_AIA_GROUP_BITS_MAX)
|
||||
return -EINVAL;
|
||||
aia->nr_group_bits = *nr;
|
||||
} else
|
||||
*nr = aia->nr_group_bits;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
|
||||
if (write) {
|
||||
if ((*nr < KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN) ||
|
||||
(*nr >= KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX))
|
||||
return -EINVAL;
|
||||
aia->nr_group_shift = *nr;
|
||||
} else
|
||||
*nr = aia->nr_group_shift;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
|
||||
if (write) {
|
||||
if (*nr >= KVM_DEV_RISCV_AIA_HART_BITS_MAX)
|
||||
return -EINVAL;
|
||||
aia->nr_hart_bits = *nr;
|
||||
} else
|
||||
*nr = aia->nr_hart_bits;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
|
||||
if (write) {
|
||||
if (*nr >= KVM_DEV_RISCV_AIA_GUEST_BITS_MAX)
|
||||
return -EINVAL;
|
||||
aia->nr_guest_bits = *nr;
|
||||
} else
|
||||
*nr = aia->nr_guest_bits;
|
||||
break;
|
||||
default:
|
||||
return -ENXIO;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write)
|
||||
{
|
||||
struct kvm_aia *aia = &kvm->arch.aia;
|
||||
|
||||
if (write) {
|
||||
/* Writes can only be done before irqchip is initialized */
|
||||
if (kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
if (*addr & (KVM_DEV_RISCV_APLIC_ALIGN - 1))
|
||||
return -EINVAL;
|
||||
|
||||
aia->aplic_addr = *addr;
|
||||
} else
|
||||
*addr = aia->aplic_addr;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int aia_imsic_addr(struct kvm *kvm, u64 *addr,
|
||||
unsigned long vcpu_idx, bool write)
|
||||
{
|
||||
struct kvm_vcpu *vcpu;
|
||||
struct kvm_vcpu_aia *vcpu_aia;
|
||||
|
||||
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
|
||||
if (!vcpu)
|
||||
return -EINVAL;
|
||||
vcpu_aia = &vcpu->arch.aia_context;
|
||||
|
||||
if (write) {
|
||||
/* Writes can only be done before irqchip is initialized */
|
||||
if (kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
if (*addr & (KVM_DEV_RISCV_IMSIC_ALIGN - 1))
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
mutex_lock(&vcpu->mutex);
|
||||
if (write)
|
||||
vcpu_aia->imsic_addr = *addr;
|
||||
else
|
||||
*addr = vcpu_aia->imsic_addr;
|
||||
mutex_unlock(&vcpu->mutex);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static gpa_t aia_imsic_ppn(struct kvm_aia *aia, gpa_t addr)
|
||||
{
|
||||
u32 h, l;
|
||||
gpa_t mask = 0;
|
||||
|
||||
h = aia->nr_hart_bits + aia->nr_guest_bits +
|
||||
IMSIC_MMIO_PAGE_SHIFT - 1;
|
||||
mask = GENMASK_ULL(h, 0);
|
||||
|
||||
if (aia->nr_group_bits) {
|
||||
h = aia->nr_group_bits + aia->nr_group_shift - 1;
|
||||
l = aia->nr_group_shift;
|
||||
mask |= GENMASK_ULL(h, l);
|
||||
}
|
||||
|
||||
return (addr & ~mask) >> IMSIC_MMIO_PAGE_SHIFT;
|
||||
}
|
||||
|
||||
static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr)
|
||||
{
|
||||
u32 hart, group = 0;
|
||||
|
||||
hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
|
||||
GENMASK_ULL(aia->nr_hart_bits - 1, 0);
|
||||
if (aia->nr_group_bits)
|
||||
group = (addr >> aia->nr_group_shift) &
|
||||
GENMASK_ULL(aia->nr_group_bits - 1, 0);
|
||||
|
||||
return (group << aia->nr_hart_bits) | hart;
|
||||
}
|
||||
|
||||
static int aia_init(struct kvm *kvm)
|
||||
{
|
||||
int ret, i;
|
||||
unsigned long idx;
|
||||
struct kvm_vcpu *vcpu;
|
||||
struct kvm_vcpu_aia *vaia;
|
||||
struct kvm_aia *aia = &kvm->arch.aia;
|
||||
gpa_t base_ppn = KVM_RISCV_AIA_UNDEF_ADDR;
|
||||
|
||||
/* Irqchip can be initialized only once */
|
||||
if (kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
/* We might be in the middle of creating a VCPU? */
|
||||
if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
|
||||
return -EBUSY;
|
||||
|
||||
/* Number of sources should be less than or equals number of IDs */
|
||||
if (aia->nr_ids < aia->nr_sources)
|
||||
return -EINVAL;
|
||||
|
||||
/* APLIC base is required for non-zero number of sources */
|
||||
if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR)
|
||||
return -EINVAL;
|
||||
|
||||
/* Initialize APLIC */
|
||||
ret = kvm_riscv_aia_aplic_init(kvm);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/* Iterate over each VCPU */
|
||||
kvm_for_each_vcpu(idx, vcpu, kvm) {
|
||||
vaia = &vcpu->arch.aia_context;
|
||||
|
||||
/* IMSIC base is required */
|
||||
if (vaia->imsic_addr == KVM_RISCV_AIA_UNDEF_ADDR) {
|
||||
ret = -EINVAL;
|
||||
goto fail_cleanup_imsics;
|
||||
}
|
||||
|
||||
/* All IMSICs should have matching base PPN */
|
||||
if (base_ppn == KVM_RISCV_AIA_UNDEF_ADDR)
|
||||
base_ppn = aia_imsic_ppn(aia, vaia->imsic_addr);
|
||||
if (base_ppn != aia_imsic_ppn(aia, vaia->imsic_addr)) {
|
||||
ret = -EINVAL;
|
||||
goto fail_cleanup_imsics;
|
||||
}
|
||||
|
||||
/* Update HART index of the IMSIC based on IMSIC base */
|
||||
vaia->hart_index = aia_imsic_hart_index(aia,
|
||||
vaia->imsic_addr);
|
||||
|
||||
/* Initialize IMSIC for this VCPU */
|
||||
ret = kvm_riscv_vcpu_aia_imsic_init(vcpu);
|
||||
if (ret)
|
||||
goto fail_cleanup_imsics;
|
||||
}
|
||||
|
||||
/* Set the initialized flag */
|
||||
kvm->arch.aia.initialized = true;
|
||||
|
||||
return 0;
|
||||
|
||||
fail_cleanup_imsics:
|
||||
for (i = idx - 1; i >= 0; i--) {
|
||||
vcpu = kvm_get_vcpu(kvm, i);
|
||||
if (!vcpu)
|
||||
continue;
|
||||
kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
|
||||
}
|
||||
kvm_riscv_aia_aplic_cleanup(kvm);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int aia_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
|
||||
{
|
||||
u32 nr;
|
||||
u64 addr;
|
||||
int nr_vcpus, r = -ENXIO;
|
||||
unsigned long v, type = (unsigned long)attr->attr;
|
||||
void __user *uaddr = (void __user *)(long)attr->addr;
|
||||
|
||||
switch (attr->group) {
|
||||
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
|
||||
if (copy_from_user(&nr, uaddr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = aia_config(dev->kvm, type, &nr, true);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
|
||||
break;
|
||||
|
||||
case KVM_DEV_RISCV_AIA_GRP_ADDR:
|
||||
if (copy_from_user(&addr, uaddr, sizeof(addr)))
|
||||
return -EFAULT;
|
||||
|
||||
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
|
||||
r = aia_aplic_addr(dev->kvm, &addr, true);
|
||||
else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
|
||||
r = aia_imsic_addr(dev->kvm, &addr,
|
||||
type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), true);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
|
||||
break;
|
||||
|
||||
case KVM_DEV_RISCV_AIA_GRP_CTRL:
|
||||
switch (type) {
|
||||
case KVM_DEV_RISCV_AIA_CTRL_INIT:
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = aia_init(dev->kvm);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_APLIC:
|
||||
if (copy_from_user(&nr, uaddr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
|
||||
if (copy_from_user(&v, uaddr, sizeof(v)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
|
||||
{
|
||||
u32 nr;
|
||||
u64 addr;
|
||||
int nr_vcpus, r = -ENXIO;
|
||||
void __user *uaddr = (void __user *)(long)attr->addr;
|
||||
unsigned long v, type = (unsigned long)attr->attr;
|
||||
|
||||
switch (attr->group) {
|
||||
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
|
||||
if (copy_from_user(&nr, uaddr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = aia_config(dev->kvm, type, &nr, false);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
if (r)
|
||||
return r;
|
||||
|
||||
if (copy_to_user(uaddr, &nr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_ADDR:
|
||||
if (copy_from_user(&addr, uaddr, sizeof(addr)))
|
||||
return -EFAULT;
|
||||
|
||||
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
|
||||
r = aia_aplic_addr(dev->kvm, &addr, false);
|
||||
else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
|
||||
r = aia_imsic_addr(dev->kvm, &addr,
|
||||
type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), false);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
if (r)
|
||||
return r;
|
||||
|
||||
if (copy_to_user(uaddr, &addr, sizeof(addr)))
|
||||
return -EFAULT;
|
||||
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_APLIC:
|
||||
if (copy_from_user(&nr, uaddr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
if (r)
|
||||
return r;
|
||||
|
||||
if (copy_to_user(uaddr, &nr, sizeof(nr)))
|
||||
return -EFAULT;
|
||||
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
|
||||
if (copy_from_user(&v, uaddr, sizeof(v)))
|
||||
return -EFAULT;
|
||||
|
||||
mutex_lock(&dev->kvm->lock);
|
||||
r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v);
|
||||
mutex_unlock(&dev->kvm->lock);
|
||||
if (r)
|
||||
return r;
|
||||
|
||||
if (copy_to_user(uaddr, &v, sizeof(v)))
|
||||
return -EFAULT;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
|
||||
{
|
||||
int nr_vcpus;
|
||||
|
||||
switch (attr->group) {
|
||||
case KVM_DEV_RISCV_AIA_GRP_CONFIG:
|
||||
switch (attr->attr) {
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_MODE:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_IDS:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
|
||||
case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_ADDR:
|
||||
nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
|
||||
if (attr->attr == KVM_DEV_RISCV_AIA_ADDR_APLIC)
|
||||
return 0;
|
||||
else if (attr->attr < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
|
||||
return 0;
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_CTRL:
|
||||
switch (attr->attr) {
|
||||
case KVM_DEV_RISCV_AIA_CTRL_INIT:
|
||||
return 0;
|
||||
}
|
||||
break;
|
||||
case KVM_DEV_RISCV_AIA_GRP_APLIC:
|
||||
return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr);
|
||||
case KVM_DEV_RISCV_AIA_GRP_IMSIC:
|
||||
return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr);
|
||||
}
|
||||
|
||||
return -ENXIO;
|
||||
}
|
||||
|
||||
struct kvm_device_ops kvm_riscv_aia_device_ops = {
|
||||
.name = "kvm-riscv-aia",
|
||||
.create = aia_create,
|
||||
.destroy = aia_destroy,
|
||||
.set_attr = aia_set_attr,
|
||||
.get_attr = aia_get_attr,
|
||||
.has_attr = aia_has_attr,
|
||||
};
|
||||
|
||||
int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(vcpu->kvm))
|
||||
return 1;
|
||||
|
||||
/* Update the IMSIC HW state before entering guest mode */
|
||||
return kvm_riscv_vcpu_aia_imsic_update(vcpu);
|
||||
}
|
||||
|
||||
void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
|
||||
struct kvm_vcpu_aia_csr *reset_csr =
|
||||
&vcpu->arch.aia_context.guest_reset_csr;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
return;
|
||||
memcpy(csr, reset_csr, sizeof(*csr));
|
||||
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(vcpu->kvm))
|
||||
return;
|
||||
|
||||
/* Reset the IMSIC context */
|
||||
kvm_riscv_vcpu_aia_imsic_reset(vcpu);
|
||||
}
|
||||
|
||||
int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* We don't do any memory allocations over here because these
|
||||
* will be done after AIA device is initialized by the user-space.
|
||||
*
|
||||
* Refer, aia_init() implementation for more details.
|
||||
*/
|
||||
|
||||
/* Initialize default values in AIA vcpu context */
|
||||
vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
|
||||
vaia->hart_index = vcpu->vcpu_idx;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(vcpu->kvm))
|
||||
return;
|
||||
|
||||
/* Cleanup IMSIC context */
|
||||
kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
|
||||
u32 guest_index, u32 iid)
|
||||
{
|
||||
unsigned long idx;
|
||||
struct kvm_vcpu *vcpu;
|
||||
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
/* Inject MSI to matching VCPU */
|
||||
kvm_for_each_vcpu(idx, vcpu, kvm) {
|
||||
if (vcpu->arch.aia_context.hart_index == hart_index)
|
||||
return kvm_riscv_vcpu_aia_imsic_inject(vcpu,
|
||||
guest_index,
|
||||
0, iid);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
|
||||
{
|
||||
gpa_t tppn, ippn;
|
||||
unsigned long idx;
|
||||
struct kvm_vcpu *vcpu;
|
||||
u32 g, toff, iid = msi->data;
|
||||
struct kvm_aia *aia = &kvm->arch.aia;
|
||||
gpa_t target = (((gpa_t)msi->address_hi) << 32) | msi->address_lo;
|
||||
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
/* Convert target address to target PPN */
|
||||
tppn = target >> IMSIC_MMIO_PAGE_SHIFT;
|
||||
|
||||
/* Extract and clear Guest ID from target PPN */
|
||||
g = tppn & (BIT(aia->nr_guest_bits) - 1);
|
||||
tppn &= ~((gpa_t)(BIT(aia->nr_guest_bits) - 1));
|
||||
|
||||
/* Inject MSI to matching VCPU */
|
||||
kvm_for_each_vcpu(idx, vcpu, kvm) {
|
||||
ippn = vcpu->arch.aia_context.imsic_addr >>
|
||||
IMSIC_MMIO_PAGE_SHIFT;
|
||||
if (ippn == tppn) {
|
||||
toff = target & (IMSIC_MMIO_PAGE_SZ - 1);
|
||||
return kvm_riscv_vcpu_aia_imsic_inject(vcpu, g,
|
||||
toff, iid);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level)
|
||||
{
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(kvm))
|
||||
return -EBUSY;
|
||||
|
||||
/* Inject interrupt level change in APLIC */
|
||||
return kvm_riscv_aia_aplic_inject(kvm, irq, level);
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_init_vm(struct kvm *kvm)
|
||||
{
|
||||
struct kvm_aia *aia = &kvm->arch.aia;
|
||||
|
||||
if (!kvm_riscv_aia_available())
|
||||
return;
|
||||
|
||||
/*
|
||||
* We don't do any memory allocations over here because these
|
||||
* will be done after AIA device is initialized by the user-space.
|
||||
*
|
||||
* Refer, aia_init() implementation for more details.
|
||||
*/
|
||||
|
||||
/* Initialize default values in AIA global context */
|
||||
aia->mode = (kvm_riscv_aia_nr_hgei) ?
|
||||
KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL;
|
||||
aia->nr_ids = kvm_riscv_aia_max_ids - 1;
|
||||
aia->nr_sources = 0;
|
||||
aia->nr_group_bits = 0;
|
||||
aia->nr_group_shift = KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN;
|
||||
aia->nr_hart_bits = 0;
|
||||
aia->nr_guest_bits = 0;
|
||||
aia->aplic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
|
||||
}
|
||||
|
||||
void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
|
||||
{
|
||||
/* Proceed only if AIA was initialized successfully */
|
||||
if (!kvm_riscv_aia_initialized(kvm))
|
||||
return;
|
||||
|
||||
/* Cleanup APLIC context */
|
||||
kvm_riscv_aia_aplic_cleanup(kvm);
|
||||
}
|
1084
arch/riscv/kvm/aia_imsic.c
Normal file
1084
arch/riscv/kvm/aia_imsic.c
Normal file
File diff suppressed because it is too large
Load diff
|
@ -116,7 +116,8 @@ static int __init riscv_kvm_init(void)
|
|||
kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits());
|
||||
|
||||
if (kvm_riscv_aia_available())
|
||||
kvm_info("AIA available\n");
|
||||
kvm_info("AIA available with %d guest external interrupts\n",
|
||||
kvm_riscv_aia_nr_hgei);
|
||||
|
||||
rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
|
||||
if (rc) {
|
||||
|
|
|
@ -296,7 +296,7 @@ static void make_xfence_request(struct kvm *kvm,
|
|||
unsigned int actual_req = req;
|
||||
DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
|
||||
|
||||
bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS);
|
||||
bitmap_zero(vcpu_mask, KVM_MAX_VCPUS);
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
if (hbase != -1UL) {
|
||||
if (vcpu->vcpu_id < hbase)
|
||||
|
|
|
@ -61,6 +61,7 @@ static const unsigned long kvm_isa_ext_arr[] = {
|
|||
KVM_ISA_EXT_ARR(SSAIA),
|
||||
KVM_ISA_EXT_ARR(SSTC),
|
||||
KVM_ISA_EXT_ARR(SVINVAL),
|
||||
KVM_ISA_EXT_ARR(SVNAPOT),
|
||||
KVM_ISA_EXT_ARR(SVPBMT),
|
||||
KVM_ISA_EXT_ARR(ZBB),
|
||||
KVM_ISA_EXT_ARR(ZIHINTPAUSE),
|
||||
|
@ -102,6 +103,7 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
|
|||
case KVM_RISCV_ISA_EXT_SSAIA:
|
||||
case KVM_RISCV_ISA_EXT_SSTC:
|
||||
case KVM_RISCV_ISA_EXT_SVINVAL:
|
||||
case KVM_RISCV_ISA_EXT_SVNAPOT:
|
||||
case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
|
||||
case KVM_RISCV_ISA_EXT_ZBB:
|
||||
return false;
|
||||
|
@ -250,10 +252,12 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
|
|||
|
||||
void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_riscv_aia_wakeon_hgei(vcpu, true);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_riscv_aia_wakeon_hgei(vcpu, false);
|
||||
}
|
||||
|
||||
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
|
||||
|
|
|
@ -183,6 +183,8 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
|
|||
run->exit_reason = KVM_EXIT_UNKNOWN;
|
||||
switch (trap->scause) {
|
||||
case EXC_INST_ILLEGAL:
|
||||
case EXC_LOAD_MISALIGNED:
|
||||
case EXC_STORE_MISALIGNED:
|
||||
if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) {
|
||||
kvm_riscv_vcpu_trap_redirect(vcpu, trap);
|
||||
ret = 1;
|
||||
|
|
|
@ -20,9 +20,7 @@ static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01 = {
|
|||
};
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_RISCV_PMU_SBI
|
||||
extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu;
|
||||
#else
|
||||
#ifndef CONFIG_RISCV_PMU_SBI
|
||||
static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu = {
|
||||
.extid_start = -1UL,
|
||||
.extid_end = -1UL,
|
||||
|
@ -31,49 +29,49 @@ static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_pmu = {
|
|||
#endif
|
||||
|
||||
struct kvm_riscv_sbi_extension_entry {
|
||||
enum KVM_RISCV_SBI_EXT_ID dis_idx;
|
||||
enum KVM_RISCV_SBI_EXT_ID ext_idx;
|
||||
const struct kvm_vcpu_sbi_extension *ext_ptr;
|
||||
};
|
||||
|
||||
static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = {
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_V01,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_V01,
|
||||
.ext_ptr = &vcpu_sbi_ext_v01,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_MAX, /* Can't be disabled */
|
||||
.ext_ptr = &vcpu_sbi_ext_base,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_TIME,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_TIME,
|
||||
.ext_ptr = &vcpu_sbi_ext_time,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_IPI,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_IPI,
|
||||
.ext_ptr = &vcpu_sbi_ext_ipi,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_RFENCE,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_RFENCE,
|
||||
.ext_ptr = &vcpu_sbi_ext_rfence,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_SRST,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_SRST,
|
||||
.ext_ptr = &vcpu_sbi_ext_srst,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_HSM,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_HSM,
|
||||
.ext_ptr = &vcpu_sbi_ext_hsm,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_PMU,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_PMU,
|
||||
.ext_ptr = &vcpu_sbi_ext_pmu,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL,
|
||||
.ext_ptr = &vcpu_sbi_ext_experimental,
|
||||
},
|
||||
{
|
||||
.dis_idx = KVM_RISCV_SBI_EXT_VENDOR,
|
||||
.ext_idx = KVM_RISCV_SBI_EXT_VENDOR,
|
||||
.ext_ptr = &vcpu_sbi_ext_vendor,
|
||||
},
|
||||
};
|
||||
|
@ -147,7 +145,7 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu,
|
|||
return -EINVAL;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
|
||||
if (sbi_ext[i].dis_idx == reg_num) {
|
||||
if (sbi_ext[i].ext_idx == reg_num) {
|
||||
sext = &sbi_ext[i];
|
||||
break;
|
||||
}
|
||||
|
@ -155,7 +153,15 @@ static int riscv_vcpu_set_sbi_ext_single(struct kvm_vcpu *vcpu,
|
|||
if (!sext)
|
||||
return -ENOENT;
|
||||
|
||||
scontext->extension_disabled[sext->dis_idx] = !reg_val;
|
||||
/*
|
||||
* We can't set the extension status to available here, since it may
|
||||
* have a probe() function which needs to confirm availability first,
|
||||
* but it may be too early to call that here. We can set the status to
|
||||
* unavailable, though.
|
||||
*/
|
||||
if (!reg_val)
|
||||
scontext->ext_status[sext->ext_idx] =
|
||||
KVM_RISCV_SBI_EXT_UNAVAILABLE;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -172,7 +178,7 @@ static int riscv_vcpu_get_sbi_ext_single(struct kvm_vcpu *vcpu,
|
|||
return -EINVAL;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
|
||||
if (sbi_ext[i].dis_idx == reg_num) {
|
||||
if (sbi_ext[i].ext_idx == reg_num) {
|
||||
sext = &sbi_ext[i];
|
||||
break;
|
||||
}
|
||||
|
@ -180,7 +186,15 @@ static int riscv_vcpu_get_sbi_ext_single(struct kvm_vcpu *vcpu,
|
|||
if (!sext)
|
||||
return -ENOENT;
|
||||
|
||||
*reg_val = !scontext->extension_disabled[sext->dis_idx];
|
||||
/*
|
||||
* If the extension status is still uninitialized, then we should probe
|
||||
* to determine if it's available, but it may be too early to do that
|
||||
* here. The best we can do is report that the extension has not been
|
||||
* disabled, i.e. we return 1 when the extension is available and also
|
||||
* when it only may be available.
|
||||
*/
|
||||
*reg_val = scontext->ext_status[sext->ext_idx] !=
|
||||
KVM_RISCV_SBI_EXT_UNAVAILABLE;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -307,18 +321,32 @@ int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu,
|
|||
const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext(
|
||||
struct kvm_vcpu *vcpu, unsigned long extid)
|
||||
{
|
||||
int i;
|
||||
const struct kvm_riscv_sbi_extension_entry *sext;
|
||||
struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context;
|
||||
const struct kvm_riscv_sbi_extension_entry *entry;
|
||||
const struct kvm_vcpu_sbi_extension *ext;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) {
|
||||
sext = &sbi_ext[i];
|
||||
if (sext->ext_ptr->extid_start <= extid &&
|
||||
sext->ext_ptr->extid_end >= extid) {
|
||||
if (sext->dis_idx < KVM_RISCV_SBI_EXT_MAX &&
|
||||
scontext->extension_disabled[sext->dis_idx])
|
||||
entry = &sbi_ext[i];
|
||||
ext = entry->ext_ptr;
|
||||
|
||||
if (ext->extid_start <= extid && ext->extid_end >= extid) {
|
||||
if (entry->ext_idx >= KVM_RISCV_SBI_EXT_MAX ||
|
||||
scontext->ext_status[entry->ext_idx] ==
|
||||
KVM_RISCV_SBI_EXT_AVAILABLE)
|
||||
return ext;
|
||||
if (scontext->ext_status[entry->ext_idx] ==
|
||||
KVM_RISCV_SBI_EXT_UNAVAILABLE)
|
||||
return NULL;
|
||||
return sbi_ext[i].ext_ptr;
|
||||
if (ext->probe && !ext->probe(vcpu)) {
|
||||
scontext->ext_status[entry->ext_idx] =
|
||||
KVM_RISCV_SBI_EXT_UNAVAILABLE;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
scontext->ext_status[entry->ext_idx] =
|
||||
KVM_RISCV_SBI_EXT_AVAILABLE;
|
||||
return ext;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -55,11 +55,129 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
|
|||
kvm_riscv_aia_destroy_vm(kvm);
|
||||
}
|
||||
|
||||
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irql,
|
||||
bool line_status)
|
||||
{
|
||||
if (!irqchip_in_kernel(kvm))
|
||||
return -ENXIO;
|
||||
|
||||
return kvm_riscv_aia_inject_irq(kvm, irql->irq, irql->level);
|
||||
}
|
||||
|
||||
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
|
||||
struct kvm *kvm, int irq_source_id,
|
||||
int level, bool line_status)
|
||||
{
|
||||
struct kvm_msi msi;
|
||||
|
||||
if (!level)
|
||||
return -1;
|
||||
|
||||
msi.address_lo = e->msi.address_lo;
|
||||
msi.address_hi = e->msi.address_hi;
|
||||
msi.data = e->msi.data;
|
||||
msi.flags = e->msi.flags;
|
||||
msi.devid = e->msi.devid;
|
||||
|
||||
return kvm_riscv_aia_inject_msi(kvm, &msi);
|
||||
}
|
||||
|
||||
static int kvm_riscv_set_irq(struct kvm_kernel_irq_routing_entry *e,
|
||||
struct kvm *kvm, int irq_source_id,
|
||||
int level, bool line_status)
|
||||
{
|
||||
return kvm_riscv_aia_inject_irq(kvm, e->irqchip.pin, level);
|
||||
}
|
||||
|
||||
int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines)
|
||||
{
|
||||
struct kvm_irq_routing_entry *ents;
|
||||
int i, rc;
|
||||
|
||||
ents = kcalloc(lines, sizeof(*ents), GFP_KERNEL);
|
||||
if (!ents)
|
||||
return -ENOMEM;
|
||||
|
||||
for (i = 0; i < lines; i++) {
|
||||
ents[i].gsi = i;
|
||||
ents[i].type = KVM_IRQ_ROUTING_IRQCHIP;
|
||||
ents[i].u.irqchip.irqchip = 0;
|
||||
ents[i].u.irqchip.pin = i;
|
||||
}
|
||||
rc = kvm_set_irq_routing(kvm, ents, lines, 0);
|
||||
kfree(ents);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
|
||||
{
|
||||
return irqchip_in_kernel(kvm);
|
||||
}
|
||||
|
||||
int kvm_set_routing_entry(struct kvm *kvm,
|
||||
struct kvm_kernel_irq_routing_entry *e,
|
||||
const struct kvm_irq_routing_entry *ue)
|
||||
{
|
||||
int r = -EINVAL;
|
||||
|
||||
switch (ue->type) {
|
||||
case KVM_IRQ_ROUTING_IRQCHIP:
|
||||
e->set = kvm_riscv_set_irq;
|
||||
e->irqchip.irqchip = ue->u.irqchip.irqchip;
|
||||
e->irqchip.pin = ue->u.irqchip.pin;
|
||||
if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) ||
|
||||
(e->irqchip.irqchip >= KVM_NR_IRQCHIPS))
|
||||
goto out;
|
||||
break;
|
||||
case KVM_IRQ_ROUTING_MSI:
|
||||
e->set = kvm_set_msi;
|
||||
e->msi.address_lo = ue->u.msi.address_lo;
|
||||
e->msi.address_hi = ue->u.msi.address_hi;
|
||||
e->msi.data = ue->u.msi.data;
|
||||
e->msi.flags = ue->flags;
|
||||
e->msi.devid = ue->u.msi.devid;
|
||||
break;
|
||||
default:
|
||||
goto out;
|
||||
}
|
||||
r = 0;
|
||||
out:
|
||||
return r;
|
||||
}
|
||||
|
||||
int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
|
||||
struct kvm *kvm, int irq_source_id, int level,
|
||||
bool line_status)
|
||||
{
|
||||
if (!level)
|
||||
return -EWOULDBLOCK;
|
||||
|
||||
switch (e->type) {
|
||||
case KVM_IRQ_ROUTING_MSI:
|
||||
return kvm_set_msi(e, kvm, irq_source_id, level, line_status);
|
||||
|
||||
case KVM_IRQ_ROUTING_IRQCHIP:
|
||||
return kvm_riscv_set_irq(e, kvm, irq_source_id,
|
||||
level, line_status);
|
||||
}
|
||||
|
||||
return -EWOULDBLOCK;
|
||||
}
|
||||
|
||||
bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
|
||||
{
|
||||
return irqchip_in_kernel(kvm);
|
||||
}
|
||||
|
||||
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
|
||||
{
|
||||
int r;
|
||||
|
||||
switch (ext) {
|
||||
case KVM_CAP_IRQCHIP:
|
||||
r = kvm_riscv_aia_available();
|
||||
break;
|
||||
case KVM_CAP_IOEVENTFD:
|
||||
case KVM_CAP_DEVICE_CTRL:
|
||||
case KVM_CAP_USER_MEMORY:
|
||||
|
|
|
@ -1442,6 +1442,8 @@ enum kvm_device_type {
|
|||
#define KVM_DEV_TYPE_XIVE KVM_DEV_TYPE_XIVE
|
||||
KVM_DEV_TYPE_ARM_PV_TIME,
|
||||
#define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME
|
||||
KVM_DEV_TYPE_RISCV_AIA,
|
||||
#define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA
|
||||
KVM_DEV_TYPE_MAX,
|
||||
};
|
||||
|
||||
|
|
Loading…
Reference in a new issue