mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-10-05 16:37:50 +00:00
Code Issues Releases Wiki Activity

Merge branch kvm-arm64/nv-6.8-prefix into kvmarm-master/next

Browse source 
* kvm-arm64/nv-6.8-prefix:
  : .
  : Nested Virtualization support update, focussing on the
  : NV2 support (VNCR mapping and such).
  : .
  KVM: arm64: nv: Handle virtual EL2 registers in vcpu_read/write_sys_reg()
  KVM: arm64: nv: Map VNCR-capable registers to a separate page
  KVM: arm64: nv: Add EL2_REG_VNCR()/EL2_REG_REDIR() sysreg helpers
  KVM: arm64: Introduce a bad_trap() primitive for unexpected trap handling
  KVM: arm64: nv: Add include containing the VNCR_EL2 offsets
  KVM: arm64: nv: Add non-VHE-EL2->EL1 translation helpers
  KVM: arm64: nv: Drop EL12 register traps that are redirected to VNCR
  KVM: arm64: nv: Compute NV view of idregs as a one-off
  KVM: arm64: nv: Hoist vcpu_has_nv() into is_hyp_ctxt()
  arm64: cpufeatures: Restrict NV support to FEAT_NV2

Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier 2023-12-19 10:06:58 +00:00
commit d016264d07
9 changed files with 457 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
arch/arm64/include/asm/kvm_emulate.h
View file

@ -17,6 +17,7 @@
#include <asm/esr.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_nested.h>
#include <asm/ptrace.h>
#include <asm/cputype.h>
#include <asm/virt.h>
@ -54,11 +55,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
static inline bool vcpu_has_feature(const struct kvm_vcpu *vcpu, int feature)
{
return test_bit(feature, vcpu->kvm->arch.vcpu_features);
}
#if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
{
@ -248,7 +244,7 @@ static inline bool __is_hyp_ctxt(const struct kvm_cpu_context *ctxt)
static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
{
return __is_hyp_ctxt(&vcpu->arch.ctxt);
return vcpu_has_nv(vcpu) && __is_hyp_ctxt(&vcpu->arch.ctxt);
}
/*

139
arch/arm64/include/asm/kvm_host.h
View file

@ -27,6 +27,7 @@
#include <asm/fpsimd.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/vncr_mapping.h>
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
@ -306,6 +307,7 @@ struct kvm_arch {
* Atomic access to multiple idregs are guarded by kvm_arch.config_lock.
*/
#define IDREG_IDX(id) (((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id))
#define IDX_IDREG(idx) sys_reg(3, 0, 0, ((idx) >> 3) + 1, (idx) & Op2_mask)
#define IDREG(kvm, id) ((kvm)->arch.id_regs[IDREG_IDX(id)])
#define KVM_ARM_ID_REG_NUM (IDREG_IDX(sys_reg(3, 0, 0, 7, 7)) + 1)
u64 id_regs[KVM_ARM_ID_REG_NUM];
@ -324,33 +326,33 @@ struct kvm_vcpu_fault_info {
u64 disr_el1; /* Deferred [SError] Status Register */
};
/*
* VNCR() just places the VNCR_capable registers in the enum after
* __VNCR_START__, and the value (after correction) to be an 8-byte offset
* from the VNCR base. As we don't require the enum to be otherwise ordered,
* we need the terrible hack below to ensure that we correctly size the
* sys_regs array, no matter what.
*
* The __MAX__ macro has been lifted from Sean Eron Anderson's wonderful
* treasure trove of bit hacks:
* https://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
*/
#define __MAX__(x,y) ((x) ^ (((x) ^ (y)) & -((x) < (y))))
#define VNCR(r) \
__before_##r, \
r = __VNCR_START__ + ((VNCR_ ## r) / 8), \
__after_##r = __MAX__(__before_##r - 1, r)
enum vcpu_sysreg {
__INVALID_SYSREG__, /* 0 is reserved as an invalid value */
MPIDR_EL1, /* MultiProcessor Affinity Register */
CLIDR_EL1, /* Cache Level ID Register */
CSSELR_EL1, /* Cache Size Selection Register */
SCTLR_EL1, /* System Control Register */
ACTLR_EL1, /* Auxiliary Control Register */
CPACR_EL1, /* Coprocessor Access Control */
ZCR_EL1, /* SVE Control */
TTBR0_EL1, /* Translation Table Base Register 0 */
TTBR1_EL1, /* Translation Table Base Register 1 */
TCR_EL1, /* Translation Control Register */
TCR2_EL1, /* Extended Translation Control Register */
ESR_EL1, /* Exception Syndrome Register */
AFSR0_EL1, /* Auxiliary Fault Status Register 0 */
AFSR1_EL1, /* Auxiliary Fault Status Register 1 */
FAR_EL1, /* Fault Address Register */
MAIR_EL1, /* Memory Attribute Indirection Register */
VBAR_EL1, /* Vector Base Address Register */
CONTEXTIDR_EL1, /* Context ID Register */
TPIDR_EL0, /* Thread ID, User R/W */
TPIDRRO_EL0, /* Thread ID, User R/O */
TPIDR_EL1, /* Thread ID, Privileged */
AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
CNTKCTL_EL1, /* Timer Control Register (EL1) */
PAR_EL1, /* Physical Address Register */
MDSCR_EL1, /* Monitor Debug System Control Register */
MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
OSLSR_EL1, /* OS Lock Status Register */
DISR_EL1, /* Deferred Interrupt Status Register */
@ -381,26 +383,11 @@ enum vcpu_sysreg {
APGAKEYLO_EL1,
APGAKEYHI_EL1,
ELR_EL1,
SP_EL1,
SPSR_EL1,
CNTVOFF_EL2,
CNTV_CVAL_EL0,
CNTV_CTL_EL0,
CNTP_CVAL_EL0,
CNTP_CTL_EL0,
/* Memory Tagging Extension registers */
RGSR_EL1, /* Random Allocation Tag Seed Register */
GCR_EL1, /* Tag Control Register */
TFSR_EL1, /* Tag Fault Status Register (EL1) */
TFSRE0_EL1, /* Tag Fault Status Register (EL0) */
/* Permission Indirection Extension registers */
PIR_EL1, /* Permission Indirection Register 1 (EL1) */
PIRE0_EL1, /* Permission Indirection Register 0 (EL1) */
/* 32bit specific registers. */
DACR32_EL2, /* Domain Access Control Register */
IFSR32_EL2, /* Instruction Fault Status Register */
@ -408,21 +395,14 @@ enum vcpu_sysreg {
DBGVCR32_EL2, /* Debug Vector Catch Register */
/* EL2 registers */
VPIDR_EL2, /* Virtualization Processor ID Register */
VMPIDR_EL2, /* Virtualization Multiprocessor ID Register */
SCTLR_EL2, /* System Control Register (EL2) */
ACTLR_EL2, /* Auxiliary Control Register (EL2) */
HCR_EL2, /* Hypervisor Configuration Register */
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
CPTR_EL2, /* Architectural Feature Trap Register (EL2) */
HSTR_EL2, /* Hypervisor System Trap Register */
HACR_EL2, /* Hypervisor Auxiliary Control Register */
HCRX_EL2, /* Extended Hypervisor Configuration Register */
TTBR0_EL2, /* Translation Table Base Register 0 (EL2) */
TTBR1_EL2, /* Translation Table Base Register 1 (EL2) */
TCR_EL2, /* Translation Control Register (EL2) */
VTTBR_EL2, /* Virtualization Translation Table Base Register */
VTCR_EL2, /* Virtualization Translation Control Register */
SPSR_EL2, /* EL2 saved program status register */
ELR_EL2, /* EL2 exception link register */
AFSR0_EL2, /* Auxiliary Fault Status Register 0 (EL2) */
@ -435,20 +415,62 @@ enum vcpu_sysreg {
VBAR_EL2, /* Vector Base Address Register (EL2) */
RVBAR_EL2, /* Reset Vector Base Address Register */
CONTEXTIDR_EL2, /* Context ID Register (EL2) */
TPIDR_EL2, /* EL2 Software Thread ID Register */
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
SP_EL2, /* EL2 Stack Pointer */
HFGRTR_EL2,
HFGWTR_EL2,
HFGITR_EL2,
HDFGRTR_EL2,
HDFGWTR_EL2,
HAFGRTR_EL2,
CNTHP_CTL_EL2,
CNTHP_CVAL_EL2,
CNTHV_CTL_EL2,
CNTHV_CVAL_EL2,
__VNCR_START__, /* Any VNCR-capable reg goes after this point */
VNCR(SCTLR_EL1),/* System Control Register */
VNCR(ACTLR_EL1),/* Auxiliary Control Register */
VNCR(CPACR_EL1),/* Coprocessor Access Control */
VNCR(ZCR_EL1), /* SVE Control */
VNCR(TTBR0_EL1),/* Translation Table Base Register 0 */
VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */
VNCR(TCR_EL1), /* Translation Control Register */
VNCR(TCR2_EL1), /* Extended Translation Control Register */
VNCR(ESR_EL1), /* Exception Syndrome Register */
VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */
VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */
VNCR(FAR_EL1), /* Fault Address Register */
VNCR(MAIR_EL1), /* Memory Attribute Indirection Register */
VNCR(VBAR_EL1), /* Vector Base Address Register */
VNCR(CONTEXTIDR_EL1), /* Context ID Register */
VNCR(AMAIR_EL1),/* Aux Memory Attribute Indirection Register */
VNCR(MDSCR_EL1),/* Monitor Debug System Control Register */
VNCR(ELR_EL1),
VNCR(SP_EL1),
VNCR(SPSR_EL1),
VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */
VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */
VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */
VNCR(HCR_EL2), /* Hypervisor Configuration Register */
VNCR(HSTR_EL2), /* Hypervisor System Trap Register */
VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */
VNCR(VTCR_EL2), /* Virtualization Translation Control Register */
VNCR(TPIDR_EL2),/* EL2 Software Thread ID Register */
VNCR(HCRX_EL2), /* Extended Hypervisor Configuration Register */
/* Permission Indirection Extension registers */
VNCR(PIR_EL1), /* Permission Indirection Register 1 (EL1) */
VNCR(PIRE0_EL1), /* Permission Indirection Register 0 (EL1) */
VNCR(HFGRTR_EL2),
VNCR(HFGWTR_EL2),
VNCR(HFGITR_EL2),
VNCR(HDFGRTR_EL2),
VNCR(HDFGWTR_EL2),
VNCR(HAFGRTR_EL2),
VNCR(CNTVOFF_EL2),
VNCR(CNTV_CVAL_EL0),
VNCR(CNTV_CTL_EL0),
VNCR(CNTP_CVAL_EL0),
VNCR(CNTP_CTL_EL0),
NR_SYS_REGS /* Nothing after this line! */
};
@ -465,6 +487,9 @@ struct kvm_cpu_context {
u64 sys_regs[NR_SYS_REGS];
struct kvm_vcpu *__hyp_running_vcpu;
/* This pointer has to be 4kB aligned. */
u64 *vncr_array;
};
struct kvm_host_data {
@ -827,8 +852,19 @@ struct kvm_vcpu_arch {
* accessed by a running VCPU. For example, for userspace access or
* for system registers that are never context switched, but only
* emulated.
*
* Don't bother with VNCR-based accesses in the nVHE code, it has no
* business dealing with NV.
*/
#define __ctxt_sys_reg(c,r) (&(c)->sys_regs[(r)])
static inline u64 *__ctxt_sys_reg(const struct kvm_cpu_context *ctxt, int r)
{
#if !defined (__KVM_NVHE_HYPERVISOR__)
if (unlikely(cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
r >= __VNCR_START__ && ctxt->vncr_array))
return &ctxt->vncr_array[r - __VNCR_START__];
#endif
return (u64 *)&ctxt->sys_regs[r];
}
#define ctxt_sys_reg(c,r) (*__ctxt_sys_reg(c,r))
@ -872,6 +908,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
case AMAIR_EL1: *val = read_sysreg_s(SYS_AMAIR_EL12); break;
case CNTKCTL_EL1: *val = read_sysreg_s(SYS_CNTKCTL_EL12); break;
case ELR_EL1: *val = read_sysreg_s(SYS_ELR_EL12); break;
case SPSR_EL1: *val = read_sysreg_s(SYS_SPSR_EL12); break;
case PAR_EL1: *val = read_sysreg_par(); break;
case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break;
case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
@ -916,6 +953,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); break;
case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); break;
case ELR_EL1: write_sysreg_s(val, SYS_ELR_EL12); break;
case SPSR_EL1: write_sysreg_s(val, SYS_SPSR_EL12); break;
case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); break;
case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break;
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
@ -1178,6 +1216,13 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
#define kvm_vm_has_ran_once(kvm) \
(test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &(kvm)->arch.flags))
static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature)
{
return test_bit(feature, ka->vcpu_features);
}
#define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f))
int kvm_trng_call(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM
extern phys_addr_t hyp_mem_base;

56
arch/arm64/include/asm/kvm_nested.h
View file

@ -2,8 +2,9 @@
#ifndef __ARM64_KVM_NESTED_H
#define __ARM64_KVM_NESTED_H
#include <asm/kvm_emulate.h>
#include <linux/bitfield.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
{
@ -12,12 +13,55 @@ static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
}
/* Translation helpers from non-VHE EL2 to EL1 */
static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
{
return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
}
static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
{
return TCR_EPD1_MASK | /* disable TTBR1_EL1 */
((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
tcr_el2_ps_to_tcr_el1_ips(tcr) |
(tcr & TCR_EL2_TG0_MASK) |
(tcr & TCR_EL2_ORGN0_MASK) |
(tcr & TCR_EL2_IRGN0_MASK) |
(tcr & TCR_EL2_T0SZ_MASK);
}
static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
{
u64 cpacr_el1 = 0;
if (cptr_el2 & CPTR_EL2_TTA)
cpacr_el1 |= CPACR_ELx_TTA;
if (!(cptr_el2 & CPTR_EL2_TFP))
cpacr_el1 |= CPACR_ELx_FPEN;
if (!(cptr_el2 & CPTR_EL2_TZ))
cpacr_el1 |= CPACR_ELx_ZEN;
return cpacr_el1;
}
static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
{
/* Only preserve the minimal set of bits we support */
val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
val |= SCTLR_EL1_RES1;
return val;
}
static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
{
/* Clear the ASID field */
return ttbr0 & ~GENMASK_ULL(63, 48);
}
extern bool __check_nv_sr_forward(struct kvm_vcpu *vcpu);
struct sys_reg_params;
struct sys_reg_desc;
void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p,
const struct sys_reg_desc *r);
int kvm_init_nv_sysregs(struct kvm *kvm);
#endif /* __ARM64_KVM_NESTED_H */

103
arch/arm64/include/asm/vncr_mapping.h Normal file
View file

@ -0,0 +1,103 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* System register offsets in the VNCR page
* All offsets are *byte* displacements!
*/
#ifndef __ARM64_VNCR_MAPPING_H__
#define __ARM64_VNCR_MAPPING_H__
#define VNCR_VTTBR_EL2 0x020
#define VNCR_VTCR_EL2 0x040
#define VNCR_VMPIDR_EL2 0x050
#define VNCR_CNTVOFF_EL2 0x060
#define VNCR_HCR_EL2 0x078
#define VNCR_HSTR_EL2 0x080
#define VNCR_VPIDR_EL2 0x088
#define VNCR_TPIDR_EL2 0x090
#define VNCR_HCRX_EL2 0x0A0
#define VNCR_VNCR_EL2 0x0B0
#define VNCR_CPACR_EL1 0x100
#define VNCR_CONTEXTIDR_EL1 0x108
#define VNCR_SCTLR_EL1 0x110
#define VNCR_ACTLR_EL1 0x118
#define VNCR_TCR_EL1 0x120
#define VNCR_AFSR0_EL1 0x128
#define VNCR_AFSR1_EL1 0x130
#define VNCR_ESR_EL1 0x138
#define VNCR_MAIR_EL1 0x140
#define VNCR_AMAIR_EL1 0x148
#define VNCR_MDSCR_EL1 0x158
#define VNCR_SPSR_EL1 0x160
#define VNCR_CNTV_CVAL_EL0 0x168
#define VNCR_CNTV_CTL_EL0 0x170
#define VNCR_CNTP_CVAL_EL0 0x178
#define VNCR_CNTP_CTL_EL0 0x180
#define VNCR_SCXTNUM_EL1 0x188
#define VNCR_TFSR_EL1 0x190
#define VNCR_HFGRTR_EL2 0x1B8
#define VNCR_HFGWTR_EL2 0x1C0
#define VNCR_HFGITR_EL2 0x1C8
#define VNCR_HDFGRTR_EL2 0x1D0
#define VNCR_HDFGWTR_EL2 0x1D8
#define VNCR_ZCR_EL1 0x1E0
#define VNCR_HAFGRTR_EL2 0x1E8
#define VNCR_TTBR0_EL1 0x200
#define VNCR_TTBR1_EL1 0x210
#define VNCR_FAR_EL1 0x220
#define VNCR_ELR_EL1 0x230
#define VNCR_SP_EL1 0x240
#define VNCR_VBAR_EL1 0x250
#define VNCR_TCR2_EL1 0x270
#define VNCR_PIRE0_EL1 0x290
#define VNCR_PIRE0_EL2 0x298
#define VNCR_PIR_EL1 0x2A0
#define VNCR_ICH_LR0_EL2 0x400
#define VNCR_ICH_LR1_EL2 0x408
#define VNCR_ICH_LR2_EL2 0x410
#define VNCR_ICH_LR3_EL2 0x418
#define VNCR_ICH_LR4_EL2 0x420
#define VNCR_ICH_LR5_EL2 0x428
#define VNCR_ICH_LR6_EL2 0x430
#define VNCR_ICH_LR7_EL2 0x438
#define VNCR_ICH_LR8_EL2 0x440
#define VNCR_ICH_LR9_EL2 0x448
#define VNCR_ICH_LR10_EL2 0x450
#define VNCR_ICH_LR11_EL2 0x458
#define VNCR_ICH_LR12_EL2 0x460
#define VNCR_ICH_LR13_EL2 0x468
#define VNCR_ICH_LR14_EL2 0x470
#define VNCR_ICH_LR15_EL2 0x478
#define VNCR_ICH_AP0R0_EL2 0x480
#define VNCR_ICH_AP0R1_EL2 0x488
#define VNCR_ICH_AP0R2_EL2 0x490
#define VNCR_ICH_AP0R3_EL2 0x498
#define VNCR_ICH_AP1R0_EL2 0x4A0
#define VNCR_ICH_AP1R1_EL2 0x4A8
#define VNCR_ICH_AP1R2_EL2 0x4B0
#define VNCR_ICH_AP1R3_EL2 0x4B8
#define VNCR_ICH_HCR_EL2 0x4C0
#define VNCR_ICH_VMCR_EL2 0x4C8
#define VNCR_VDISR_EL2 0x500
#define VNCR_PMBLIMITR_EL1 0x800
#define VNCR_PMBPTR_EL1 0x810
#define VNCR_PMBSR_EL1 0x820
#define VNCR_PMSCR_EL1 0x828
#define VNCR_PMSEVFR_EL1 0x830
#define VNCR_PMSICR_EL1 0x838
#define VNCR_PMSIRR_EL1 0x840
#define VNCR_PMSLATFR_EL1 0x848
#define VNCR_TRFCR_EL1 0x880
#define VNCR_MPAM1_EL1 0x900
#define VNCR_MPAMHCR_EL2 0x930
#define VNCR_MPAMVPMV_EL2 0x938
#define VNCR_MPAMVPM0_EL2 0x940
#define VNCR_MPAMVPM1_EL2 0x948
#define VNCR_MPAMVPM2_EL2 0x950
#define VNCR_MPAMVPM3_EL2 0x958
#define VNCR_MPAMVPM4_EL2 0x960
#define VNCR_MPAMVPM5_EL2 0x968
#define VNCR_MPAMVPM6_EL2 0x970
#define VNCR_MPAMVPM7_EL2 0x978
#endif /* __ARM64_VNCR_MAPPING_H__ */

2
arch/arm64/kernel/cpufeature.c
View file

@ -2372,7 +2372,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.capability = ARM64_HAS_NESTED_VIRT,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_nested_virt_support,
ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, IMP)
ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, NV2)
},
{
.capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,

3
arch/arm64/kvm/arch_timer.c
View file

@ -295,8 +295,7 @@ static u64 wfit_delay_ns(struct kvm_vcpu *vcpu)
u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
struct arch_timer_context *ctx;
ctx = (vcpu_has_nv(vcpu) && is_hyp_ctxt(vcpu)) ? vcpu_hvtimer(vcpu)
: vcpu_vtimer(vcpu);
ctx = is_hyp_ctxt(vcpu) ? vcpu_hvtimer(vcpu) : vcpu_vtimer(vcpu);
return kvm_counter_compute_delta(ctx, val);
}

6
arch/arm64/kvm/arm.c
View file

@ -669,6 +669,12 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
return ret;
}
if (vcpu_has_nv(vcpu)) {
ret = kvm_init_nv_sysregs(vcpu->kvm);
if (ret)
return ret;
}
ret = kvm_timer_enable(vcpu);
if (ret)
return ret;

22
arch/arm64/kvm/nested.c
View file

@ -23,13 +23,9 @@
* This list should get updated as new features get added to the NV
* support, and new extension to the architecture.
*/
void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p,
const struct sys_reg_desc *r)
static u64 limit_nv_id_reg(u32 id, u64 val)
{
u32 id = reg_to_encoding(r);
u64 val, tmp;
val = p->regval;
u64 tmp;
switch (id) {
case SYS_ID_AA64ISAR0_EL1:
@ -158,5 +154,17 @@ void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p,
break;
}
p->regval = val;
return val;
}
int kvm_init_nv_sysregs(struct kvm *kvm)
{
mutex_lock(&kvm->arch.config_lock);
for (int i = 0; i < KVM_ARM_ID_REG_NUM; i++)
kvm->arch.id_regs[i] = limit_nv_id_reg(IDX_IDREG(i),
kvm->arch.id_regs[i]);
mutex_unlock(&kvm->arch.config_lock);
return 0;
}

236
arch/arm64/kvm/sys_regs.c
View file

@ -45,44 +45,170 @@ static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
u64 val);
static bool bad_trap(struct kvm_vcpu *vcpu,
struct sys_reg_params *params,
const struct sys_reg_desc *r,
const char *msg)
{
WARN_ONCE(1, "Unexpected %s\n", msg);
print_sys_reg_instr(params);
kvm_inject_undefined(vcpu);
return false;
}
static bool read_from_write_only(struct kvm_vcpu *vcpu,
struct sys_reg_params *params,
const struct sys_reg_desc *r)
{
WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
print_sys_reg_instr(params);
kvm_inject_undefined(vcpu);
return false;
return bad_trap(vcpu, params, r,
"sys_reg read to write-only register");
}
static bool write_to_read_only(struct kvm_vcpu *vcpu,
struct sys_reg_params *params,
const struct sys_reg_desc *r)
{
WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
print_sys_reg_instr(params);
kvm_inject_undefined(vcpu);
return false;
return bad_trap(vcpu, params, r,
"sys_reg write to read-only register");
}
#define PURE_EL2_SYSREG(el2) \
case el2: { \
*el1r = el2; \
return true; \
}
#define MAPPED_EL2_SYSREG(el2, el1, fn) \
case el2: { \
*xlate = fn; \
*el1r = el1; \
return true; \
}
static bool get_el2_to_el1_mapping(unsigned int reg,
unsigned int *el1r, u64 (**xlate)(u64))
{
switch (reg) {
PURE_EL2_SYSREG( VPIDR_EL2 );
PURE_EL2_SYSREG( VMPIDR_EL2 );
PURE_EL2_SYSREG( ACTLR_EL2 );
PURE_EL2_SYSREG( HCR_EL2 );
PURE_EL2_SYSREG( MDCR_EL2 );
PURE_EL2_SYSREG( HSTR_EL2 );
PURE_EL2_SYSREG( HACR_EL2 );
PURE_EL2_SYSREG( VTTBR_EL2 );
PURE_EL2_SYSREG( VTCR_EL2 );
PURE_EL2_SYSREG( RVBAR_EL2 );
PURE_EL2_SYSREG( TPIDR_EL2 );
PURE_EL2_SYSREG( HPFAR_EL2 );
PURE_EL2_SYSREG( CNTHCTL_EL2 );
MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
translate_sctlr_el2_to_sctlr_el1 );
MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
translate_cptr_el2_to_cpacr_el1 );
MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
translate_ttbr0_el2_to_ttbr0_el1 );
MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1,
translate_tcr_el2_to_tcr_el1 );
MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL );
MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL );
MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL );
MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL );
MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL );
MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL );
MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
default:
return false;
}
}
u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
{
u64 val = 0x8badf00d8badf00d;
u64 (*xlate)(u64) = NULL;
unsigned int el1r;
if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) &&
__vcpu_read_sys_reg_from_cpu(reg, &val))
if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
goto memory_read;
if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
if (!is_hyp_ctxt(vcpu))
goto memory_read;
/*
* If this register does not have an EL1 counterpart,
* then read the stored EL2 version.
*/
if (reg == el1r)
goto memory_read;
/*
* If we have a non-VHE guest and that the sysreg
* requires translation to be used at EL1, use the
* in-memory copy instead.
*/
if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
goto memory_read;
/* Get the current version of the EL1 counterpart. */
WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
return val;
}
/* EL1 register can't be on the CPU if the guest is in vEL2. */
if (unlikely(is_hyp_ctxt(vcpu)))
goto memory_read;
if (__vcpu_read_sys_reg_from_cpu(reg, &val))
return val;
memory_read:
return __vcpu_sys_reg(vcpu, reg);
}
void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
{
if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) &&
__vcpu_write_sys_reg_to_cpu(val, reg))
u64 (*xlate)(u64) = NULL;
unsigned int el1r;
if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
goto memory_write;
if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
if (!is_hyp_ctxt(vcpu))
goto memory_write;
/*
* Always store a copy of the write to memory to avoid having
* to reverse-translate virtual EL2 system registers for a
* non-VHE guest hypervisor.
*/
__vcpu_sys_reg(vcpu, reg) = val;
/* No EL1 counterpart? We're done here.? */
if (reg == el1r)
return;
if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
val = xlate(val);
/* Redirect this to the EL1 version of the register. */
WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
return;
}
/* EL1 register can't be on the CPU if the guest is in vEL2. */
if (unlikely(is_hyp_ctxt(vcpu)))
goto memory_write;
if (__vcpu_write_sys_reg_to_cpu(val, reg))
return;
__vcpu_sys_reg(vcpu, reg) = val;
memory_write:
__vcpu_sys_reg(vcpu, reg) = val;
}
/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
@ -1505,8 +1631,6 @@ static bool access_id_reg(struct kvm_vcpu *vcpu,
return write_to_read_only(vcpu, p, r);
p->regval = read_id_reg(vcpu, r);
if (vcpu_has_nv(vcpu))
access_nested_id_reg(vcpu, p, r);
return true;
}
@ -1885,6 +2009,32 @@ static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
return REG_HIDDEN;
}
static bool bad_vncr_trap(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
/*
* We really shouldn't be here, and this is likely the result
* of a misconfigured trap, as this register should target the
* VNCR page, and nothing else.
*/
return bad_trap(vcpu, p, r,
"trap of VNCR-backed register");
}
static bool bad_redir_trap(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
/*
* We really shouldn't be here, and this is likely the result
* of a misconfigured trap, as this register should target the
* corresponding EL1, and nothing else.
*/
return bad_trap(vcpu, p, r,
"trap of EL2 register redirected to EL1");
}
#define EL2_REG(name, acc, rst, v) { \
SYS_DESC(SYS_##name), \
.access = acc, \
@ -1894,6 +2044,9 @@ static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
.val = v, \
}
#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
/*
* EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when
* HCR_EL2.E2H==1, and only in the sysreg table for convenience of
@ -2508,33 +2661,33 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper,
.reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
EL2_REG(VPIDR_EL2, access_rw, reset_unknown, 0),
EL2_REG(VMPIDR_EL2, access_rw, reset_unknown, 0),
EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0),
EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
EL2_REG(HCR_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(HCR_EL2, reset_val, 0),
EL2_REG(MDCR_EL2, access_rw, reset_val, 0),
EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
EL2_REG(HSTR_EL2, access_rw, reset_val, 0),
EL2_REG(HFGRTR_EL2, access_rw, reset_val, 0),
EL2_REG(HFGWTR_EL2, access_rw, reset_val, 0),
EL2_REG(HFGITR_EL2, access_rw, reset_val, 0),
EL2_REG(HACR_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
EL2_REG_VNCR(HACR_EL2, reset_val, 0),
EL2_REG(HCRX_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),
EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),
EL2_REG(VTTBR_EL2, access_rw, reset_val, 0),
EL2_REG(VTCR_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
{ SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
EL2_REG(HDFGRTR_EL2, access_rw, reset_val, 0),
EL2_REG(HDFGWTR_EL2, access_rw, reset_val, 0),
EL2_REG(HAFGRTR_EL2, access_rw, reset_val, 0),
EL2_REG(SPSR_EL2, access_rw, reset_val, 0),
EL2_REG(ELR_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
EL2_REG_REDIR(ELR_EL2, reset_val, 0),
{ SYS_DESC(SYS_SP_EL1), access_sp_el1},
/* AArch32 SPSR_* are RES0 if trapped from a NV guest */
@ -2550,10 +2703,10 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
EL2_REG(ESR_EL2, access_rw, reset_val, 0),
EL2_REG_REDIR(ESR_EL2, reset_val, 0),
{ SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
EL2_REG(FAR_EL2, access_rw, reset_val, 0),
EL2_REG_REDIR(FAR_EL2, reset_val, 0),
EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),
EL2_REG(MAIR_EL2, access_rw, reset_val, 0),
@ -2566,24 +2719,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {
EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
EL2_REG(CNTVOFF_EL2, access_rw, reset_val, 0),
EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
EL12_REG(SCTLR, access_vm_reg, reset_val, 0x00C50078),
EL12_REG(CPACR, access_rw, reset_val, 0),
EL12_REG(TTBR0, access_vm_reg, reset_unknown, 0),
EL12_REG(TTBR1, access_vm_reg, reset_unknown, 0),
EL12_REG(TCR, access_vm_reg, reset_val, 0),
{ SYS_DESC(SYS_SPSR_EL12), access_spsr},
{ SYS_DESC(SYS_ELR_EL12), access_elr},
EL12_REG(AFSR0, access_vm_reg, reset_unknown, 0),
EL12_REG(AFSR1, access_vm_reg, reset_unknown, 0),
EL12_REG(ESR, access_vm_reg, reset_unknown, 0),
EL12_REG(FAR, access_vm_reg, reset_unknown, 0),
EL12_REG(MAIR, access_vm_reg, reset_unknown, 0),
EL12_REG(AMAIR, access_vm_reg, reset_amair_el1, 0),
EL12_REG(VBAR, access_rw, reset_val, 0),
EL12_REG(CONTEXTIDR, access_vm_reg, reset_val, 0),
EL12_REG(CNTKCTL, access_rw, reset_val, 0),
EL2_REG(SP_EL2, NULL, reset_unknown, 0),