mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-28 21:33:52 +00:00
Code Issues Releases Wiki Activity
linux-stable/arch/arm64/kvm/emulate-nested.c
Fuad Tabba 676f482354 KVM: arm64: Handle HAFGRTR_EL2 trapping in nested virt 
Add the encodings to fine grain trapping fields for HAFGRTR_EL2
and add the associated handling code in nested virt. Based on
DDI0601 2023-09. Add the missing field definitions as well,
both to generate the correct RES0 mask and to be able to toggle
their FGT bits.

Also add the code for handling FGT trapping, reading of the
register, to nested virt.

Reviewed-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Fuad Tabba <tabba@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20231214100158.2305400-10-tabba@google.com
2023-12-18 11:25:50 +00:00

2185 lines
80 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 - Linaro and Columbia University
* Author: Jintack Lim <jintack.lim@linaro.org>
*/
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_nested.h>
#include "hyp/include/hyp/adjust_pc.h"
#include "trace.h"
enum trap_behaviour {
BEHAVE_HANDLE_LOCALLY = 0,
BEHAVE_FORWARD_READ = BIT(0),
BEHAVE_FORWARD_WRITE = BIT(1),
BEHAVE_FORWARD_ANY = BEHAVE_FORWARD_READ | BEHAVE_FORWARD_WRITE,
};
struct trap_bits {
const enum vcpu_sysreg index;
const enum trap_behaviour behaviour;
const u64 value;
const u64 mask;
};
/* Coarse Grained Trap definitions */
enum cgt_group_id {
/* Indicates no coarse trap control */
__RESERVED__,
/*
* The first batch of IDs denote coarse trapping that are used
* on their own instead of being part of a combination of
* trap controls.
*/
CGT_HCR_TID1,
CGT_HCR_TID2,
CGT_HCR_TID3,
CGT_HCR_IMO,
CGT_HCR_FMO,
CGT_HCR_TIDCP,
CGT_HCR_TACR,
CGT_HCR_TSW,
CGT_HCR_TPC,
CGT_HCR_TPU,
CGT_HCR_TTLB,
CGT_HCR_TVM,
CGT_HCR_TDZ,
CGT_HCR_TRVM,
CGT_HCR_TLOR,
CGT_HCR_TERR,
CGT_HCR_APK,
CGT_HCR_NV,
CGT_HCR_NV_nNV2,
CGT_HCR_NV1_nNV2,
CGT_HCR_AT,
CGT_HCR_nFIEN,
CGT_HCR_TID4,
CGT_HCR_TICAB,
CGT_HCR_TOCU,
CGT_HCR_ENSCXT,
CGT_HCR_TTLBIS,
CGT_HCR_TTLBOS,
CGT_MDCR_TPMCR,
CGT_MDCR_TPM,
CGT_MDCR_TDE,
CGT_MDCR_TDA,
CGT_MDCR_TDOSA,
CGT_MDCR_TDRA,
CGT_MDCR_E2PB,
CGT_MDCR_TPMS,
CGT_MDCR_TTRF,
CGT_MDCR_E2TB,
CGT_MDCR_TDCC,
/*
* Anything after this point is a combination of coarse trap
* controls, which must all be evaluated to decide what to do.
*/
__MULTIPLE_CONTROL_BITS__,
CGT_HCR_IMO_FMO = __MULTIPLE_CONTROL_BITS__,
CGT_HCR_TID2_TID4,
CGT_HCR_TTLB_TTLBIS,
CGT_HCR_TTLB_TTLBOS,
CGT_HCR_TVM_TRVM,
CGT_HCR_TPU_TICAB,
CGT_HCR_TPU_TOCU,
CGT_HCR_NV1_nNV2_ENSCXT,
CGT_MDCR_TPM_TPMCR,
CGT_MDCR_TDE_TDA,
CGT_MDCR_TDE_TDOSA,
CGT_MDCR_TDE_TDRA,
CGT_MDCR_TDCC_TDE_TDA,
/*
* Anything after this point requires a callback evaluating a
* complex trap condition. Ugly stuff.
*/
__COMPLEX_CONDITIONS__,
CGT_CNTHCTL_EL1PCTEN = __COMPLEX_CONDITIONS__,
CGT_CNTHCTL_EL1PTEN,
/* Must be last */
__NR_CGT_GROUP_IDS__
};
static const struct trap_bits coarse_trap_bits[] = {
[CGT_HCR_TID1] = {
.index = HCR_EL2,
.value = HCR_TID1,
.mask = HCR_TID1,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_TID2] = {
.index = HCR_EL2,
.value = HCR_TID2,
.mask = HCR_TID2,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TID3] = {
.index = HCR_EL2,
.value = HCR_TID3,
.mask = HCR_TID3,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_IMO] = {
.index = HCR_EL2,
.value = HCR_IMO,
.mask = HCR_IMO,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_FMO] = {
.index = HCR_EL2,
.value = HCR_FMO,
.mask = HCR_FMO,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_TIDCP] = {
.index = HCR_EL2,
.value = HCR_TIDCP,
.mask = HCR_TIDCP,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TACR] = {
.index = HCR_EL2,
.value = HCR_TACR,
.mask = HCR_TACR,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TSW] = {
.index = HCR_EL2,
.value = HCR_TSW,
.mask = HCR_TSW,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */
.index = HCR_EL2,
.value = HCR_TPC,
.mask = HCR_TPC,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TPU] = {
.index = HCR_EL2,
.value = HCR_TPU,
.mask = HCR_TPU,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TTLB] = {
.index = HCR_EL2,
.value = HCR_TTLB,
.mask = HCR_TTLB,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TVM] = {
.index = HCR_EL2,
.value = HCR_TVM,
.mask = HCR_TVM,
.behaviour = BEHAVE_FORWARD_WRITE,
},
[CGT_HCR_TDZ] = {
.index = HCR_EL2,
.value = HCR_TDZ,
.mask = HCR_TDZ,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TRVM] = {
.index = HCR_EL2,
.value = HCR_TRVM,
.mask = HCR_TRVM,
.behaviour = BEHAVE_FORWARD_READ,
},
[CGT_HCR_TLOR] = {
.index = HCR_EL2,
.value = HCR_TLOR,
.mask = HCR_TLOR,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TERR] = {
.index = HCR_EL2,
.value = HCR_TERR,
.mask = HCR_TERR,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_APK] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_APK,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_NV] = {
.index = HCR_EL2,
.value = HCR_NV,
.mask = HCR_NV,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_NV_nNV2] = {
.index = HCR_EL2,
.value = HCR_NV,
.mask = HCR_NV | HCR_NV2,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_NV1_nNV2] = {
.index = HCR_EL2,
.value = HCR_NV | HCR_NV1,
.mask = HCR_NV | HCR_NV1 | HCR_NV2,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_AT] = {
.index = HCR_EL2,
.value = HCR_AT,
.mask = HCR_AT,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_nFIEN] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_FIEN,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TID4] = {
.index = HCR_EL2,
.value = HCR_TID4,
.mask = HCR_TID4,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TICAB] = {
.index = HCR_EL2,
.value = HCR_TICAB,
.mask = HCR_TICAB,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TOCU] = {
.index = HCR_EL2,
.value = HCR_TOCU,
.mask = HCR_TOCU,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_ENSCXT] = {
.index = HCR_EL2,
.value = 0,
.mask = HCR_ENSCXT,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TTLBIS] = {
.index = HCR_EL2,
.value = HCR_TTLBIS,
.mask = HCR_TTLBIS,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_HCR_TTLBOS] = {
.index = HCR_EL2,
.value = HCR_TTLBOS,
.mask = HCR_TTLBOS,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TPMCR] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPMCR,
.mask = MDCR_EL2_TPMCR,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TPM] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPM,
.mask = MDCR_EL2_TPM,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TDE] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDE,
.mask = MDCR_EL2_TDE,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TDA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDA,
.mask = MDCR_EL2_TDA,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TDOSA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDOSA,
.mask = MDCR_EL2_TDOSA,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TDRA] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDRA,
.mask = MDCR_EL2_TDRA,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_E2PB] = {
.index = MDCR_EL2,
.value = 0,
.mask = BIT(MDCR_EL2_E2PB_SHIFT),
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TPMS] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TPMS,
.mask = MDCR_EL2_TPMS,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TTRF] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TTRF,
.mask = MDCR_EL2_TTRF,
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_E2TB] = {
.index = MDCR_EL2,
.value = 0,
.mask = BIT(MDCR_EL2_E2TB_SHIFT),
.behaviour = BEHAVE_FORWARD_ANY,
},
[CGT_MDCR_TDCC] = {
.index = MDCR_EL2,
.value = MDCR_EL2_TDCC,
.mask = MDCR_EL2_TDCC,
.behaviour = BEHAVE_FORWARD_ANY,
},
};
#define MCB(id, ...) \
[id - __MULTIPLE_CONTROL_BITS__] = \
(const enum cgt_group_id[]){ \
__VA_ARGS__, __RESERVED__ \
}
static const enum cgt_group_id *coarse_control_combo[] = {
MCB(CGT_HCR_IMO_FMO, CGT_HCR_IMO, CGT_HCR_FMO),
MCB(CGT_HCR_TID2_TID4, CGT_HCR_TID2, CGT_HCR_TID4),
MCB(CGT_HCR_TTLB_TTLBIS, CGT_HCR_TTLB, CGT_HCR_TTLBIS),
MCB(CGT_HCR_TTLB_TTLBOS, CGT_HCR_TTLB, CGT_HCR_TTLBOS),
MCB(CGT_HCR_TVM_TRVM, CGT_HCR_TVM, CGT_HCR_TRVM),
MCB(CGT_HCR_TPU_TICAB, CGT_HCR_TPU, CGT_HCR_TICAB),
MCB(CGT_HCR_TPU_TOCU, CGT_HCR_TPU, CGT_HCR_TOCU),
MCB(CGT_HCR_NV1_nNV2_ENSCXT, CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
MCB(CGT_MDCR_TPM_TPMCR, CGT_MDCR_TPM, CGT_MDCR_TPMCR),
MCB(CGT_MDCR_TDE_TDA, CGT_MDCR_TDE, CGT_MDCR_TDA),
MCB(CGT_MDCR_TDE_TDOSA, CGT_MDCR_TDE, CGT_MDCR_TDOSA),
MCB(CGT_MDCR_TDE_TDRA, CGT_MDCR_TDE, CGT_MDCR_TDRA),
MCB(CGT_MDCR_TDCC_TDE_TDA, CGT_MDCR_TDCC, CGT_MDCR_TDE, CGT_MDCR_TDA),
};
typedef enum trap_behaviour (*complex_condition_check)(struct kvm_vcpu *);
/*
* Warning, maximum confusion ahead.
*
* When E2H=0, CNTHCTL_EL2[1:0] are defined as EL1PCEN:EL1PCTEN
* When E2H=1, CNTHCTL_EL2[11:10] are defined as EL1PTEN:EL1PCTEN
*
* Note the single letter difference? Yet, the bits have the same
* function despite a different layout and a different name.
*
* We don't try to reconcile this mess. We just use the E2H=0 bits
* to generate something that is in the E2H=1 format, and live with
* it. You're welcome.
*/
static u64 get_sanitized_cnthctl(struct kvm_vcpu *vcpu)
{
u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
if (!vcpu_el2_e2h_is_set(vcpu))
val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
return val & ((CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN) << 10);
}
static enum trap_behaviour check_cnthctl_el1pcten(struct kvm_vcpu *vcpu)
{
if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCTEN << 10))
return BEHAVE_HANDLE_LOCALLY;
return BEHAVE_FORWARD_ANY;
}
static enum trap_behaviour check_cnthctl_el1pten(struct kvm_vcpu *vcpu)
{
if (get_sanitized_cnthctl(vcpu) & (CNTHCTL_EL1PCEN << 10))
return BEHAVE_HANDLE_LOCALLY;
return BEHAVE_FORWARD_ANY;
}
#define CCC(id, fn) \
[id - __COMPLEX_CONDITIONS__] = fn
static const complex_condition_check ccc[] = {
CCC(CGT_CNTHCTL_EL1PCTEN, check_cnthctl_el1pcten),
CCC(CGT_CNTHCTL_EL1PTEN, check_cnthctl_el1pten),
};
/*
* Bit assignment for the trap controls. We use a 64bit word with the
* following layout for each trapped sysreg:
*
* [9:0] enum cgt_group_id (10 bits)
* [13:10] enum fgt_group_id (4 bits)
* [19:14] bit number in the FGT register (6 bits)
* [20] trap polarity (1 bit)
* [25:21] FG filter (5 bits)
* [62:26] Unused (37 bits)
* [63] RES0 - Must be zero, as lost on insertion in the xarray
*/
#define TC_CGT_BITS 10
#define TC_FGT_BITS 4
#define TC_FGF_BITS 5
union trap_config {
u64 val;
struct {
unsigned long cgt:TC_CGT_BITS; /* Coarse Grained Trap id */
unsigned long fgt:TC_FGT_BITS; /* Fine Grained Trap id */
unsigned long bit:6; /* Bit number */
unsigned long pol:1; /* Polarity */
unsigned long fgf:TC_FGF_BITS; /* Fine Grained Filter */
unsigned long unused:37; /* Unused, should be zero */
unsigned long mbz:1; /* Must Be Zero */
};
};
struct encoding_to_trap_config {
const u32 encoding;
const u32 end;
const union trap_config tc;
const unsigned int line;
};
#define SR_RANGE_TRAP(sr_start, sr_end, trap_id) \
{ \
.encoding = sr_start, \
.end = sr_end, \
.tc = { \
.cgt = trap_id, \
}, \
.line = __LINE__, \
}
#define SR_TRAP(sr, trap_id) SR_RANGE_TRAP(sr, sr, trap_id)
/*
* Map encoding to trap bits for exception reported with EC=0x18.
* These must only be evaluated when running a nested hypervisor, but
* that the current context is not a hypervisor context. When the
* trapped access matches one of the trap controls, the exception is
* re-injected in the nested hypervisor.
*/
static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
SR_TRAP(SYS_REVIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_AIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_SMIDR_EL1, CGT_HCR_TID1),
SR_TRAP(SYS_CTR_EL0, CGT_HCR_TID2),
SR_TRAP(SYS_CCSIDR_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CCSIDR2_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CLIDR_EL1, CGT_HCR_TID2_TID4),
SR_TRAP(SYS_CSSELR_EL1, CGT_HCR_TID2_TID4),
SR_RANGE_TRAP(SYS_ID_PFR0_EL1,
sys_reg(3, 0, 0, 7, 7), CGT_HCR_TID3),
SR_TRAP(SYS_ICC_SGI0R_EL1, CGT_HCR_IMO_FMO),
SR_TRAP(SYS_ICC_ASGI1R_EL1, CGT_HCR_IMO_FMO),
SR_TRAP(SYS_ICC_SGI1R_EL1, CGT_HCR_IMO_FMO),
SR_RANGE_TRAP(sys_reg(3, 0, 11, 0, 0),
sys_reg(3, 0, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 1, 11, 0, 0),
sys_reg(3, 1, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 2, 11, 0, 0),
sys_reg(3, 2, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 3, 11, 0, 0),
sys_reg(3, 3, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 4, 11, 0, 0),
sys_reg(3, 4, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 5, 11, 0, 0),
sys_reg(3, 5, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 6, 11, 0, 0),
sys_reg(3, 6, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 7, 11, 0, 0),
sys_reg(3, 7, 11, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 0, 15, 0, 0),
sys_reg(3, 0, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 1, 15, 0, 0),
sys_reg(3, 1, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 2, 15, 0, 0),
sys_reg(3, 2, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 3, 15, 0, 0),
sys_reg(3, 3, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 4, 15, 0, 0),
sys_reg(3, 4, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 5, 15, 0, 0),
sys_reg(3, 5, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 6, 15, 0, 0),
sys_reg(3, 6, 15, 15, 7), CGT_HCR_TIDCP),
SR_RANGE_TRAP(sys_reg(3, 7, 15, 0, 0),
sys_reg(3, 7, 15, 15, 7), CGT_HCR_TIDCP),
SR_TRAP(SYS_ACTLR_EL1, CGT_HCR_TACR),
SR_TRAP(SYS_DC_ISW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CISW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_IGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_IGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIGSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIGDSW, CGT_HCR_TSW),
SR_TRAP(SYS_DC_CIVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CVADP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CIGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CIGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_IGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVAC, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVAP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGVADP, CGT_HCR_TPC),
SR_TRAP(SYS_DC_CGDVADP, CGT_HCR_TPC),
SR_TRAP(SYS_IC_IVAU, CGT_HCR_TPU_TOCU),
SR_TRAP(SYS_IC_IALLU, CGT_HCR_TPU_TOCU),
SR_TRAP(SYS_IC_IALLUIS, CGT_HCR_TPU_TICAB),
SR_TRAP(SYS_DC_CVAU, CGT_HCR_TPU_TOCU),
SR_TRAP(OP_TLBI_RVAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VMALLE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_ASIDE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAAE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAALE1, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VMALLE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_ASIDE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAAE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_VAALE1NXS, CGT_HCR_TTLB),
SR_TRAP(OP_TLBI_RVAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_ASIDE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAAE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAALE1IS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_RVAALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_ASIDE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAAE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VAALE1ISNXS, CGT_HCR_TTLB_TTLBIS),
SR_TRAP(OP_TLBI_VMALLE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_ASIDE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAAE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VMALLE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_ASIDE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_VAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAAE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(SYS_SCTLR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TCR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_ESR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_FAR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AFSR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AFSR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_MAIR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_AMAIR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_CONTEXTIDR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_DC_ZVA, CGT_HCR_TDZ),
SR_TRAP(SYS_DC_GVA, CGT_HCR_TDZ),
SR_TRAP(SYS_DC_GZVA, CGT_HCR_TDZ),
SR_TRAP(SYS_LORSA_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LOREA_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORN_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORC_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_LORID_EL1, CGT_HCR_TLOR),
SR_TRAP(SYS_ERRIDR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERRSELR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXADDR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXCTLR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXFR_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC0_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC1_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC2_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXMISC3_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_ERXSTATUS_EL1, CGT_HCR_TERR),
SR_TRAP(SYS_APIAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIAKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIBKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APIBKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDAKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDBKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APDBKEYHI_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APGAKEYLO_EL1, CGT_HCR_APK),
SR_TRAP(SYS_APGAKEYHI_EL1, CGT_HCR_APK),
/* All _EL2 registers */
SR_TRAP(SYS_BRBCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VMPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCTLR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ACTLR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCTLR2_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_HCR_EL2,
SYS_HCRX_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SMPRIMAP_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SMCR_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_TTBR0_EL2,
SYS_TCR2_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VTTBR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VTCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VNCR_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_HDFGRTR_EL2,
SYS_HAFGRTR_EL2, CGT_HCR_NV),
/* Skip the SP_EL1 encoding... */
SR_TRAP(SYS_SPSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ELR_EL2, CGT_HCR_NV),
/* Skip SPSR_irq, SPSR_abt, SPSR_und, SPSR_fiq */
SR_TRAP(SYS_AFSR0_EL2, CGT_HCR_NV),
SR_TRAP(SYS_AFSR1_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ESR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VSESR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_TFSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_FAR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_HPFAR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_PMSCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MAIR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_AMAIR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAMHCR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAMVPMV_EL2, CGT_HCR_NV),
SR_TRAP(SYS_MPAM2_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_MPAMVPM0_EL2,
SYS_MPAMVPM7_EL2, CGT_HCR_NV),
/*
* Note that the spec. describes a group of MEC registers
* whose access should not trap, therefore skip the following:
* MECID_A0_EL2, MECID_A1_EL2, MECID_P0_EL2,
* MECID_P1_EL2, MECIDR_EL2, VMECID_A_EL2,
* VMECID_P_EL2.
*/
SR_RANGE_TRAP(SYS_VBAR_EL2,
SYS_RMR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_VDISR_EL2, CGT_HCR_NV),
/* ICH_AP0R<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_AP0R0_EL2,
SYS_ICH_AP0R3_EL2, CGT_HCR_NV),
/* ICH_AP1R<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_AP1R0_EL2,
SYS_ICH_AP1R3_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICC_SRE_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_ICH_HCR_EL2,
SYS_ICH_EISR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICH_ELRSR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_ICH_VMCR_EL2, CGT_HCR_NV),
/* ICH_LR<m>_EL2 */
SR_RANGE_TRAP(SYS_ICH_LR0_EL2,
SYS_ICH_LR15_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CONTEXTIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_TPIDR_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SCXTNUM_EL2, CGT_HCR_NV),
/* AMEVCNTVOFF0<n>_EL2, AMEVCNTVOFF1<n>_EL2 */
SR_RANGE_TRAP(SYS_AMEVCNTVOFF0n_EL2(0),
SYS_AMEVCNTVOFF1n_EL2(15), CGT_HCR_NV),
/* CNT*_EL2 */
SR_TRAP(SYS_CNTVOFF_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CNTPOFF_EL2, CGT_HCR_NV),
SR_TRAP(SYS_CNTHCTL_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_CNTHP_TVAL_EL2,
SYS_CNTHP_CVAL_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_CNTHV_TVAL_EL2,
SYS_CNTHV_CVAL_EL2, CGT_HCR_NV),
/* All _EL02, _EL12 registers */
SR_RANGE_TRAP(sys_reg(3, 5, 0, 0, 0),
sys_reg(3, 5, 10, 15, 7), CGT_HCR_NV),
SR_RANGE_TRAP(sys_reg(3, 5, 12, 0, 0),
sys_reg(3, 5, 14, 15, 7), CGT_HCR_NV),
SR_TRAP(OP_AT_S1E2R, CGT_HCR_NV),
SR_TRAP(OP_AT_S1E2W, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E1R, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E1W, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E0R, CGT_HCR_NV),
SR_TRAP(OP_AT_S12E0W, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1NXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1IS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2ISNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1ISNXS,CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2OS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VAE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_ALLE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VALE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_VMALLS12E1OSNXS,CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2E1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2E1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_IPAS2LE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RIPAS2LE1OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVAE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_TLBI_RVALE2OSNXS, CGT_HCR_NV),
SR_TRAP(OP_CPP_RCTX, CGT_HCR_NV),
SR_TRAP(OP_DVP_RCTX, CGT_HCR_NV),
SR_TRAP(OP_CFP_RCTX, CGT_HCR_NV),
SR_TRAP(SYS_SP_EL1, CGT_HCR_NV_nNV2),
SR_TRAP(SYS_VBAR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_ELR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_SPSR_EL1, CGT_HCR_NV1_nNV2),
SR_TRAP(SYS_SCXTNUM_EL1, CGT_HCR_NV1_nNV2_ENSCXT),
SR_TRAP(SYS_SCXTNUM_EL0, CGT_HCR_ENSCXT),
SR_TRAP(OP_AT_S1E1R, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1W, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E0R, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E0W, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1RP, CGT_HCR_AT),
SR_TRAP(OP_AT_S1E1WP, CGT_HCR_AT),
SR_TRAP(SYS_ERXPFGF_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_ERXPFGCTL_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_ERXPFGCDN_EL1, CGT_HCR_nFIEN),
SR_TRAP(SYS_PMCR_EL0, CGT_MDCR_TPM_TPMCR),
SR_TRAP(SYS_PMCNTENSET_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCNTENCLR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMOVSSET_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMOVSCLR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCEID0_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCEID1_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMXEVTYPER_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMSWINC_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMSELR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMXEVCNTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMCCNTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMUSERENR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_PMINTENSET_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMINTENCLR_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMMIR_EL1, CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(0), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(1), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(2), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(3), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(4), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(5), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(6), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(7), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(8), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(9), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(10), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(11), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(12), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(13), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(14), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(15), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(16), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(17), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(18), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(19), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(20), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(21), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(22), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(23), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(24), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(25), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(26), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(27), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(28), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(29), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVCNTRn_EL0(30), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(0), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(1), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(2), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(3), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(4), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(5), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(6), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(7), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(8), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(9), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(10), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(11), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(12), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(13), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(14), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(15), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(16), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(17), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(18), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(19), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(20), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(21), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(22), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(23), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(24), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(25), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(26), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(27), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(28), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(29), CGT_MDCR_TPM),
SR_TRAP(SYS_PMEVTYPERn_EL0(30), CGT_MDCR_TPM),
SR_TRAP(SYS_PMCCFILTR_EL0, CGT_MDCR_TPM),
SR_TRAP(SYS_MDCCSR_EL0, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_MDCCINT_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_OSDTRRX_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_OSDTRTX_EL1, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_DBGDTR_EL0, CGT_MDCR_TDCC_TDE_TDA),
/*
* Also covers DBGDTRRX_EL0, which has the same encoding as
* SYS_DBGDTRTX_EL0...
*/
SR_TRAP(SYS_DBGDTRTX_EL0, CGT_MDCR_TDCC_TDE_TDA),
SR_TRAP(SYS_MDSCR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_OSECCR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBVRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGBCRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWVRn_EL1(15), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(0), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(1), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(2), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(3), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(4), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(5), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(6), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(7), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(8), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(9), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(10), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(11), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(12), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(13), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGWCRn_EL1(14), CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGCLAIMSET_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGCLAIMCLR_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_DBGAUTHSTATUS_EL1, CGT_MDCR_TDE_TDA),
SR_TRAP(SYS_OSLAR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_OSLSR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_OSDLR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_DBGPRCR_EL1, CGT_MDCR_TDE_TDOSA),
SR_TRAP(SYS_MDRAR_EL1, CGT_MDCR_TDE_TDRA),
SR_TRAP(SYS_PMBLIMITR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMBPTR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMBSR_EL1, CGT_MDCR_E2PB),
SR_TRAP(SYS_PMSCR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSEVFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSFCR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSICR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSIDR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSIRR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSLATFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_PMSNEVFR_EL1, CGT_MDCR_TPMS),
SR_TRAP(SYS_TRFCR_EL1, CGT_MDCR_TTRF),
SR_TRAP(SYS_TRBBASER_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBLIMITR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBMAR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBPTR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBSR_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_TRBTRG_EL1, CGT_MDCR_E2TB),
SR_TRAP(SYS_CNTP_TVAL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTP_CVAL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTP_CTL_EL0, CGT_CNTHCTL_EL1PTEN),
SR_TRAP(SYS_CNTPCT_EL0, CGT_CNTHCTL_EL1PCTEN),
SR_TRAP(SYS_CNTPCTSS_EL0, CGT_CNTHCTL_EL1PCTEN),
};
static DEFINE_XARRAY(sr_forward_xa);
enum fgt_group_id {
__NO_FGT_GROUP__,
HFGxTR_GROUP,
HDFGRTR_GROUP,
HDFGWTR_GROUP,
HFGITR_GROUP,
HAFGRTR_GROUP,
/* Must be last */
__NR_FGT_GROUP_IDS__
};
enum fg_filter_id {
__NO_FGF__,
HCRX_FGTnXS,
/* Must be last */
__NR_FG_FILTER_IDS__
};
#define SR_FGF(sr, g, b, p, f) \
{ \
.encoding = sr, \
.end = sr, \
.tc = { \
.fgt = g ## _GROUP, \
.bit = g ## _EL2_ ## b ## _SHIFT, \
.pol = p, \
.fgf = f, \
}, \
.line = __LINE__, \
}
#define SR_FGT(sr, g, b, p) SR_FGF(sr, g, b, p, __NO_FGF__)
static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
/* HFGRTR_EL2, HFGWTR_EL2 */
SR_FGT(SYS_AMAIR2_EL1, HFGxTR, nAMAIR2_EL1, 0),
SR_FGT(SYS_MAIR2_EL1, HFGxTR, nMAIR2_EL1, 0),
SR_FGT(SYS_S2POR_EL1, HFGxTR, nS2POR_EL1, 0),
SR_FGT(SYS_POR_EL1, HFGxTR, nPOR_EL1, 0),
SR_FGT(SYS_POR_EL0, HFGxTR, nPOR_EL0, 0),
SR_FGT(SYS_PIR_EL1, HFGxTR, nPIR_EL1, 0),
SR_FGT(SYS_PIRE0_EL1, HFGxTR, nPIRE0_EL1, 0),
SR_FGT(SYS_RCWMASK_EL1, HFGxTR, nRCWMASK_EL1, 0),
SR_FGT(SYS_TPIDR2_EL0, HFGxTR, nTPIDR2_EL0, 0),
SR_FGT(SYS_SMPRI_EL1, HFGxTR, nSMPRI_EL1, 0),
SR_FGT(SYS_GCSCR_EL1, HFGxTR, nGCS_EL1, 0),
SR_FGT(SYS_GCSPR_EL1, HFGxTR, nGCS_EL1, 0),
SR_FGT(SYS_GCSCRE0_EL1, HFGxTR, nGCS_EL0, 0),
SR_FGT(SYS_GCSPR_EL0, HFGxTR, nGCS_EL0, 0),
SR_FGT(SYS_ACCDATA_EL1, HFGxTR, nACCDATA_EL1, 0),
SR_FGT(SYS_ERXADDR_EL1, HFGxTR, ERXADDR_EL1, 1),
SR_FGT(SYS_ERXPFGCDN_EL1, HFGxTR, ERXPFGCDN_EL1, 1),
SR_FGT(SYS_ERXPFGCTL_EL1, HFGxTR, ERXPFGCTL_EL1, 1),
SR_FGT(SYS_ERXPFGF_EL1, HFGxTR, ERXPFGF_EL1, 1),
SR_FGT(SYS_ERXMISC0_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC1_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC2_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXMISC3_EL1, HFGxTR, ERXMISCn_EL1, 1),
SR_FGT(SYS_ERXSTATUS_EL1, HFGxTR, ERXSTATUS_EL1, 1),
SR_FGT(SYS_ERXCTLR_EL1, HFGxTR, ERXCTLR_EL1, 1),
SR_FGT(SYS_ERXFR_EL1, HFGxTR, ERXFR_EL1, 1),
SR_FGT(SYS_ERRSELR_EL1, HFGxTR, ERRSELR_EL1, 1),
SR_FGT(SYS_ERRIDR_EL1, HFGxTR, ERRIDR_EL1, 1),
SR_FGT(SYS_ICC_IGRPEN0_EL1, HFGxTR, ICC_IGRPENn_EL1, 1),
SR_FGT(SYS_ICC_IGRPEN1_EL1, HFGxTR, ICC_IGRPENn_EL1, 1),
SR_FGT(SYS_VBAR_EL1, HFGxTR, VBAR_EL1, 1),
SR_FGT(SYS_TTBR1_EL1, HFGxTR, TTBR1_EL1, 1),
SR_FGT(SYS_TTBR0_EL1, HFGxTR, TTBR0_EL1, 1),
SR_FGT(SYS_TPIDR_EL0, HFGxTR, TPIDR_EL0, 1),
SR_FGT(SYS_TPIDRRO_EL0, HFGxTR, TPIDRRO_EL0, 1),
SR_FGT(SYS_TPIDR_EL1, HFGxTR, TPIDR_EL1, 1),
SR_FGT(SYS_TCR_EL1, HFGxTR, TCR_EL1, 1),
SR_FGT(SYS_SCXTNUM_EL0, HFGxTR, SCXTNUM_EL0, 1),
SR_FGT(SYS_SCXTNUM_EL1, HFGxTR, SCXTNUM_EL1, 1),
SR_FGT(SYS_SCTLR_EL1, HFGxTR, SCTLR_EL1, 1),
SR_FGT(SYS_REVIDR_EL1, HFGxTR, REVIDR_EL1, 1),
SR_FGT(SYS_PAR_EL1, HFGxTR, PAR_EL1, 1),
SR_FGT(SYS_MPIDR_EL1, HFGxTR, MPIDR_EL1, 1),
SR_FGT(SYS_MIDR_EL1, HFGxTR, MIDR_EL1, 1),
SR_FGT(SYS_MAIR_EL1, HFGxTR, MAIR_EL1, 1),
SR_FGT(SYS_LORSA_EL1, HFGxTR, LORSA_EL1, 1),
SR_FGT(SYS_LORN_EL1, HFGxTR, LORN_EL1, 1),
SR_FGT(SYS_LORID_EL1, HFGxTR, LORID_EL1, 1),
SR_FGT(SYS_LOREA_EL1, HFGxTR, LOREA_EL1, 1),
SR_FGT(SYS_LORC_EL1, HFGxTR, LORC_EL1, 1),
SR_FGT(SYS_ISR_EL1, HFGxTR, ISR_EL1, 1),
SR_FGT(SYS_FAR_EL1, HFGxTR, FAR_EL1, 1),
SR_FGT(SYS_ESR_EL1, HFGxTR, ESR_EL1, 1),
SR_FGT(SYS_DCZID_EL0, HFGxTR, DCZID_EL0, 1),
SR_FGT(SYS_CTR_EL0, HFGxTR, CTR_EL0, 1),
SR_FGT(SYS_CSSELR_EL1, HFGxTR, CSSELR_EL1, 1),
SR_FGT(SYS_CPACR_EL1, HFGxTR, CPACR_EL1, 1),
SR_FGT(SYS_CONTEXTIDR_EL1, HFGxTR, CONTEXTIDR_EL1, 1),
SR_FGT(SYS_CLIDR_EL1, HFGxTR, CLIDR_EL1, 1),
SR_FGT(SYS_CCSIDR_EL1, HFGxTR, CCSIDR_EL1, 1),
SR_FGT(SYS_APIBKEYLO_EL1, HFGxTR, APIBKey, 1),
SR_FGT(SYS_APIBKEYHI_EL1, HFGxTR, APIBKey, 1),
SR_FGT(SYS_APIAKEYLO_EL1, HFGxTR, APIAKey, 1),
SR_FGT(SYS_APIAKEYHI_EL1, HFGxTR, APIAKey, 1),
SR_FGT(SYS_APGAKEYLO_EL1, HFGxTR, APGAKey, 1),
SR_FGT(SYS_APGAKEYHI_EL1, HFGxTR, APGAKey, 1),
SR_FGT(SYS_APDBKEYLO_EL1, HFGxTR, APDBKey, 1),
SR_FGT(SYS_APDBKEYHI_EL1, HFGxTR, APDBKey, 1),
SR_FGT(SYS_APDAKEYLO_EL1, HFGxTR, APDAKey, 1),
SR_FGT(SYS_APDAKEYHI_EL1, HFGxTR, APDAKey, 1),
SR_FGT(SYS_AMAIR_EL1, HFGxTR, AMAIR_EL1, 1),
SR_FGT(SYS_AIDR_EL1, HFGxTR, AIDR_EL1, 1),
SR_FGT(SYS_AFSR1_EL1, HFGxTR, AFSR1_EL1, 1),
SR_FGT(SYS_AFSR0_EL1, HFGxTR, AFSR0_EL1, 1),
/* HFGITR_EL2 */
SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1),
SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1),
SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0),
SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0),
SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0),
SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0),
SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0),
SR_FGT(SYS_DC_CVAC, HFGITR, DCCVAC, 1),
SR_FGT(SYS_DC_CGVAC, HFGITR, DCCVAC, 1),
SR_FGT(SYS_DC_CGDVAC, HFGITR, DCCVAC, 1),
SR_FGT(OP_CPP_RCTX, HFGITR, CPPRCTX, 1),
SR_FGT(OP_DVP_RCTX, HFGITR, DVPRCTX, 1),
SR_FGT(OP_CFP_RCTX, HFGITR, CFPRCTX, 1),
SR_FGT(OP_TLBI_VAALE1, HFGITR, TLBIVAALE1, 1),
SR_FGT(OP_TLBI_VALE1, HFGITR, TLBIVALE1, 1),
SR_FGT(OP_TLBI_VAAE1, HFGITR, TLBIVAAE1, 1),
SR_FGT(OP_TLBI_ASIDE1, HFGITR, TLBIASIDE1, 1),
SR_FGT(OP_TLBI_VAE1, HFGITR, TLBIVAE1, 1),
SR_FGT(OP_TLBI_VMALLE1, HFGITR, TLBIVMALLE1, 1),
SR_FGT(OP_TLBI_RVAALE1, HFGITR, TLBIRVAALE1, 1),
SR_FGT(OP_TLBI_RVALE1, HFGITR, TLBIRVALE1, 1),
SR_FGT(OP_TLBI_RVAAE1, HFGITR, TLBIRVAAE1, 1),
SR_FGT(OP_TLBI_RVAE1, HFGITR, TLBIRVAE1, 1),
SR_FGT(OP_TLBI_RVAALE1IS, HFGITR, TLBIRVAALE1IS, 1),
SR_FGT(OP_TLBI_RVALE1IS, HFGITR, TLBIRVALE1IS, 1),
SR_FGT(OP_TLBI_RVAAE1IS, HFGITR, TLBIRVAAE1IS, 1),
SR_FGT(OP_TLBI_RVAE1IS, HFGITR, TLBIRVAE1IS, 1),
SR_FGT(OP_TLBI_VAALE1IS, HFGITR, TLBIVAALE1IS, 1),
SR_FGT(OP_TLBI_VALE1IS, HFGITR, TLBIVALE1IS, 1),
SR_FGT(OP_TLBI_VAAE1IS, HFGITR, TLBIVAAE1IS, 1),
SR_FGT(OP_TLBI_ASIDE1IS, HFGITR, TLBIASIDE1IS, 1),
SR_FGT(OP_TLBI_VAE1IS, HFGITR, TLBIVAE1IS, 1),
SR_FGT(OP_TLBI_VMALLE1IS, HFGITR, TLBIVMALLE1IS, 1),
SR_FGT(OP_TLBI_RVAALE1OS, HFGITR, TLBIRVAALE1OS, 1),
SR_FGT(OP_TLBI_RVALE1OS, HFGITR, TLBIRVALE1OS, 1),
SR_FGT(OP_TLBI_RVAAE1OS, HFGITR, TLBIRVAAE1OS, 1),
SR_FGT(OP_TLBI_RVAE1OS, HFGITR, TLBIRVAE1OS, 1),
SR_FGT(OP_TLBI_VAALE1OS, HFGITR, TLBIVAALE1OS, 1),
SR_FGT(OP_TLBI_VALE1OS, HFGITR, TLBIVALE1OS, 1),
SR_FGT(OP_TLBI_VAAE1OS, HFGITR, TLBIVAAE1OS, 1),
SR_FGT(OP_TLBI_ASIDE1OS, HFGITR, TLBIASIDE1OS, 1),
SR_FGT(OP_TLBI_VAE1OS, HFGITR, TLBIVAE1OS, 1),
SR_FGT(OP_TLBI_VMALLE1OS, HFGITR, TLBIVMALLE1OS, 1),
/* nXS variants must be checked against HCRX_EL2.FGTnXS */
SR_FGF(OP_TLBI_VAALE1NXS, HFGITR, TLBIVAALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1NXS, HFGITR, TLBIVALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1NXS, HFGITR, TLBIVAAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1NXS, HFGITR, TLBIASIDE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1NXS, HFGITR, TLBIVAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1NXS, HFGITR, TLBIVMALLE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1NXS, HFGITR, TLBIRVAALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1NXS, HFGITR, TLBIRVALE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1NXS, HFGITR, TLBIRVAAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1NXS, HFGITR, TLBIRVAE1, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1ISNXS, HFGITR, TLBIRVAALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1ISNXS, HFGITR, TLBIRVALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1ISNXS, HFGITR, TLBIRVAAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1ISNXS, HFGITR, TLBIRVAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAALE1ISNXS, HFGITR, TLBIVAALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1ISNXS, HFGITR, TLBIVALE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1ISNXS, HFGITR, TLBIVAAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1ISNXS, HFGITR, TLBIASIDE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1ISNXS, HFGITR, TLBIVAE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1ISNXS, HFGITR, TLBIVMALLE1IS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAALE1OSNXS, HFGITR, TLBIRVAALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVALE1OSNXS, HFGITR, TLBIRVALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAAE1OSNXS, HFGITR, TLBIRVAAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_RVAE1OSNXS, HFGITR, TLBIRVAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAALE1OSNXS, HFGITR, TLBIVAALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VALE1OSNXS, HFGITR, TLBIVALE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAAE1OSNXS, HFGITR, TLBIVAAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_ASIDE1OSNXS, HFGITR, TLBIASIDE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VAE1OSNXS, HFGITR, TLBIVAE1OS, 1, HCRX_FGTnXS),
SR_FGF(OP_TLBI_VMALLE1OSNXS, HFGITR, TLBIVMALLE1OS, 1, HCRX_FGTnXS),
SR_FGT(OP_AT_S1E1WP, HFGITR, ATS1E1WP, 1),
SR_FGT(OP_AT_S1E1RP, HFGITR, ATS1E1RP, 1),
SR_FGT(OP_AT_S1E0W, HFGITR, ATS1E0W, 1),
SR_FGT(OP_AT_S1E0R, HFGITR, ATS1E0R, 1),
SR_FGT(OP_AT_S1E1W, HFGITR, ATS1E1W, 1),
SR_FGT(OP_AT_S1E1R, HFGITR, ATS1E1R, 1),
SR_FGT(SYS_DC_ZVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_GVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_GZVA, HFGITR, DCZVA, 1),
SR_FGT(SYS_DC_CIVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CIGVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CIGDVAC, HFGITR, DCCIVAC, 1),
SR_FGT(SYS_DC_CVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CGVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CGDVADP, HFGITR, DCCVADP, 1),
SR_FGT(SYS_DC_CVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CGVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CGDVAP, HFGITR, DCCVAP, 1),
SR_FGT(SYS_DC_CVAU, HFGITR, DCCVAU, 1),
SR_FGT(SYS_DC_CISW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CIGSW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CIGDSW, HFGITR, DCCISW, 1),
SR_FGT(SYS_DC_CSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_CGSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_CGDSW, HFGITR, DCCSW, 1),
SR_FGT(SYS_DC_ISW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IGSW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IGDSW, HFGITR, DCISW, 1),
SR_FGT(SYS_DC_IVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_DC_IGVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_DC_IGDVAC, HFGITR, DCIVAC, 1),
SR_FGT(SYS_IC_IVAU, HFGITR, ICIVAU, 1),
SR_FGT(SYS_IC_IALLU, HFGITR, ICIALLU, 1),
SR_FGT(SYS_IC_IALLUIS, HFGITR, ICIALLUIS, 1),
/* HDFGRTR_EL2 */
SR_FGT(SYS_PMBIDR_EL1, HDFGRTR, PMBIDR_EL1, 1),
SR_FGT(SYS_PMSNEVFR_EL1, HDFGRTR, nPMSNEVFR_EL1, 0),
SR_FGT(SYS_BRBINF_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINF_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBINFINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRC_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBSRCINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(0), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(1), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(2), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(3), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(4), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(5), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(6), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(7), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(8), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(9), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(10), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(11), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(12), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(13), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(14), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(15), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(16), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(17), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(18), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(19), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(20), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(21), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(22), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(23), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(24), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(25), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(26), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(27), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(28), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(29), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(30), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGT_EL1(31), HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTGTINJ_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBTS_EL1, HDFGRTR, nBRBDATA, 0),
SR_FGT(SYS_BRBCR_EL1, HDFGRTR, nBRBCTL, 0),
SR_FGT(SYS_BRBFCR_EL1, HDFGRTR, nBRBCTL, 0),
SR_FGT(SYS_BRBIDR0_EL1, HDFGRTR, nBRBIDR, 0),
SR_FGT(SYS_PMCEID0_EL0, HDFGRTR, PMCEIDn_EL0, 1),
SR_FGT(SYS_PMCEID1_EL0, HDFGRTR, PMCEIDn_EL0, 1),
SR_FGT(SYS_PMUSERENR_EL0, HDFGRTR, PMUSERENR_EL0, 1),
SR_FGT(SYS_TRBTRG_EL1, HDFGRTR, TRBTRG_EL1, 1),
SR_FGT(SYS_TRBSR_EL1, HDFGRTR, TRBSR_EL1, 1),
SR_FGT(SYS_TRBPTR_EL1, HDFGRTR, TRBPTR_EL1, 1),
SR_FGT(SYS_TRBMAR_EL1, HDFGRTR, TRBMAR_EL1, 1),
SR_FGT(SYS_TRBLIMITR_EL1, HDFGRTR, TRBLIMITR_EL1, 1),
SR_FGT(SYS_TRBIDR_EL1, HDFGRTR, TRBIDR_EL1, 1),
SR_FGT(SYS_TRBBASER_EL1, HDFGRTR, TRBBASER_EL1, 1),
SR_FGT(SYS_TRCVICTLR, HDFGRTR, TRCVICTLR, 1),
SR_FGT(SYS_TRCSTATR, HDFGRTR, TRCSTATR, 1),
SR_FGT(SYS_TRCSSCSR(0), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(1), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(2), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(3), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(4), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(5), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(6), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSSCSR(7), HDFGRTR, TRCSSCSRn, 1),
SR_FGT(SYS_TRCSEQSTR, HDFGRTR, TRCSEQSTR, 1),
SR_FGT(SYS_TRCPRGCTLR, HDFGRTR, TRCPRGCTLR, 1),
SR_FGT(SYS_TRCOSLSR, HDFGRTR, TRCOSLSR, 1),
SR_FGT(SYS_TRCIMSPEC(0), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(1), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(2), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(3), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(4), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(5), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(6), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCIMSPEC(7), HDFGRTR, TRCIMSPECn, 1),
SR_FGT(SYS_TRCDEVARCH, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCDEVID, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR0, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR1, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR2, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR3, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR4, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR5, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR6, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR7, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR8, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR9, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR10, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR11, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR12, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCIDR13, HDFGRTR, TRCID, 1),
SR_FGT(SYS_TRCCNTVR(0), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(1), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(2), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCNTVR(3), HDFGRTR, TRCCNTVRn, 1),
SR_FGT(SYS_TRCCLAIMCLR, HDFGRTR, TRCCLAIM, 1),
SR_FGT(SYS_TRCCLAIMSET, HDFGRTR, TRCCLAIM, 1),
SR_FGT(SYS_TRCAUXCTLR, HDFGRTR, TRCAUXCTLR, 1),
SR_FGT(SYS_TRCAUTHSTATUS, HDFGRTR, TRCAUTHSTATUS, 1),
SR_FGT(SYS_TRCACATR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACATR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCACVR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCBBCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCCCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCCTLR0, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCCTLR1, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCIDCVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTCTLR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCNTRLDVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCCONFIGR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEVENTCTL0R, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEVENTCTL1R, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCEXTINSELR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCQCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(8), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(9), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(10), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(11), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(12), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(13), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(14), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(15), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(16), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(17), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(18), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(19), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(20), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(21), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(22), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(23), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(24), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(25), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(26), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(27), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(28), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(29), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(30), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSCTLR(31), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCRSR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQEVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSEQRSTEVR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSCCR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSSPCICR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSTALLCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCSYNCPR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCTRACEIDR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCTSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVIIECTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVIPCSSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVISSCTLR, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCCTLR0, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCCTLR1, HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(0), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(1), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(2), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(3), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(4), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(5), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(6), HDFGRTR, TRC, 1),
SR_FGT(SYS_TRCVMIDCVR(7), HDFGRTR, TRC, 1),
SR_FGT(SYS_PMSLATFR_EL1, HDFGRTR, PMSLATFR_EL1, 1),
SR_FGT(SYS_PMSIRR_EL1, HDFGRTR, PMSIRR_EL1, 1),
SR_FGT(SYS_PMSIDR_EL1, HDFGRTR, PMSIDR_EL1, 1),
SR_FGT(SYS_PMSICR_EL1, HDFGRTR, PMSICR_EL1, 1),
SR_FGT(SYS_PMSFCR_EL1, HDFGRTR, PMSFCR_EL1, 1),
SR_FGT(SYS_PMSEVFR_EL1, HDFGRTR, PMSEVFR_EL1, 1),
SR_FGT(SYS_PMSCR_EL1, HDFGRTR, PMSCR_EL1, 1),
SR_FGT(SYS_PMBSR_EL1, HDFGRTR, PMBSR_EL1, 1),
SR_FGT(SYS_PMBPTR_EL1, HDFGRTR, PMBPTR_EL1, 1),
SR_FGT(SYS_PMBLIMITR_EL1, HDFGRTR, PMBLIMITR_EL1, 1),
SR_FGT(SYS_PMMIR_EL1, HDFGRTR, PMMIR_EL1, 1),
SR_FGT(SYS_PMSELR_EL0, HDFGRTR, PMSELR_EL0, 1),
SR_FGT(SYS_PMOVSCLR_EL0, HDFGRTR, PMOVS, 1),
SR_FGT(SYS_PMOVSSET_EL0, HDFGRTR, PMOVS, 1),
SR_FGT(SYS_PMINTENCLR_EL1, HDFGRTR, PMINTEN, 1),
SR_FGT(SYS_PMINTENSET_EL1, HDFGRTR, PMINTEN, 1),
SR_FGT(SYS_PMCNTENCLR_EL0, HDFGRTR, PMCNTEN, 1),
SR_FGT(SYS_PMCNTENSET_EL0, HDFGRTR, PMCNTEN, 1),
SR_FGT(SYS_PMCCNTR_EL0, HDFGRTR, PMCCNTR_EL0, 1),
SR_FGT(SYS_PMCCFILTR_EL0, HDFGRTR, PMCCFILTR_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(0), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(1), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(2), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(3), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(4), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(5), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(6), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(7), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(8), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(9), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(10), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(11), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(12), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(13), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(14), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(15), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(16), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(17), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(18), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(19), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(20), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(21), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(22), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(23), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(24), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(25), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(26), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(27), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(28), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(29), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVTYPERn_EL0(30), HDFGRTR, PMEVTYPERn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(0), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(1), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(2), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(3), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(4), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(5), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(6), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(7), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(8), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(9), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(10), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(11), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(12), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(13), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(14), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(15), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(16), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(17), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(18), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(19), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(20), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(21), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(22), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(23), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(24), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(25), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(26), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(27), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(28), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(29), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_PMEVCNTRn_EL0(30), HDFGRTR, PMEVCNTRn_EL0, 1),
SR_FGT(SYS_OSDLR_EL1, HDFGRTR, OSDLR_EL1, 1),
SR_FGT(SYS_OSECCR_EL1, HDFGRTR, OSECCR_EL1, 1),
SR_FGT(SYS_OSLSR_EL1, HDFGRTR, OSLSR_EL1, 1),
SR_FGT(SYS_DBGPRCR_EL1, HDFGRTR, DBGPRCR_EL1, 1),
SR_FGT(SYS_DBGAUTHSTATUS_EL1, HDFGRTR, DBGAUTHSTATUS_EL1, 1),
SR_FGT(SYS_DBGCLAIMSET_EL1, HDFGRTR, DBGCLAIM, 1),
SR_FGT(SYS_DBGCLAIMCLR_EL1, HDFGRTR, DBGCLAIM, 1),
SR_FGT(SYS_MDSCR_EL1, HDFGRTR, MDSCR_EL1, 1),
/*
* The trap bits capture *64* debug registers per bit, but the
* ARM ARM only describes the encoding for the first 16, and
* we don't really support more than that anyway.
*/
SR_FGT(SYS_DBGWVRn_EL1(0), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(1), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(2), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(3), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(4), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(5), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(6), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(7), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(8), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(9), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(10), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(11), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(12), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(13), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(14), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWVRn_EL1(15), HDFGRTR, DBGWVRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(0), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(1), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(2), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(3), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(4), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(5), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(6), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(7), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(8), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(9), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(10), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(11), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(12), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(13), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(14), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGWCRn_EL1(15), HDFGRTR, DBGWCRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(0), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(1), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(2), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(3), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(4), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(5), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(6), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(7), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(8), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(9), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(10), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(11), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(12), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(13), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(14), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBVRn_EL1(15), HDFGRTR, DBGBVRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(0), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(1), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(2), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(3), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(4), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(5), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(6), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(7), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(8), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(9), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(10), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(11), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(12), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(13), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(14), HDFGRTR, DBGBCRn_EL1, 1),
SR_FGT(SYS_DBGBCRn_EL1(15), HDFGRTR, DBGBCRn_EL1, 1),
/*
* HDFGWTR_EL2
*
* Although HDFGRTR_EL2 and HDFGWTR_EL2 registers largely
* overlap in their bit assignment, there are a number of bits
* that are RES0 on one side, and an actual trap bit on the
* other. The policy chosen here is to describe all the
* read-side mappings, and only the write-side mappings that
* differ from the read side, and the trap handler will pick
* the correct shadow register based on the access type.
*/
SR_FGT(SYS_TRFCR_EL1, HDFGWTR, TRFCR_EL1, 1),
SR_FGT(SYS_TRCOSLAR, HDFGWTR, TRCOSLAR, 1),
SR_FGT(SYS_PMCR_EL0, HDFGWTR, PMCR_EL0, 1),
SR_FGT(SYS_PMSWINC_EL0, HDFGWTR, PMSWINC_EL0, 1),
SR_FGT(SYS_OSLAR_EL1, HDFGWTR, OSLAR_EL1, 1),
/*
* HAFGRTR_EL2
*/
SR_FGT(SYS_AMEVTYPER1_EL0(15), HAFGRTR, AMEVTYPER115_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(14), HAFGRTR, AMEVTYPER114_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(13), HAFGRTR, AMEVTYPER113_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(12), HAFGRTR, AMEVTYPER112_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(11), HAFGRTR, AMEVTYPER111_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(10), HAFGRTR, AMEVTYPER110_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(9), HAFGRTR, AMEVTYPER19_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(8), HAFGRTR, AMEVTYPER18_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(7), HAFGRTR, AMEVTYPER17_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(6), HAFGRTR, AMEVTYPER16_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(5), HAFGRTR, AMEVTYPER15_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(4), HAFGRTR, AMEVTYPER14_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(3), HAFGRTR, AMEVTYPER13_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(2), HAFGRTR, AMEVTYPER12_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(1), HAFGRTR, AMEVTYPER11_EL0, 1),
SR_FGT(SYS_AMEVTYPER1_EL0(0), HAFGRTR, AMEVTYPER10_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(15), HAFGRTR, AMEVCNTR115_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(14), HAFGRTR, AMEVCNTR114_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(13), HAFGRTR, AMEVCNTR113_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(12), HAFGRTR, AMEVCNTR112_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(11), HAFGRTR, AMEVCNTR111_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(10), HAFGRTR, AMEVCNTR110_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(9), HAFGRTR, AMEVCNTR19_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(8), HAFGRTR, AMEVCNTR18_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(7), HAFGRTR, AMEVCNTR17_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(6), HAFGRTR, AMEVCNTR16_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(5), HAFGRTR, AMEVCNTR15_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(4), HAFGRTR, AMEVCNTR14_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(3), HAFGRTR, AMEVCNTR13_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(2), HAFGRTR, AMEVCNTR12_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(1), HAFGRTR, AMEVCNTR11_EL0, 1),
SR_FGT(SYS_AMEVCNTR1_EL0(0), HAFGRTR, AMEVCNTR10_EL0, 1),
SR_FGT(SYS_AMCNTENCLR1_EL0, HAFGRTR, AMCNTEN1, 1),
SR_FGT(SYS_AMCNTENSET1_EL0, HAFGRTR, AMCNTEN1, 1),
SR_FGT(SYS_AMCNTENCLR0_EL0, HAFGRTR, AMCNTEN0, 1),
SR_FGT(SYS_AMCNTENSET0_EL0, HAFGRTR, AMCNTEN0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(3), HAFGRTR, AMEVCNTR03_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(2), HAFGRTR, AMEVCNTR02_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(1), HAFGRTR, AMEVCNTR01_EL0, 1),
SR_FGT(SYS_AMEVCNTR0_EL0(0), HAFGRTR, AMEVCNTR00_EL0, 1),
};
static union trap_config get_trap_config(u32 sysreg)
{
return (union trap_config) {
.val = xa_to_value(xa_load(&sr_forward_xa, sysreg)),
};
}
static __init void print_nv_trap_error(const struct encoding_to_trap_config *tc,
const char *type, int err)
{
kvm_err("%s line %d encoding range "
"(%d, %d, %d, %d, %d) - (%d, %d, %d, %d, %d) (err=%d)\n",
type, tc->line,
sys_reg_Op0(tc->encoding), sys_reg_Op1(tc->encoding),
sys_reg_CRn(tc->encoding), sys_reg_CRm(tc->encoding),
sys_reg_Op2(tc->encoding),
sys_reg_Op0(tc->end), sys_reg_Op1(tc->end),
sys_reg_CRn(tc->end), sys_reg_CRm(tc->end),
sys_reg_Op2(tc->end),
err);
}
int __init populate_nv_trap_config(void)
{
int ret = 0;
BUILD_BUG_ON(sizeof(union trap_config) != sizeof(void *));
BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS));
BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS));
BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS));
for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) {
const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i];
void *prev;
if (cgt->tc.val & BIT(63)) {
kvm_err("CGT[%d] has MBZ bit set\n", i);
ret = -EINVAL;
}
if (cgt->encoding != cgt->end) {
prev = xa_store_range(&sr_forward_xa,
cgt->encoding, cgt->end,
xa_mk_value(cgt->tc.val),
GFP_KERNEL);
} else {
prev = xa_store(&sr_forward_xa, cgt->encoding,
xa_mk_value(cgt->tc.val), GFP_KERNEL);
if (prev && !xa_is_err(prev)) {
ret = -EINVAL;
print_nv_trap_error(cgt, "Duplicate CGT", ret);
}
}
if (xa_is_err(prev)) {
ret = xa_err(prev);
print_nv_trap_error(cgt, "Failed CGT insertion", ret);
}
}
kvm_info("nv: %ld coarse grained trap handlers\n",
ARRAY_SIZE(encoding_to_cgt));
if (!cpus_have_final_cap(ARM64_HAS_FGT))
goto check_mcb;
for (int i = 0; i < ARRAY_SIZE(encoding_to_fgt); i++) {
const struct encoding_to_trap_config *fgt = &encoding_to_fgt[i];
union trap_config tc;
if (fgt->tc.fgt >= __NR_FGT_GROUP_IDS__) {
ret = -EINVAL;
print_nv_trap_error(fgt, "Invalid FGT", ret);
}
tc = get_trap_config(fgt->encoding);
if (tc.fgt) {
ret = -EINVAL;
print_nv_trap_error(fgt, "Duplicate FGT", ret);
}
tc.val |= fgt->tc.val;
xa_store(&sr_forward_xa, fgt->encoding,
xa_mk_value(tc.val), GFP_KERNEL);
}
kvm_info("nv: %ld fine grained trap handlers\n",
ARRAY_SIZE(encoding_to_fgt));
check_mcb:
for (int id = __MULTIPLE_CONTROL_BITS__; id < __COMPLEX_CONDITIONS__; id++) {
const enum cgt_group_id *cgids;
cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
for (int i = 0; cgids[i] != __RESERVED__; i++) {
if (cgids[i] >= __MULTIPLE_CONTROL_BITS__) {
kvm_err("Recursive MCB %d/%d\n", id, cgids[i]);
ret = -EINVAL;
}
}
}
if (ret)
xa_destroy(&sr_forward_xa);
return ret;
}
static enum trap_behaviour get_behaviour(struct kvm_vcpu *vcpu,
const struct trap_bits *tb)
{
enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
u64 val;
val = __vcpu_sys_reg(vcpu, tb->index);
if ((val & tb->mask) == tb->value)
b |= tb->behaviour;
return b;
}
static enum trap_behaviour __compute_trap_behaviour(struct kvm_vcpu *vcpu,
const enum cgt_group_id id,
enum trap_behaviour b)
{
switch (id) {
const enum cgt_group_id *cgids;
case __RESERVED__ ... __MULTIPLE_CONTROL_BITS__ - 1:
if (likely(id != __RESERVED__))
b |= get_behaviour(vcpu, &coarse_trap_bits[id]);
break;
case __MULTIPLE_CONTROL_BITS__ ... __COMPLEX_CONDITIONS__ - 1:
/* Yes, this is recursive. Don't do anything stupid. */
cgids = coarse_control_combo[id - __MULTIPLE_CONTROL_BITS__];
for (int i = 0; cgids[i] != __RESERVED__; i++)
b |= __compute_trap_behaviour(vcpu, cgids[i], b);
break;
default:
if (ARRAY_SIZE(ccc))
b |= ccc[id - __COMPLEX_CONDITIONS__](vcpu);
break;
}
return b;
}
static enum trap_behaviour compute_trap_behaviour(struct kvm_vcpu *vcpu,
const union trap_config tc)
{
enum trap_behaviour b = BEHAVE_HANDLE_LOCALLY;
return __compute_trap_behaviour(vcpu, tc.cgt, b);
}
static bool check_fgt_bit(u64 val, const union trap_config tc)
{
return ((val >> tc.bit) & 1) == tc.pol;
}
#define sanitised_sys_reg(vcpu, reg) \
({ \
u64 __val; \
__val = __vcpu_sys_reg(vcpu, reg); \
__val &= ~__ ## reg ## _RES0; \
(__val); \
})
bool __check_nv_sr_forward(struct kvm_vcpu *vcpu)
{
union trap_config tc;
enum trap_behaviour b;
bool is_read;
u32 sysreg;
u64 esr, val;
if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
return false;
esr = kvm_vcpu_get_esr(vcpu);
sysreg = esr_sys64_to_sysreg(esr);
is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
tc = get_trap_config(sysreg);
/*
* A value of 0 for the whole entry means that we know nothing
* for this sysreg, and that it cannot be re-injected into the
* nested hypervisor. In this situation, let's cut it short.
*
* Note that ultimately, we could also make use of the xarray
* to store the index of the sysreg in the local descriptor
* array, avoiding another search... Hint, hint...
*/
if (!tc.val)
return false;
switch ((enum fgt_group_id)tc.fgt) {
case __NO_FGT_GROUP__:
break;
case HFGxTR_GROUP:
if (is_read)
val = sanitised_sys_reg(vcpu, HFGRTR_EL2);
else
val = sanitised_sys_reg(vcpu, HFGWTR_EL2);
break;
case HDFGRTR_GROUP:
case HDFGWTR_GROUP:
if (is_read)
val = sanitised_sys_reg(vcpu, HDFGRTR_EL2);
else
val = sanitised_sys_reg(vcpu, HDFGWTR_EL2);
break;
case HAFGRTR_GROUP:
val = sanitised_sys_reg(vcpu, HAFGRTR_EL2);
break;
case HFGITR_GROUP:
val = sanitised_sys_reg(vcpu, HFGITR_EL2);
switch (tc.fgf) {
u64 tmp;
case __NO_FGF__:
break;
case HCRX_FGTnXS:
tmp = sanitised_sys_reg(vcpu, HCRX_EL2);
if (tmp & HCRX_EL2_FGTnXS)
tc.fgt = __NO_FGT_GROUP__;
}
break;
case __NR_FGT_GROUP_IDS__:
/* Something is really wrong, bail out */
WARN_ONCE(1, "__NR_FGT_GROUP_IDS__");
return false;
}
if (tc.fgt != __NO_FGT_GROUP__ && check_fgt_bit(val, tc))
goto inject;
b = compute_trap_behaviour(vcpu, tc);
if (((b & BEHAVE_FORWARD_READ) && is_read) ||
((b & BEHAVE_FORWARD_WRITE) && !is_read))
goto inject;
return false;
inject:
trace_kvm_forward_sysreg_trap(vcpu, sysreg, is_read);
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return true;
}
static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr)
{
u64 mode = spsr & PSR_MODE_MASK;
/*
* Possible causes for an Illegal Exception Return from EL2:
* - trying to return to EL3
* - trying to return to an illegal M value
* - trying to return to a 32bit EL
* - trying to return to EL1 with HCR_EL2.TGE set
*/
if (mode == PSR_MODE_EL3t || mode == PSR_MODE_EL3h ||
mode == 0b00001 || (mode & BIT(1)) ||
(spsr & PSR_MODE32_BIT) ||
(vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
mode == PSR_MODE_EL1h))) {
/*
* The guest is playing with our nerves. Preserve EL, SP,
* masks, flags from the existing PSTATE, and set IL.
* The HW will then generate an Illegal State Exception
* immediately after ERET.
*/
spsr = *vcpu_cpsr(vcpu);
spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
PSR_MODE_MASK | PSR_MODE32_BIT);
spsr |= PSR_IL_BIT;
}
return spsr;
}
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)
{
u64 spsr, elr, mode;
bool direct_eret;
/*
* Going through the whole put/load motions is a waste of time
* if this is a VHE guest hypervisor returning to its own
* userspace, or the hypervisor performing a local exception
* return. No need to save/restore registers, no need to
* switch S2 MMU. Just do the canonical ERET.
*/
spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
spsr = kvm_check_illegal_exception_return(vcpu, spsr);
mode = spsr & (PSR_MODE_MASK | PSR_MODE32_BIT);
direct_eret = (mode == PSR_MODE_EL0t &&
vcpu_el2_e2h_is_set(vcpu) &&
vcpu_el2_tge_is_set(vcpu));
direct_eret |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
if (direct_eret) {
*vcpu_pc(vcpu) = vcpu_read_sys_reg(vcpu, ELR_EL2);
*vcpu_cpsr(vcpu) = spsr;
trace_kvm_nested_eret(vcpu, *vcpu_pc(vcpu), spsr);
return;
}
preempt_disable();
kvm_arch_vcpu_put(vcpu);
elr = __vcpu_sys_reg(vcpu, ELR_EL2);
trace_kvm_nested_eret(vcpu, elr, spsr);
/*
* Note that the current exception level is always the virtual EL2,
* since we set HCR_EL2.NV bit only when entering the virtual EL2.
*/
*vcpu_pc(vcpu) = elr;
*vcpu_cpsr(vcpu) = spsr;
kvm_arch_vcpu_load(vcpu, smp_processor_id());
preempt_enable();
}
static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
enum exception_type type)
{
trace_kvm_inject_nested_exception(vcpu, esr_el2, type);
switch (type) {
case except_type_sync:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
break;
case except_type_irq:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
break;
default:
WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
}
}
/*
* Emulate taking an exception to EL2.
* See ARM ARM J8.1.2 AArch64.TakeException()
*/
static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
enum exception_type type)
{
u64 pstate, mode;
bool direct_inject;
if (!vcpu_has_nv(vcpu)) {
kvm_err("Unexpected call to %s for the non-nesting configuration\n",
__func__);
return -EINVAL;
}
/*
* As for ERET, we can avoid doing too much on the injection path by
* checking that we either took the exception from a VHE host
* userspace or from vEL2. In these cases, there is no change in
* translation regime (or anything else), so let's do as little as
* possible.
*/
pstate = *vcpu_cpsr(vcpu);
mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT);
direct_inject = (mode == PSR_MODE_EL0t &&
vcpu_el2_e2h_is_set(vcpu) &&
vcpu_el2_tge_is_set(vcpu));
direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
if (direct_inject) {
kvm_inject_el2_exception(vcpu, esr_el2, type);
return 1;
}
preempt_disable();
/*
* We may have an exception or PC update in the EL0/EL1 context.
* Commit it before entering EL2.
*/
__kvm_adjust_pc(vcpu);
kvm_arch_vcpu_put(vcpu);
kvm_inject_el2_exception(vcpu, esr_el2, type);
/*
* A hard requirement is that a switch between EL1 and EL2
* contexts has to happen between a put/load, so that we can
* pick the correct timer and interrupt configuration, among
* other things.
*
* Make sure the exception actually took place before we load
* the new context.
*/
__kvm_adjust_pc(vcpu);
kvm_arch_vcpu_load(vcpu, smp_processor_id());
preempt_enable();
return 1;
}
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2)
{
return kvm_inject_nested(vcpu, esr_el2, except_type_sync);
}
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
{
/*
* Do not inject an irq if the:
* - Current exception level is EL2, and
* - virtual HCR_EL2.TGE == 0
* - virtual HCR_EL2.IMO == 0
*
* See Table D1-17 "Physical interrupt target and masking when EL3 is
* not implemented and EL2 is implemented" in ARM DDI 0487C.a.
*/
if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
!(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
return 1;
/* esr_el2 value doesn't matter for exits due to irqs. */
return kvm_inject_nested(vcpu, 0, except_type_irq);
}
Reference in a new issue View git blame Copy permalink