Merge branch 'svm' of https://github.com/kvm-x86/linux into HEAD

Clean up SVM's enter/exit assembly code so that it can be compiled
without OBJECT_FILES_NON_STANDARD.  The "standard" __svm_vcpu_run() can't
be made 100% bulletproof, as RBP isn't restored on #VMEXIT, but that's
also the case for __vmx_vcpu_run(), and getting "close enough" is better
than not even trying.

As for SEV-ES, after yet another refresher on swap types, I realized
KVM can simply let the hardware restore registers after #VMEXIT, all
that's missing is storing the current values to the host save area
(they are swap type B).  This should provide 100% accuracy when using
stack frames for unwinding, and requires less assembly.

In between, build the SEV-ES code iff CONFIG_KVM_AMD_SEV=y, and yank out
"support" for 32-bit kernels in __svm_sev_es_vcpu_run, which was
unnecessarily polluting the code for a configuration that is disabled
at build time.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
Paolo Bonzini 2024-04-17 11:42:51 -04:00
commit 44ecfa3e5f
5 changed files with 57 additions and 67 deletions

View File

@ -3,11 +3,6 @@
ccflags-y += -I $(srctree)/arch/x86/kvm ccflags-y += -I $(srctree)/arch/x86/kvm
ccflags-$(CONFIG_KVM_WERROR) += -Werror ccflags-$(CONFIG_KVM_WERROR) += -Werror
ifeq ($(CONFIG_FRAME_POINTER),y)
OBJECT_FILES_NON_STANDARD_vmx/vmenter.o := y
OBJECT_FILES_NON_STANDARD_svm/vmenter.o := y
endif
include $(srctree)/virt/kvm/Makefile.kvm include $(srctree)/virt/kvm/Makefile.kvm
kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \

View File

@ -434,7 +434,7 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
/* Avoid using vmalloc for smaller buffers. */ /* Avoid using vmalloc for smaller buffers. */
size = npages * sizeof(struct page *); size = npages * sizeof(struct page *);
if (size > PAGE_SIZE) if (size > PAGE_SIZE)
pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); pages = __vmalloc(size, GFP_KERNEL_ACCOUNT);
else else
pages = kmalloc(size, GFP_KERNEL_ACCOUNT); pages = kmalloc(size, GFP_KERNEL_ACCOUNT);

View File

@ -1503,6 +1503,11 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu)
__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
} }
static struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
{
return page_address(sd->save_area) + 0x400;
}
static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
{ {
struct vcpu_svm *svm = to_svm(vcpu); struct vcpu_svm *svm = to_svm(vcpu);
@ -1519,12 +1524,8 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
* or subsequent vmload of host save area. * or subsequent vmload of host save area.
*/ */
vmsave(sd->save_area_pa); vmsave(sd->save_area_pa);
if (sev_es_guest(vcpu->kvm)) { if (sev_es_guest(vcpu->kvm))
struct sev_es_save_area *hostsa; sev_es_prepare_switch_to_guest(sev_es_host_save_area(sd));
hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
sev_es_prepare_switch_to_guest(hostsa);
}
if (tsc_scaling) if (tsc_scaling)
__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
@ -4101,6 +4102,7 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted) static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
{ {
struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
struct vcpu_svm *svm = to_svm(vcpu); struct vcpu_svm *svm = to_svm(vcpu);
guest_state_enter_irqoff(); guest_state_enter_irqoff();
@ -4108,7 +4110,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in
amd_clear_divider(); amd_clear_divider();
if (sev_es_guest(vcpu->kvm)) if (sev_es_guest(vcpu->kvm))
__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
sev_es_host_save_area(sd));
else else
__svm_vcpu_run(svm, spec_ctrl_intercepted); __svm_vcpu_run(svm, spec_ctrl_intercepted);

View File

@ -698,7 +698,8 @@ struct page *snp_safe_alloc_page(struct kvm_vcpu *vcpu);
/* vmenter.S */ /* vmenter.S */
void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted); void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted,
struct sev_es_save_area *hostsa);
void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted); void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
#define DEFINE_KVM_GHCB_ACCESSORS(field) \ #define DEFINE_KVM_GHCB_ACCESSORS(field) \

View File

@ -3,6 +3,7 @@
#include <asm/asm.h> #include <asm/asm.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/bitsperlong.h> #include <asm/bitsperlong.h>
#include <asm/frame.h>
#include <asm/kvm_vcpu_regs.h> #include <asm/kvm_vcpu_regs.h>
#include <asm/nospec-branch.h> #include <asm/nospec-branch.h>
#include "kvm-asm-offsets.h" #include "kvm-asm-offsets.h"
@ -67,7 +68,7 @@
"", X86_FEATURE_V_SPEC_CTRL "", X86_FEATURE_V_SPEC_CTRL
901: 901:
.endm .endm
.macro RESTORE_HOST_SPEC_CTRL_BODY .macro RESTORE_HOST_SPEC_CTRL_BODY spec_ctrl_intercepted:req
900: 900:
/* Same for after vmexit. */ /* Same for after vmexit. */
mov $MSR_IA32_SPEC_CTRL, %ecx mov $MSR_IA32_SPEC_CTRL, %ecx
@ -76,7 +77,7 @@
* Load the value that the guest had written into MSR_IA32_SPEC_CTRL, * Load the value that the guest had written into MSR_IA32_SPEC_CTRL,
* if it was not intercepted during guest execution. * if it was not intercepted during guest execution.
*/ */
cmpb $0, (%_ASM_SP) cmpb $0, \spec_ctrl_intercepted
jnz 998f jnz 998f
rdmsr rdmsr
movl %eax, SVM_spec_ctrl(%_ASM_DI) movl %eax, SVM_spec_ctrl(%_ASM_DI)
@ -99,6 +100,7 @@
*/ */
SYM_FUNC_START(__svm_vcpu_run) SYM_FUNC_START(__svm_vcpu_run)
push %_ASM_BP push %_ASM_BP
mov %_ASM_SP, %_ASM_BP
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
push %r15 push %r15
push %r14 push %r14
@ -268,7 +270,7 @@ SYM_FUNC_START(__svm_vcpu_run)
RET RET
RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_GUEST_SPEC_CTRL_BODY
RESTORE_HOST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY (%_ASM_SP)
10: cmpb $0, _ASM_RIP(kvm_rebooting) 10: cmpb $0, _ASM_RIP(kvm_rebooting)
jne 2b jne 2b
@ -290,66 +292,68 @@ SYM_FUNC_START(__svm_vcpu_run)
SYM_FUNC_END(__svm_vcpu_run) SYM_FUNC_END(__svm_vcpu_run)
#ifdef CONFIG_KVM_AMD_SEV
#ifdef CONFIG_X86_64
#define SEV_ES_GPRS_BASE 0x300
#define SEV_ES_RBX (SEV_ES_GPRS_BASE + __VCPU_REGS_RBX * WORD_SIZE)
#define SEV_ES_RBP (SEV_ES_GPRS_BASE + __VCPU_REGS_RBP * WORD_SIZE)
#define SEV_ES_RSI (SEV_ES_GPRS_BASE + __VCPU_REGS_RSI * WORD_SIZE)
#define SEV_ES_RDI (SEV_ES_GPRS_BASE + __VCPU_REGS_RDI * WORD_SIZE)
#define SEV_ES_R12 (SEV_ES_GPRS_BASE + __VCPU_REGS_R12 * WORD_SIZE)
#define SEV_ES_R13 (SEV_ES_GPRS_BASE + __VCPU_REGS_R13 * WORD_SIZE)
#define SEV_ES_R14 (SEV_ES_GPRS_BASE + __VCPU_REGS_R14 * WORD_SIZE)
#define SEV_ES_R15 (SEV_ES_GPRS_BASE + __VCPU_REGS_R15 * WORD_SIZE)
#endif
/** /**
* __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode
* @svm: struct vcpu_svm * * @svm: struct vcpu_svm *
* @spec_ctrl_intercepted: bool * @spec_ctrl_intercepted: bool
*/ */
SYM_FUNC_START(__svm_sev_es_vcpu_run) SYM_FUNC_START(__svm_sev_es_vcpu_run)
push %_ASM_BP FRAME_BEGIN
#ifdef CONFIG_X86_64
push %r15
push %r14
push %r13
push %r12
#else
push %edi
push %esi
#endif
push %_ASM_BX
/* /*
* Save variables needed after vmexit on the stack, in inverse * Save non-volatile (callee-saved) registers to the host save area.
* order compared to when they are needed. * Except for RAX and RSP, all GPRs are restored on #VMEXIT, but not
* saved on VMRUN.
*/ */
mov %rbp, SEV_ES_RBP (%rdx)
mov %r15, SEV_ES_R15 (%rdx)
mov %r14, SEV_ES_R14 (%rdx)
mov %r13, SEV_ES_R13 (%rdx)
mov %r12, SEV_ES_R12 (%rdx)
mov %rbx, SEV_ES_RBX (%rdx)
/* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL. */
push %_ASM_ARG2
/* Save @svm. */
push %_ASM_ARG1
.ifnc _ASM_ARG1, _ASM_DI
/* /*
* Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX * Save volatile registers that hold arguments that are needed after
* and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL. * #VMEXIT (RDI=@svm and RSI=@spec_ctrl_intercepted).
*/ */
mov %_ASM_ARG1, %_ASM_DI mov %rdi, SEV_ES_RDI (%rdx)
.endif mov %rsi, SEV_ES_RSI (%rdx)
/* Clobbers RAX, RCX, RDX. */ /* Clobbers RAX, RCX, RDX (@hostsa). */
RESTORE_GUEST_SPEC_CTRL RESTORE_GUEST_SPEC_CTRL
/* Get svm->current_vmcb->pa into RAX. */ /* Get svm->current_vmcb->pa into RAX. */
mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX mov SVM_current_vmcb(%rdi), %rax
mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX mov KVM_VMCB_pa(%rax), %rax
/* Enter guest mode */ /* Enter guest mode */
sti sti
1: vmrun %_ASM_AX 1: vmrun %rax
2: cli 2: cli
/* Pop @svm to RDI, guest registers have been saved already. */
pop %_ASM_DI
#ifdef CONFIG_MITIGATION_RETPOLINE #ifdef CONFIG_MITIGATION_RETPOLINE
/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
#endif #endif
/* Clobbers RAX, RCX, RDX. */ /* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@spec_ctrl_intercepted). */
RESTORE_HOST_SPEC_CTRL RESTORE_HOST_SPEC_CTRL
/* /*
@ -361,30 +365,17 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
*/ */
UNTRAIN_RET_VM UNTRAIN_RET_VM
/* "Pop" @spec_ctrl_intercepted. */ FRAME_END
pop %_ASM_BX
pop %_ASM_BX
#ifdef CONFIG_X86_64
pop %r12
pop %r13
pop %r14
pop %r15
#else
pop %esi
pop %edi
#endif
pop %_ASM_BP
RET RET
RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_GUEST_SPEC_CTRL_BODY
RESTORE_HOST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY %sil
3: cmpb $0, _ASM_RIP(kvm_rebooting) 3: cmpb $0, kvm_rebooting(%rip)
jne 2b jne 2b
ud2 ud2
_ASM_EXTABLE(1b, 3b) _ASM_EXTABLE(1b, 3b)
SYM_FUNC_END(__svm_sev_es_vcpu_run) SYM_FUNC_END(__svm_sev_es_vcpu_run)
#endif /* CONFIG_KVM_AMD_SEV */