mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-06 08:46:46 +00:00
fa729bc7c9
Currently there is no synchronisation between finalize_pkvm() and
kvm_arm_init() initcalls. The finalize_pkvm() proceeds happily even if
kvm_arm_init() fails resulting in the following warning on all the CPUs
and eventually a HYP panic:
| kvm [1]: IPA Size Limit: 48 bits
| kvm [1]: Failed to init hyp memory protection
| kvm [1]: error initializing Hyp mode: -22
|
| <snip>
|
| WARNING: CPU: 0 PID: 0 at arch/arm64/kvm/pkvm.c:226 _kvm_host_prot_finalize+0x30/0x50
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.4.0 #237
| Hardware name: FVP Base RevC (DT)
| pstate: 634020c5 (nZCv daIF +PAN -UAO +TCO +DIT -SSBS BTYPE=--)
| pc : _kvm_host_prot_finalize+0x30/0x50
| lr : __flush_smp_call_function_queue+0xd8/0x230
|
| Call trace:
| _kvm_host_prot_finalize+0x3c/0x50
| on_each_cpu_cond_mask+0x3c/0x6c
| pkvm_drop_host_privileges+0x4c/0x78
| finalize_pkvm+0x3c/0x5c
| do_one_initcall+0xcc/0x240
| do_initcall_level+0x8c/0xac
| do_initcalls+0x54/0x94
| do_basic_setup+0x1c/0x28
| kernel_init_freeable+0x100/0x16c
| kernel_init+0x20/0x1a0
| ret_from_fork+0x10/0x20
| Failed to finalize Hyp protection: -22
| dtb=fvp-base-revc.dtb
| kvm [95]: nVHE hyp BUG at: arch/arm64/kvm/hyp/nvhe/mem_protect.c:540!
| kvm [95]: nVHE call trace:
| kvm [95]: [<ffff800081052984>] __kvm_nvhe_hyp_panic+0xac/0xf8
| kvm [95]: [<ffff800081059644>] __kvm_nvhe_handle_host_mem_abort+0x1a0/0x2ac
| kvm [95]: [<ffff80008105511c>] __kvm_nvhe_handle_trap+0x4c/0x160
| kvm [95]: [<ffff8000810540fc>] __kvm_nvhe___skip_pauth_save+0x4/0x4
| kvm [95]: ---[ end nVHE call trace ]---
| kvm [95]: Hyp Offset: 0xfffe8db00ffa0000
| Kernel panic - not syncing: HYP panic:
| PS:a34023c9 PC:0000f250710b973c ESR:00000000f2000800
| FAR:ffff000800cb00d0 HPFAR:000000000880cb00 PAR:0000000000000000
| VCPU:0000000000000000
| CPU: 3 PID: 95 Comm: kworker/u16:2 Tainted: G W 6.4.0 #237
| Hardware name: FVP Base RevC (DT)
| Workqueue: rpciod rpc_async_schedule
| Call trace:
| dump_backtrace+0xec/0x108
| show_stack+0x18/0x2c
| dump_stack_lvl+0x50/0x68
| dump_stack+0x18/0x24
| panic+0x138/0x33c
| nvhe_hyp_panic_handler+0x100/0x184
| new_slab+0x23c/0x54c
| ___slab_alloc+0x3e4/0x770
| kmem_cache_alloc_node+0x1f0/0x278
| __alloc_skb+0xdc/0x294
| tcp_stream_alloc_skb+0x2c/0xf0
| tcp_sendmsg_locked+0x3d0/0xda4
| tcp_sendmsg+0x38/0x5c
| inet_sendmsg+0x44/0x60
| sock_sendmsg+0x1c/0x34
| xprt_sock_sendmsg+0xdc/0x274
| xs_tcp_send_request+0x1ac/0x28c
| xprt_transmit+0xcc/0x300
| call_transmit+0x78/0x90
| __rpc_execute+0x114/0x3d8
| rpc_async_schedule+0x28/0x48
| process_one_work+0x1d8/0x314
| worker_thread+0x248/0x474
| kthread+0xfc/0x184
| ret_from_fork+0x10/0x20
| SMP: stopping secondary CPUs
| Kernel Offset: 0x57c5cb460000 from 0xffff800080000000
| PHYS_OFFSET: 0x80000000
| CPU features: 0x00000000,1035b7a3,ccfe773f
| Memory Limit: none
| ---[ end Kernel panic - not syncing: HYP panic:
| PS:a34023c9 PC:0000f250710b973c ESR:00000000f2000800
| FAR:ffff000800cb00d0 HPFAR:000000000880cb00 PAR:0000000000000000
| VCPU:0000000000000000 ]---
Fix it by checking for the successfull initialisation of kvm_arm_init()
in finalize_pkvm() before proceeding any futher.
Fixes: 87727ba2bb
("KVM: arm64: Ensure CPU PMU probes before pKVM host de-privilege")
Cc: Will Deacon <will@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Oliver Upton <oliver.upton@linux.dev>
Cc: James Morse <james.morse@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Zenghui Yu <yuzenghui@huawei.com>
Signed-off-by: Sudeep Holla <sudeep.holla@arm.com>
Acked-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20230704193243.3300506-1-sudeep.holla@arm.com
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
263 lines
6 KiB
C
263 lines
6 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2020 - Google LLC
|
|
* Author: Quentin Perret <qperret@google.com>
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/kmemleak.h>
|
|
#include <linux/kvm_host.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/sort.h>
|
|
|
|
#include <asm/kvm_pkvm.h>
|
|
|
|
#include "hyp_constants.h"
|
|
|
|
DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
|
|
|
|
static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
|
|
static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
|
|
|
|
phys_addr_t hyp_mem_base;
|
|
phys_addr_t hyp_mem_size;
|
|
|
|
static int cmp_hyp_memblock(const void *p1, const void *p2)
|
|
{
|
|
const struct memblock_region *r1 = p1;
|
|
const struct memblock_region *r2 = p2;
|
|
|
|
return r1->base < r2->base ? -1 : (r1->base > r2->base);
|
|
}
|
|
|
|
static void __init sort_memblock_regions(void)
|
|
{
|
|
sort(hyp_memory,
|
|
*hyp_memblock_nr_ptr,
|
|
sizeof(struct memblock_region),
|
|
cmp_hyp_memblock,
|
|
NULL);
|
|
}
|
|
|
|
static int __init register_memblock_regions(void)
|
|
{
|
|
struct memblock_region *reg;
|
|
|
|
for_each_mem_region(reg) {
|
|
if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
|
|
return -ENOMEM;
|
|
|
|
hyp_memory[*hyp_memblock_nr_ptr] = *reg;
|
|
(*hyp_memblock_nr_ptr)++;
|
|
}
|
|
sort_memblock_regions();
|
|
|
|
return 0;
|
|
}
|
|
|
|
void __init kvm_hyp_reserve(void)
|
|
{
|
|
u64 hyp_mem_pages = 0;
|
|
int ret;
|
|
|
|
if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
|
|
return;
|
|
|
|
if (kvm_get_mode() != KVM_MODE_PROTECTED)
|
|
return;
|
|
|
|
ret = register_memblock_regions();
|
|
if (ret) {
|
|
*hyp_memblock_nr_ptr = 0;
|
|
kvm_err("Failed to register hyp memblocks: %d\n", ret);
|
|
return;
|
|
}
|
|
|
|
hyp_mem_pages += hyp_s1_pgtable_pages();
|
|
hyp_mem_pages += host_s2_pgtable_pages();
|
|
hyp_mem_pages += hyp_vm_table_pages();
|
|
hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
|
|
hyp_mem_pages += hyp_ffa_proxy_pages();
|
|
|
|
/*
|
|
* Try to allocate a PMD-aligned region to reduce TLB pressure once
|
|
* this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
|
|
*/
|
|
hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
|
|
hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE),
|
|
PMD_SIZE);
|
|
if (!hyp_mem_base)
|
|
hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE);
|
|
else
|
|
hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
|
|
|
|
if (!hyp_mem_base) {
|
|
kvm_err("Failed to reserve hyp memory\n");
|
|
return;
|
|
}
|
|
|
|
kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
|
|
hyp_mem_base);
|
|
}
|
|
|
|
/*
|
|
* Allocates and donates memory for hypervisor VM structs at EL2.
|
|
*
|
|
* Allocates space for the VM state, which includes the hyp vm as well as
|
|
* the hyp vcpus.
|
|
*
|
|
* Stores an opaque handler in the kvm struct for future reference.
|
|
*
|
|
* Return 0 on success, negative error code on failure.
|
|
*/
|
|
static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
|
|
{
|
|
size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz;
|
|
struct kvm_vcpu *host_vcpu;
|
|
pkvm_handle_t handle;
|
|
void *pgd, *hyp_vm;
|
|
unsigned long idx;
|
|
int ret;
|
|
|
|
if (host_kvm->created_vcpus < 1)
|
|
return -EINVAL;
|
|
|
|
pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.vtcr);
|
|
|
|
/*
|
|
* The PGD pages will be reclaimed using a hyp_memcache which implies
|
|
* page granularity. So, use alloc_pages_exact() to get individual
|
|
* refcounts.
|
|
*/
|
|
pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
|
|
if (!pgd)
|
|
return -ENOMEM;
|
|
|
|
/* Allocate memory to donate to hyp for vm and vcpu pointers. */
|
|
hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
|
|
size_mul(sizeof(void *),
|
|
host_kvm->created_vcpus)));
|
|
hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
|
|
if (!hyp_vm) {
|
|
ret = -ENOMEM;
|
|
goto free_pgd;
|
|
}
|
|
|
|
/* Donate the VM memory to hyp and let hyp initialize it. */
|
|
ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd);
|
|
if (ret < 0)
|
|
goto free_vm;
|
|
|
|
handle = ret;
|
|
|
|
host_kvm->arch.pkvm.handle = handle;
|
|
|
|
/* Donate memory for the vcpus at hyp and initialize it. */
|
|
hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
|
|
kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
|
|
void *hyp_vcpu;
|
|
|
|
/* Indexing of the vcpus to be sequential starting at 0. */
|
|
if (WARN_ON(host_vcpu->vcpu_idx != idx)) {
|
|
ret = -EINVAL;
|
|
goto destroy_vm;
|
|
}
|
|
|
|
hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
|
|
if (!hyp_vcpu) {
|
|
ret = -ENOMEM;
|
|
goto destroy_vm;
|
|
}
|
|
|
|
ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu,
|
|
hyp_vcpu);
|
|
if (ret) {
|
|
free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
|
|
goto destroy_vm;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
|
|
destroy_vm:
|
|
pkvm_destroy_hyp_vm(host_kvm);
|
|
return ret;
|
|
free_vm:
|
|
free_pages_exact(hyp_vm, hyp_vm_sz);
|
|
free_pgd:
|
|
free_pages_exact(pgd, pgd_sz);
|
|
return ret;
|
|
}
|
|
|
|
int pkvm_create_hyp_vm(struct kvm *host_kvm)
|
|
{
|
|
int ret = 0;
|
|
|
|
mutex_lock(&host_kvm->lock);
|
|
if (!host_kvm->arch.pkvm.handle)
|
|
ret = __pkvm_create_hyp_vm(host_kvm);
|
|
mutex_unlock(&host_kvm->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
|
|
{
|
|
if (host_kvm->arch.pkvm.handle) {
|
|
WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm,
|
|
host_kvm->arch.pkvm.handle));
|
|
}
|
|
|
|
host_kvm->arch.pkvm.handle = 0;
|
|
free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
|
|
}
|
|
|
|
int pkvm_init_host_vm(struct kvm *host_kvm)
|
|
{
|
|
mutex_init(&host_kvm->lock);
|
|
return 0;
|
|
}
|
|
|
|
static void __init _kvm_host_prot_finalize(void *arg)
|
|
{
|
|
int *err = arg;
|
|
|
|
if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)))
|
|
WRITE_ONCE(*err, -EINVAL);
|
|
}
|
|
|
|
static int __init pkvm_drop_host_privileges(void)
|
|
{
|
|
int ret = 0;
|
|
|
|
/*
|
|
* Flip the static key upfront as that may no longer be possible
|
|
* once the host stage 2 is installed.
|
|
*/
|
|
static_branch_enable(&kvm_protected_mode_initialized);
|
|
on_each_cpu(_kvm_host_prot_finalize, &ret, 1);
|
|
return ret;
|
|
}
|
|
|
|
static int __init finalize_pkvm(void)
|
|
{
|
|
int ret;
|
|
|
|
if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised())
|
|
return 0;
|
|
|
|
/*
|
|
* Exclude HYP sections from kmemleak so that they don't get peeked
|
|
* at, which would end badly once inaccessible.
|
|
*/
|
|
kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
|
|
kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size);
|
|
|
|
ret = pkvm_drop_host_privileges();
|
|
if (ret)
|
|
pr_err("Failed to finalize Hyp protection: %d\n", ret);
|
|
|
|
return ret;
|
|
}
|
|
device_initcall_sync(finalize_pkvm);
|