2018-11-02 09:05:17 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2008-02-15 19:52:48 +00:00
|
|
|
/* KVM paravirtual clock driver. A clocksource implementation
|
|
|
|
Copyright (C) 2008 Glauber de Oliveira Costa, Red Hat Inc.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/clocksource.h>
|
|
|
|
#include <linux/kvm_para.h>
|
2008-06-03 14:17:32 +00:00
|
|
|
#include <asm/pvclock.h>
|
2008-02-15 19:52:48 +00:00
|
|
|
#include <asm/msr.h>
|
|
|
|
#include <asm/apic.h>
|
|
|
|
#include <linux/percpu.h>
|
2012-03-10 19:37:26 +00:00
|
|
|
#include <linux/hardirq.h>
|
2018-07-19 20:55:26 +00:00
|
|
|
#include <linux/cpuhotplug.h>
|
2015-05-28 23:20:40 +00:00
|
|
|
#include <linux/sched.h>
|
2017-02-01 15:36:40 +00:00
|
|
|
#include <linux/sched/clock.h>
|
2018-07-19 20:55:20 +00:00
|
|
|
#include <linux/mm.h>
|
Merge branch 'l1tf-final' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Merge L1 Terminal Fault fixes from Thomas Gleixner:
"L1TF, aka L1 Terminal Fault, is yet another speculative hardware
engineering trainwreck. It's a hardware vulnerability which allows
unprivileged speculative access to data which is available in the
Level 1 Data Cache when the page table entry controlling the virtual
address, which is used for the access, has the Present bit cleared or
other reserved bits set.
If an instruction accesses a virtual address for which the relevant
page table entry (PTE) has the Present bit cleared or other reserved
bits set, then speculative execution ignores the invalid PTE and loads
the referenced data if it is present in the Level 1 Data Cache, as if
the page referenced by the address bits in the PTE was still present
and accessible.
While this is a purely speculative mechanism and the instruction will
raise a page fault when it is retired eventually, the pure act of
loading the data and making it available to other speculative
instructions opens up the opportunity for side channel attacks to
unprivileged malicious code, similar to the Meltdown attack.
While Meltdown breaks the user space to kernel space protection, L1TF
allows to attack any physical memory address in the system and the
attack works across all protection domains. It allows an attack of SGX
and also works from inside virtual machines because the speculation
bypasses the extended page table (EPT) protection mechanism.
The assoicated CVEs are: CVE-2018-3615, CVE-2018-3620, CVE-2018-3646
The mitigations provided by this pull request include:
- Host side protection by inverting the upper address bits of a non
present page table entry so the entry points to uncacheable memory.
- Hypervisor protection by flushing L1 Data Cache on VMENTER.
- SMT (HyperThreading) control knobs, which allow to 'turn off' SMT
by offlining the sibling CPU threads. The knobs are available on
the kernel command line and at runtime via sysfs
- Control knobs for the hypervisor mitigation, related to L1D flush
and SMT control. The knobs are available on the kernel command line
and at runtime via sysfs
- Extensive documentation about L1TF including various degrees of
mitigations.
Thanks to all people who have contributed to this in various ways -
patches, review, testing, backporting - and the fruitful, sometimes
heated, but at the end constructive discussions.
There is work in progress to provide other forms of mitigations, which
might be less horrible performance wise for a particular kind of
workloads, but this is not yet ready for consumption due to their
complexity and limitations"
* 'l1tf-final' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (75 commits)
x86/microcode: Allow late microcode loading with SMT disabled
tools headers: Synchronise x86 cpufeatures.h for L1TF additions
x86/mm/kmmio: Make the tracer robust against L1TF
x86/mm/pat: Make set_memory_np() L1TF safe
x86/speculation/l1tf: Make pmd/pud_mknotpresent() invert
x86/speculation/l1tf: Invert all not present mappings
cpu/hotplug: Fix SMT supported evaluation
KVM: VMX: Tell the nested hypervisor to skip L1D flush on vmentry
x86/speculation: Use ARCH_CAPABILITIES to skip L1D flush on vmentry
x86/speculation: Simplify sysfs report of VMX L1TF vulnerability
Documentation/l1tf: Remove Yonah processors from not vulnerable list
x86/KVM/VMX: Don't set l1tf_flush_l1d from vmx_handle_external_intr()
x86/irq: Let interrupt handlers set kvm_cpu_l1tf_flush_l1d
x86: Don't include linux/irq.h from asm/hardirq.h
x86/KVM/VMX: Introduce per-host-cpu analogue of l1tf_flush_l1d
x86/irq: Demote irq_cpustat_t::__softirq_pending to u16
x86/KVM/VMX: Move the l1tf_flush_l1d test to vmx_l1d_flush()
x86/KVM/VMX: Replace 'vmx_l1d_flush_always' with 'vmx_l1d_flush_cond'
x86/KVM/VMX: Don't set l1tf_flush_l1d to true from vmx_l1d_flush()
cpu/hotplug: detect SMT disabled by BIOS
...
2018-08-14 16:46:06 +00:00
|
|
|
#include <linux/slab.h>
|
2018-09-14 13:45:59 +00:00
|
|
|
#include <linux/set_memory.h>
|
2009-08-19 10:35:53 +00:00
|
|
|
|
2018-07-19 20:55:25 +00:00
|
|
|
#include <asm/hypervisor.h>
|
2017-10-20 14:30:59 +00:00
|
|
|
#include <asm/mem_encrypt.h>
|
2009-08-19 10:35:53 +00:00
|
|
|
#include <asm/x86_init.h>
|
2008-03-17 19:08:40 +00:00
|
|
|
#include <asm/reboot.h>
|
2017-01-24 17:09:41 +00:00
|
|
|
#include <asm/kvmclock.h>
|
2008-02-15 19:52:48 +00:00
|
|
|
|
2018-07-19 20:55:24 +00:00
|
|
|
static int kvmclock __initdata = 1;
|
2018-07-19 20:55:25 +00:00
|
|
|
static int kvmclock_vsyscall __initdata = 1;
|
2018-07-19 20:55:24 +00:00
|
|
|
static int msr_kvm_system_time __ro_after_init = MSR_KVM_SYSTEM_TIME;
|
|
|
|
static int msr_kvm_wall_clock __ro_after_init = MSR_KVM_WALL_CLOCK;
|
|
|
|
static u64 kvm_sched_clock_offset __ro_after_init;
|
2008-02-15 19:52:48 +00:00
|
|
|
|
2018-07-19 20:55:23 +00:00
|
|
|
static int __init parse_no_kvmclock(char *arg)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
|
|
|
kvmclock = 0;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("no-kvmclock", parse_no_kvmclock);
|
|
|
|
|
2018-07-19 20:55:25 +00:00
|
|
|
static int __init parse_no_kvmclock_vsyscall(char *arg)
|
|
|
|
{
|
|
|
|
kvmclock_vsyscall = 0;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
|
|
|
|
|
2018-07-19 20:55:20 +00:00
|
|
|
/* Aligned to page sizes to match whats mapped via vsyscalls to userspace */
|
|
|
|
#define HV_CLOCK_SIZE (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS)
|
2018-07-19 20:55:26 +00:00
|
|
|
#define HVC_BOOT_ARRAY_SIZE \
|
|
|
|
(PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
|
2018-07-19 20:55:20 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
static struct pvclock_vsyscall_time_info
|
2018-09-14 13:45:59 +00:00
|
|
|
hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __bss_decrypted __aligned(PAGE_SIZE);
|
|
|
|
static struct pvclock_wall_clock wall_clock __bss_decrypted;
|
2018-07-19 20:55:26 +00:00
|
|
|
static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
|
2018-09-14 13:45:59 +00:00
|
|
|
static struct pvclock_vsyscall_time_info *hvclock_mem;
|
2018-07-19 20:55:26 +00:00
|
|
|
|
|
|
|
static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
|
|
|
|
{
|
|
|
|
return &this_cpu_read(hv_clock_per_cpu)->pvti;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct pvclock_vsyscall_time_info *this_cpu_hvclock(void)
|
|
|
|
{
|
|
|
|
return this_cpu_read(hv_clock_per_cpu);
|
|
|
|
}
|
2008-02-15 19:52:48 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The wallclock is the time of day when we booted. Since then, some time may
|
|
|
|
* have elapsed since the hypervisor wrote the data. So we try to account for
|
|
|
|
* that with system time
|
|
|
|
*/
|
2018-04-27 20:13:23 +00:00
|
|
|
static void kvm_get_wallclock(struct timespec64 *now)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
2018-07-19 20:55:23 +00:00
|
|
|
wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(&wall_clock));
|
2018-07-19 20:55:26 +00:00
|
|
|
preempt_disable();
|
|
|
|
pvclock_read_wallclock(&wall_clock, this_cpu_pvti(), now);
|
|
|
|
preempt_enable();
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|
|
|
|
|
2018-04-27 20:13:23 +00:00
|
|
|
static int kvm_set_wallclock(const struct timespec64 *now)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
2017-10-31 20:28:09 +00:00
|
|
|
return -ENODEV;
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|
|
|
|
|
2016-12-21 19:32:01 +00:00
|
|
|
static u64 kvm_clock_read(void)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
2016-12-21 19:32:01 +00:00
|
|
|
u64 ret;
|
2008-02-15 19:52:48 +00:00
|
|
|
|
2011-11-15 12:59:07 +00:00
|
|
|
preempt_disable_notrace();
|
2018-07-19 20:55:26 +00:00
|
|
|
ret = pvclock_clocksource_read(this_cpu_pvti());
|
2011-11-15 12:59:07 +00:00
|
|
|
preempt_enable_notrace();
|
2008-06-03 14:17:32 +00:00
|
|
|
return ret;
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|
2008-06-03 14:17:32 +00:00
|
|
|
|
2016-12-21 19:32:01 +00:00
|
|
|
static u64 kvm_clock_get_cycles(struct clocksource *cs)
|
2009-04-21 19:24:00 +00:00
|
|
|
{
|
|
|
|
return kvm_clock_read();
|
|
|
|
}
|
|
|
|
|
2016-12-21 19:32:01 +00:00
|
|
|
static u64 kvm_sched_clock_read(void)
|
2015-09-18 15:54:29 +00:00
|
|
|
{
|
|
|
|
return kvm_clock_read() - kvm_sched_clock_offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void kvm_sched_clock_init(bool stable)
|
|
|
|
{
|
2019-01-26 17:49:56 +00:00
|
|
|
if (!stable)
|
2017-01-19 13:36:33 +00:00
|
|
|
clear_sched_clock_stable();
|
2015-09-18 15:54:29 +00:00
|
|
|
kvm_sched_clock_offset = kvm_clock_read();
|
2018-08-28 07:40:19 +00:00
|
|
|
pv_ops.time.sched_clock = kvm_sched_clock_read;
|
2015-09-18 15:54:29 +00:00
|
|
|
|
2018-07-19 20:55:23 +00:00
|
|
|
pr_info("kvm-clock: using sched offset of %llu cycles",
|
|
|
|
kvm_sched_clock_offset);
|
2015-09-18 15:54:29 +00:00
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(kvm_sched_clock_offset) >
|
2018-07-19 20:55:23 +00:00
|
|
|
sizeof(((struct pvclock_vcpu_time_info *)NULL)->system_time));
|
2015-09-18 15:54:29 +00:00
|
|
|
}
|
|
|
|
|
2008-07-28 14:47:53 +00:00
|
|
|
/*
|
|
|
|
* If we don't do that, there is the possibility that the guest
|
|
|
|
* will calibrate under heavy load - thus, getting a lower lpj -
|
|
|
|
* and execute the delays themselves without load. This is wrong,
|
|
|
|
* because no delay loop can finish beforehand.
|
|
|
|
* Any heuristics is subject to fail, because ultimately, a large
|
|
|
|
* poll of guests can be running and trouble each other. So we preset
|
|
|
|
* lpj here
|
|
|
|
*/
|
|
|
|
static unsigned long kvm_get_tsc_khz(void)
|
|
|
|
{
|
2018-07-14 15:28:29 +00:00
|
|
|
setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
|
2018-07-19 20:55:26 +00:00
|
|
|
return pvclock_tsc_khz(this_cpu_pvti());
|
2008-07-28 14:47:53 +00:00
|
|
|
}
|
|
|
|
|
2018-07-30 07:54:21 +00:00
|
|
|
static void __init kvm_get_preset_lpj(void)
|
2008-07-28 14:47:53 +00:00
|
|
|
{
|
|
|
|
unsigned long khz;
|
|
|
|
u64 lpj;
|
|
|
|
|
2008-12-05 20:36:45 +00:00
|
|
|
khz = kvm_get_tsc_khz();
|
2008-07-28 14:47:53 +00:00
|
|
|
|
|
|
|
lpj = ((u64)khz * 1000);
|
|
|
|
do_div(lpj, HZ);
|
|
|
|
preset_lpj = lpj;
|
|
|
|
}
|
|
|
|
|
2012-03-10 19:37:26 +00:00
|
|
|
bool kvm_check_and_clear_guest_paused(void)
|
|
|
|
{
|
2018-07-19 20:55:26 +00:00
|
|
|
struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
|
2018-07-19 20:55:23 +00:00
|
|
|
bool ret = false;
|
2012-11-28 01:28:48 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
if (!src)
|
2012-11-28 01:28:48 +00:00
|
|
|
return ret;
|
2012-03-10 19:37:26 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
if ((src->pvti.flags & PVCLOCK_GUEST_STOPPED) != 0) {
|
|
|
|
src->pvti.flags &= ~PVCLOCK_GUEST_STOPPED;
|
2013-10-12 00:39:25 +00:00
|
|
|
pvclock_touch_watchdogs();
|
2012-03-10 19:37:26 +00:00
|
|
|
ret = true;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-01-24 17:09:41 +00:00
|
|
|
struct clocksource kvm_clock = {
|
2018-07-19 20:55:23 +00:00
|
|
|
.name = "kvm-clock",
|
|
|
|
.read = kvm_clock_get_cycles,
|
|
|
|
.rating = 400,
|
|
|
|
.mask = CLOCKSOURCE_MASK(64),
|
|
|
|
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
|
2008-02-15 19:52:48 +00:00
|
|
|
};
|
2017-01-24 17:09:41 +00:00
|
|
|
EXPORT_SYMBOL_GPL(kvm_clock);
|
2008-02-15 19:52:48 +00:00
|
|
|
|
2018-07-19 20:55:22 +00:00
|
|
|
static void kvm_register_clock(char *txt)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
2018-07-19 20:55:26 +00:00
|
|
|
struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
|
2018-07-19 20:55:22 +00:00
|
|
|
u64 pa;
|
2013-02-23 16:05:29 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
if (!src)
|
2018-07-19 20:55:22 +00:00
|
|
|
return;
|
2010-08-02 21:35:28 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
pa = slow_virt_to_phys(&src->pvti) | 0x01ULL;
|
2018-07-19 20:55:22 +00:00
|
|
|
wrmsrl(msr_kvm_system_time, pa);
|
2018-07-19 20:55:26 +00:00
|
|
|
pr_info("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt);
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|
|
|
|
|
2012-02-13 13:07:27 +00:00
|
|
|
static void kvm_save_sched_clock_state(void)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static void kvm_restore_sched_clock_state(void)
|
|
|
|
{
|
|
|
|
kvm_register_clock("primary cpu clock, resume");
|
|
|
|
}
|
|
|
|
|
2008-04-30 15:39:05 +00:00
|
|
|
#ifdef CONFIG_X86_LOCAL_APIC
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void kvm_setup_secondary_clock(void)
|
2008-02-15 19:52:48 +00:00
|
|
|
{
|
2018-07-19 20:55:22 +00:00
|
|
|
kvm_register_clock("secondary cpu clock");
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|
2008-04-30 15:39:05 +00:00
|
|
|
#endif
|
2008-02-15 19:52:48 +00:00
|
|
|
|
2008-03-17 19:08:40 +00:00
|
|
|
/*
|
|
|
|
* After the clock is registered, the host will keep writing to the
|
|
|
|
* registered memory location. If the guest happens to shutdown, this memory
|
|
|
|
* won't be valid. In cases like kexec, in which you install a new kernel, this
|
|
|
|
* means a random memory location will be kept being written. So before any
|
2016-02-23 23:34:30 +00:00
|
|
|
* kind of shutdown from our side, we unregister the clock by writing anything
|
2008-03-17 19:08:40 +00:00
|
|
|
* that does not have the 'enable' bit set in the msr
|
|
|
|
*/
|
2015-09-09 22:38:55 +00:00
|
|
|
#ifdef CONFIG_KEXEC_CORE
|
2008-03-17 19:08:40 +00:00
|
|
|
static void kvm_crash_shutdown(struct pt_regs *regs)
|
|
|
|
{
|
2010-05-11 16:17:44 +00:00
|
|
|
native_write_msr(msr_kvm_system_time, 0, 0);
|
2011-07-11 19:28:19 +00:00
|
|
|
kvm_disable_steal_time();
|
2008-03-17 19:08:40 +00:00
|
|
|
native_machine_crash_shutdown(regs);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void kvm_shutdown(void)
|
|
|
|
{
|
2010-05-11 16:17:44 +00:00
|
|
|
native_write_msr(msr_kvm_system_time, 0, 0);
|
2011-07-11 19:28:19 +00:00
|
|
|
kvm_disable_steal_time();
|
2008-03-17 19:08:40 +00:00
|
|
|
native_machine_shutdown();
|
|
|
|
}
|
|
|
|
|
2018-09-14 13:45:59 +00:00
|
|
|
static void __init kvmclock_init_mem(void)
|
|
|
|
{
|
|
|
|
unsigned long ncpus;
|
|
|
|
unsigned int order;
|
|
|
|
struct page *p;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
if (HVC_BOOT_ARRAY_SIZE >= num_possible_cpus())
|
|
|
|
return;
|
|
|
|
|
|
|
|
ncpus = num_possible_cpus() - HVC_BOOT_ARRAY_SIZE;
|
|
|
|
order = get_order(ncpus * sizeof(*hvclock_mem));
|
|
|
|
|
|
|
|
p = alloc_pages(GFP_KERNEL, order);
|
|
|
|
if (!p) {
|
|
|
|
pr_warn("%s: failed to alloc %d pages", __func__, (1U << order));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
hvclock_mem = page_address(p);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* hvclock is shared between the guest and the hypervisor, must
|
|
|
|
* be mapped decrypted.
|
|
|
|
*/
|
|
|
|
if (sev_active()) {
|
|
|
|
r = set_memory_decrypted((unsigned long) hvclock_mem,
|
|
|
|
1UL << order);
|
|
|
|
if (r) {
|
|
|
|
__free_pages(p, order);
|
|
|
|
hvclock_mem = NULL;
|
|
|
|
pr_warn("kvmclock: set_memory_decrypted() failed. Disabling\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
memset(hvclock_mem, 0, PAGE_SIZE << order);
|
|
|
|
}
|
|
|
|
|
2018-07-19 20:55:25 +00:00
|
|
|
static int __init kvm_setup_vsyscall_timeinfo(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
u8 flags;
|
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
if (!per_cpu(hv_clock_per_cpu, 0) || !kvmclock_vsyscall)
|
2018-07-19 20:55:25 +00:00
|
|
|
return 0;
|
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
flags = pvclock_read_flags(&hv_clock_boot[0].pvti);
|
2018-07-19 20:55:25 +00:00
|
|
|
if (!(flags & PVCLOCK_TSC_STABLE_BIT))
|
2018-07-19 20:55:26 +00:00
|
|
|
return 0;
|
2018-07-19 20:55:25 +00:00
|
|
|
|
|
|
|
kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK;
|
|
|
|
#endif
|
2018-09-14 13:45:59 +00:00
|
|
|
|
|
|
|
kvmclock_init_mem();
|
|
|
|
|
2018-07-19 20:55:25 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_initcall(kvm_setup_vsyscall_timeinfo);
|
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
static int kvmclock_setup_percpu(unsigned int cpu)
|
|
|
|
{
|
|
|
|
struct pvclock_vsyscall_time_info *p = per_cpu(hv_clock_per_cpu, cpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The per cpu area setup replicates CPU0 data to all cpu
|
|
|
|
* pointers. So carefully check. CPU0 has been set up in init
|
|
|
|
* already.
|
|
|
|
*/
|
|
|
|
if (!cpu || (p && p != per_cpu(hv_clock_per_cpu, 0)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Use the static page for the first CPUs, allocate otherwise */
|
|
|
|
if (cpu < HVC_BOOT_ARRAY_SIZE)
|
|
|
|
p = &hv_clock_boot[cpu];
|
2018-09-14 13:45:59 +00:00
|
|
|
else if (hvclock_mem)
|
|
|
|
p = hvclock_mem + cpu - HVC_BOOT_ARRAY_SIZE;
|
2018-07-19 20:55:26 +00:00
|
|
|
else
|
2018-09-14 13:45:59 +00:00
|
|
|
return -ENOMEM;
|
2018-07-19 20:55:26 +00:00
|
|
|
|
|
|
|
per_cpu(hv_clock_per_cpu, cpu) = p;
|
|
|
|
return p ? 0 : -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2008-02-15 19:52:48 +00:00
|
|
|
void __init kvmclock_init(void)
|
|
|
|
{
|
2015-05-28 23:20:40 +00:00
|
|
|
u8 flags;
|
2013-02-19 01:58:14 +00:00
|
|
|
|
2018-07-19 20:55:23 +00:00
|
|
|
if (!kvm_para_available() || !kvmclock)
|
2008-02-15 19:52:48 +00:00
|
|
|
return;
|
|
|
|
|
2018-07-19 20:55:23 +00:00
|
|
|
if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) {
|
2010-05-11 16:17:44 +00:00
|
|
|
msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW;
|
|
|
|
msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW;
|
2018-07-19 20:55:23 +00:00
|
|
|
} else if (!kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) {
|
2010-05-11 16:17:44 +00:00
|
|
|
return;
|
2018-07-19 20:55:23 +00:00
|
|
|
}
|
2010-05-11 16:17:44 +00:00
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
if (cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "kvmclock:setup_percpu",
|
|
|
|
kvmclock_setup_percpu, NULL) < 0) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-07-19 20:55:23 +00:00
|
|
|
pr_info("kvm-clock: Using msrs %x and %x",
|
2017-10-20 14:30:59 +00:00
|
|
|
msr_kvm_system_time, msr_kvm_wall_clock);
|
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
this_cpu_write(hv_clock_per_cpu, &hv_clock_boot[0]);
|
2018-07-19 20:55:22 +00:00
|
|
|
kvm_register_clock("primary cpu clock");
|
2018-07-19 20:55:26 +00:00
|
|
|
pvclock_set_pvti_cpu0_va(hv_clock_boot);
|
2018-07-15 15:43:11 +00:00
|
|
|
|
2015-09-18 15:54:29 +00:00
|
|
|
if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT))
|
|
|
|
pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
|
|
|
|
|
2018-07-19 20:55:26 +00:00
|
|
|
flags = pvclock_read_flags(&hv_clock_boot[0].pvti);
|
2015-09-18 15:54:29 +00:00
|
|
|
kvm_sched_clock_init(flags & PVCLOCK_TSC_STABLE_BIT);
|
|
|
|
|
2010-05-11 16:17:44 +00:00
|
|
|
x86_platform.calibrate_tsc = kvm_get_tsc_khz;
|
2016-09-08 14:15:28 +00:00
|
|
|
x86_platform.calibrate_cpu = kvm_get_tsc_khz;
|
2010-05-11 16:17:44 +00:00
|
|
|
x86_platform.get_wallclock = kvm_get_wallclock;
|
|
|
|
x86_platform.set_wallclock = kvm_set_wallclock;
|
2008-04-30 15:39:05 +00:00
|
|
|
#ifdef CONFIG_X86_LOCAL_APIC
|
2018-07-19 20:55:23 +00:00
|
|
|
x86_cpuinit.early_percpu_clock_init = kvm_setup_secondary_clock;
|
2008-04-30 15:39:05 +00:00
|
|
|
#endif
|
2012-02-13 13:07:27 +00:00
|
|
|
x86_platform.save_sched_clock_state = kvm_save_sched_clock_state;
|
|
|
|
x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state;
|
2010-05-11 16:17:44 +00:00
|
|
|
machine_ops.shutdown = kvm_shutdown;
|
2015-09-09 22:38:55 +00:00
|
|
|
#ifdef CONFIG_KEXEC_CORE
|
2010-05-11 16:17:44 +00:00
|
|
|
machine_ops.crash_shutdown = kvm_crash_shutdown;
|
2008-03-17 19:08:40 +00:00
|
|
|
#endif
|
2010-05-11 16:17:44 +00:00
|
|
|
kvm_get_preset_lpj();
|
2010-04-27 02:03:05 +00:00
|
|
|
clocksource_register_hz(&kvm_clock, NSEC_PER_SEC);
|
2010-05-11 16:17:44 +00:00
|
|
|
pv_info.name = "KVM";
|
2008-02-15 19:52:48 +00:00
|
|
|
}
|