2017-12-20 17:28:54 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/percpu.h>
|
2018-06-06 12:54:10 +00:00
|
|
|
#include <linux/kallsyms.h>
|
x86: Add entry trampolines to kcore
Without program headers for PTI entry trampoline pages, the trampoline
virtual addresses do not map to anything.
Example before:
sudo gdb --quiet vmlinux /proc/kcore
Reading symbols from vmlinux...done.
[New process 1]
Core was generated by `BOOT_IMAGE=/boot/vmlinuz-4.16.0 root=UUID=a6096b83-b763-4101-807e-f33daff63233'.
#0 0x0000000000000000 in irq_stack_union ()
(gdb) x /21ib 0xfffffe0000006000
0xfffffe0000006000: Cannot access memory at address 0xfffffe0000006000
(gdb) quit
After:
sudo gdb --quiet vmlinux /proc/kcore
[sudo] password for ahunter:
Reading symbols from vmlinux...done.
[New process 1]
Core was generated by `BOOT_IMAGE=/boot/vmlinuz-4.16.0-fix-4-00005-gd6e65a8b4072 root=UUID=a6096b83-b7'.
#0 0x0000000000000000 in irq_stack_union ()
(gdb) x /21ib 0xfffffe0000006000
0xfffffe0000006000: swapgs
0xfffffe0000006003: mov %rsp,-0x3e12(%rip) # 0xfffffe00000021f8
0xfffffe000000600a: xchg %ax,%ax
0xfffffe000000600c: mov %cr3,%rsp
0xfffffe000000600f: bts $0x3f,%rsp
0xfffffe0000006014: and $0xffffffffffffe7ff,%rsp
0xfffffe000000601b: mov %rsp,%cr3
0xfffffe000000601e: mov -0x3019(%rip),%rsp # 0xfffffe000000300c
0xfffffe0000006025: pushq $0x2b
0xfffffe0000006027: pushq -0x3e35(%rip) # 0xfffffe00000021f8
0xfffffe000000602d: push %r11
0xfffffe000000602f: pushq $0x33
0xfffffe0000006031: push %rcx
0xfffffe0000006032: push %rdi
0xfffffe0000006033: mov $0xffffffff91a00010,%rdi
0xfffffe000000603a: callq 0xfffffe0000006046
0xfffffe000000603f: pause
0xfffffe0000006041: lfence
0xfffffe0000006044: jmp 0xfffffe000000603f
0xfffffe0000006046: mov %rdi,(%rsp)
0xfffffe000000604a: retq
(gdb) quit
In addition, entry trampolines all map to the same page. Represent that
by giving the corresponding program headers in kcore the same offset.
This has the benefit that, when perf tools uses /proc/kcore as a source
for kernel object code, samples from different CPU trampolines are
aggregated together. Note, such aggregation is normal for profiling
i.e. people want to profile the object code, not every different virtual
address the object code might be mapped to (across different processes
for example).
Notes by PeterZ:
This also adds the KCORE_REMAP functionality.
Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
Acked-by: Andi Kleen <ak@linux.intel.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: x86@kernel.org
Link: http://lkml.kernel.org/r/1528289651-4113-4-git-send-email-adrian.hunter@intel.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
2018-06-06 12:54:11 +00:00
|
|
|
#include <linux/kcore.h>
|
2017-12-20 17:28:54 +00:00
|
|
|
|
|
|
|
#include <asm/cpu_entry_area.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/fixmap.h>
|
|
|
|
#include <asm/desc.h>
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage);
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
|
|
|
|
[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
|
|
|
|
#endif
|
|
|
|
|
2017-12-20 17:51:31 +00:00
|
|
|
struct cpu_entry_area *get_cpu_entry_area(int cpu)
|
|
|
|
{
|
|
|
|
unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
|
|
|
|
BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
|
|
|
|
|
|
|
|
return (struct cpu_entry_area *) va;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(get_cpu_entry_area);
|
|
|
|
|
|
|
|
void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
|
|
|
|
{
|
|
|
|
unsigned long va = (unsigned long) cea_vaddr;
|
2018-04-06 20:55:15 +00:00
|
|
|
pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The cpu_entry_area is shared between the user and kernel
|
|
|
|
* page tables. All of its ptes can safely be global.
|
|
|
|
* _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
|
|
|
|
* non-present PTEs, so be careful not to set it in that
|
|
|
|
* case to avoid confusion.
|
|
|
|
*/
|
|
|
|
if (boot_cpu_has(X86_FEATURE_PGE) &&
|
|
|
|
(pgprot_val(flags) & _PAGE_PRESENT))
|
|
|
|
pte = pte_set_flags(pte, _PAGE_GLOBAL);
|
|
|
|
|
|
|
|
set_pte_vaddr(va, pte);
|
2017-12-20 17:51:31 +00:00
|
|
|
}
|
|
|
|
|
2017-12-20 17:28:54 +00:00
|
|
|
static void __init
|
2017-12-20 17:51:31 +00:00
|
|
|
cea_map_percpu_pages(void *cea_vaddr, void *ptr, int pages, pgprot_t prot)
|
2017-12-20 17:28:54 +00:00
|
|
|
{
|
2017-12-20 17:51:31 +00:00
|
|
|
for ( ; pages; pages--, cea_vaddr+= PAGE_SIZE, ptr += PAGE_SIZE)
|
|
|
|
cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot);
|
2017-12-20 17:28:54 +00:00
|
|
|
}
|
|
|
|
|
2019-01-08 13:02:24 +00:00
|
|
|
static void __init percpu_setup_debug_store(int cpu)
|
2017-12-04 14:07:49 +00:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_CPU_SUP_INTEL
|
|
|
|
int npages;
|
|
|
|
void *cea;
|
|
|
|
|
|
|
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
cea = &get_cpu_entry_area(cpu)->cpu_debug_store;
|
|
|
|
npages = sizeof(struct debug_store) / PAGE_SIZE;
|
|
|
|
BUILD_BUG_ON(sizeof(struct debug_store) % PAGE_SIZE != 0);
|
|
|
|
cea_map_percpu_pages(cea, &per_cpu(cpu_debug_store, cpu), npages,
|
|
|
|
PAGE_KERNEL);
|
|
|
|
|
|
|
|
cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers;
|
|
|
|
/*
|
|
|
|
* Force the population of PMDs for not yet allocated per cpu
|
|
|
|
* memory like debug store buffers.
|
|
|
|
*/
|
|
|
|
npages = sizeof(struct debug_store_buffers) / PAGE_SIZE;
|
|
|
|
for (; npages; npages--, cea += PAGE_SIZE)
|
|
|
|
cea_set_pte(cea, 0, PAGE_NONE);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-12-20 17:28:54 +00:00
|
|
|
/* Setup the fixmap mappings only once per-processor */
|
|
|
|
static void __init setup_cpu_entry_area(int cpu)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
/* On 64-bit systems, we use a read-only fixmap GDT and TSS. */
|
|
|
|
pgprot_t gdt_prot = PAGE_KERNEL_RO;
|
|
|
|
pgprot_t tss_prot = PAGE_KERNEL_RO;
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* On native 32-bit systems, the GDT cannot be read-only because
|
|
|
|
* our double fault handler uses a task gate, and entering through
|
|
|
|
* a task gate needs to change an available TSS to busy. If the
|
|
|
|
* GDT is read-only, that will triple fault. The TSS cannot be
|
|
|
|
* read-only because the CPU writes to it on task switches.
|
|
|
|
*
|
|
|
|
* On Xen PV, the GDT must be read-only because the hypervisor
|
|
|
|
* requires it.
|
|
|
|
*/
|
|
|
|
pgprot_t gdt_prot = boot_cpu_has(X86_FEATURE_XENPV) ?
|
|
|
|
PAGE_KERNEL_RO : PAGE_KERNEL;
|
|
|
|
pgprot_t tss_prot = PAGE_KERNEL;
|
|
|
|
#endif
|
|
|
|
|
2017-12-20 17:51:31 +00:00
|
|
|
cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu),
|
|
|
|
gdt_prot);
|
|
|
|
|
|
|
|
cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page,
|
|
|
|
per_cpu_ptr(&entry_stack_storage, cpu), 1,
|
|
|
|
PAGE_KERNEL);
|
2017-12-20 17:28:54 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The Intel SDM says (Volume 3, 7.2.1):
|
|
|
|
*
|
|
|
|
* Avoid placing a page boundary in the part of the TSS that the
|
|
|
|
* processor reads during a task switch (the first 104 bytes). The
|
|
|
|
* processor may not correctly perform address translations if a
|
|
|
|
* boundary occurs in this area. During a task switch, the processor
|
|
|
|
* reads and writes into the first 104 bytes of each TSS (using
|
|
|
|
* contiguous physical addresses beginning with the physical address
|
|
|
|
* of the first byte of the TSS). So, after TSS access begins, if
|
|
|
|
* part of the 104 bytes is not physically contiguous, the processor
|
|
|
|
* will access incorrect information without generating a page-fault
|
|
|
|
* exception.
|
|
|
|
*
|
|
|
|
* There are also a lot of errata involving the TSS spanning a page
|
|
|
|
* boundary. Assert that we're not doing that.
|
|
|
|
*/
|
|
|
|
BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
|
|
|
|
offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
|
|
|
|
BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
|
2017-12-20 17:51:31 +00:00
|
|
|
cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss,
|
|
|
|
&per_cpu(cpu_tss_rw, cpu),
|
|
|
|
sizeof(struct tss_struct) / PAGE_SIZE, tss_prot);
|
2017-12-20 17:28:54 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
|
|
|
|
BUILD_BUG_ON(sizeof(exception_stacks) !=
|
|
|
|
sizeof(((struct cpu_entry_area *)0)->exception_stacks));
|
2017-12-20 17:51:31 +00:00
|
|
|
cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks,
|
|
|
|
&per_cpu(exception_stacks, cpu),
|
|
|
|
sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL);
|
2017-12-20 17:28:54 +00:00
|
|
|
#endif
|
2017-12-04 14:07:49 +00:00
|
|
|
percpu_setup_debug_store(cpu);
|
2017-12-20 17:28:54 +00:00
|
|
|
}
|
|
|
|
|
2017-12-20 17:51:31 +00:00
|
|
|
static __init void setup_cpu_entry_area_ptes(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
unsigned long start, end;
|
|
|
|
|
|
|
|
BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE);
|
|
|
|
BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
|
|
|
|
|
|
|
|
start = CPU_ENTRY_AREA_BASE;
|
|
|
|
end = start + CPU_ENTRY_AREA_MAP_SIZE;
|
|
|
|
|
2017-12-23 18:45:11 +00:00
|
|
|
/* Careful here: start + PMD_SIZE might wrap around */
|
|
|
|
for (; start < end && start >= CPU_ENTRY_AREA_BASE; start += PMD_SIZE)
|
2017-12-20 17:51:31 +00:00
|
|
|
populate_extra_pte(start);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-12-20 17:28:54 +00:00
|
|
|
void __init setup_cpu_entry_areas(void)
|
|
|
|
{
|
|
|
|
unsigned int cpu;
|
|
|
|
|
2017-12-20 17:51:31 +00:00
|
|
|
setup_cpu_entry_area_ptes();
|
|
|
|
|
2017-12-20 17:28:54 +00:00
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
setup_cpu_entry_area(cpu);
|
2018-02-28 20:14:26 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* This is the last essential update to swapper_pgdir which needs
|
|
|
|
* to be synchronized to initial_page_table on 32bit.
|
|
|
|
*/
|
|
|
|
sync_initial_page_table();
|
2017-12-20 17:28:54 +00:00
|
|
|
}
|