mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 03:10:12 +00:00
![Linus Torvalds](/assets/img/avatar_default.png)
Highlights include: - Larger virtual address space on 64-bit server CPUs. By default we use a 128TB virtual address space, but a process can request access to the full 512TB by passing a hint to mmap(). - Support for the new Power9 "XIVE" interrupt controller. - TLB flushing optimisations for the radix MMU on Power9. - Support for CAPI cards on Power9, using the "Coherent Accelerator Interface Architecture 2.0". - The ability to configure the mmap randomisation limits at build and runtime. - Several small fixes and cleanups to the kprobes code, as well as support for KPROBES_ON_FTRACE. - Major improvements to handling of system reset interrupts, correctly treating them as NMIs, giving them a dedicated stack and using a new hypervisor call to trigger them, all of which should aid debugging and robustness. Many fixes and other minor enhancements. Thanks to: Alastair D'Silva, Alexey Kardashevskiy, Alistair Popple, Andrew Donnellan, Aneesh Kumar K.V, Anshuman Khandual, Anton Blanchard, Balbir Singh, Ben Hutchings, Benjamin Herrenschmidt, Bhupesh Sharma, Chris Packham, Christian Zigotzky, Christophe Leroy, Christophe Lombard, Daniel Axtens, David Gibson, Gautham R. Shenoy, Gavin Shan, Geert Uytterhoeven, Guilherme G. Piccoli, Hamish Martin, Hari Bathini, Kees Cook, Laurent Dufour, Madhavan Srinivasan, Mahesh J Salgaonkar, Mahesh Salgaonkar, Masami Hiramatsu, Matt Brown, Matthew R. Ochs, Michael Neuling, Naveen N. Rao, Nicholas Piggin, Oliver O'Halloran, Pan Xinhui, Paul Mackerras, Rashmica Gupta, Russell Currey, Sukadev Bhattiprolu, Thadeu Lima de Souza Cascardo, Tobin C. Harding, Tyrel Datwyler, Uma Krishnan, Vaibhav Jain, Vipin K Parashar, Yang Shi. -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQIcBAABAgAGBQJZDHUMAAoJEFHr6jzI4aWAT7oQALkE2Nj3gjcn1z0SkFhq/1iO Py9Elmqm4E+L6NKYtBY5dS8xVAJ088ffzERyqJ1FY1LHkB8tn8bWRcMQmbjAFzTI V4TAzDNI890BN/F4ptrYRwNFxRBHAvZ4NDunTzagwYnwmTzW9PYHmOi4pvWTo3Tw KFUQ0joLSEgHzyfXxYB3fyj41u8N0FZvhfazdNSqia2Y5Vwwv/ION5jKplDM+09Y EtVEXFvaKAS1sjbM/d/Jo5rblHfR0D9/lYV10+jjyIokjzslIpyTbnj3izeYoM5V I4h99372zfsEjBGPPXyM3khL3zizGMSDYRmJHQSaKxjtecS9SPywPTZ8ufO/aSzV Ngq6nlND+f1zep29VQ0cxd3Jh40skWOXzxJaFjfDT25xa6FbfsWP2NCtk8PGylZ7 EyqTuCWkMgIP02KlX3oHvEB2LRRPCDmRU2zECecRGNJrIQwYC2xjoiVi7Q8Qe8rY gr7Ib5Jj/a+uiTcCIy37+5nXq2s14/JBOKqxuYZIxeuZFvKYuRUipbKWO05WDOAz m/pSzeC3J8AAoYiqR0gcSOuJTOnJpGhs7zrQFqnEISbXIwLW+ICumzOmTAiBqOEY Rt8uW2gYkPwKLrE05445RfVUoERaAjaE06eRMOWS6slnngHmmnRJbf3PcoALiJkT ediqGEj0/N1HMB31V5tS =vSF3 -----END PGP SIGNATURE----- Merge tag 'powerpc-4.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux Pull powerpc updates from Michael Ellerman: "Highlights include: - Larger virtual address space on 64-bit server CPUs. By default we use a 128TB virtual address space, but a process can request access to the full 512TB by passing a hint to mmap(). - Support for the new Power9 "XIVE" interrupt controller. - TLB flushing optimisations for the radix MMU on Power9. - Support for CAPI cards on Power9, using the "Coherent Accelerator Interface Architecture 2.0". - The ability to configure the mmap randomisation limits at build and runtime. - Several small fixes and cleanups to the kprobes code, as well as support for KPROBES_ON_FTRACE. - Major improvements to handling of system reset interrupts, correctly treating them as NMIs, giving them a dedicated stack and using a new hypervisor call to trigger them, all of which should aid debugging and robustness. - Many fixes and other minor enhancements. Thanks to: Alastair D'Silva, Alexey Kardashevskiy, Alistair Popple, Andrew Donnellan, Aneesh Kumar K.V, Anshuman Khandual, Anton Blanchard, Balbir Singh, Ben Hutchings, Benjamin Herrenschmidt, Bhupesh Sharma, Chris Packham, Christian Zigotzky, Christophe Leroy, Christophe Lombard, Daniel Axtens, David Gibson, Gautham R. Shenoy, Gavin Shan, Geert Uytterhoeven, Guilherme G. Piccoli, Hamish Martin, Hari Bathini, Kees Cook, Laurent Dufour, Madhavan Srinivasan, Mahesh J Salgaonkar, Mahesh Salgaonkar, Masami Hiramatsu, Matt Brown, Matthew R. Ochs, Michael Neuling, Naveen N. Rao, Nicholas Piggin, Oliver O'Halloran, Pan Xinhui, Paul Mackerras, Rashmica Gupta, Russell Currey, Sukadev Bhattiprolu, Thadeu Lima de Souza Cascardo, Tobin C. Harding, Tyrel Datwyler, Uma Krishnan, Vaibhav Jain, Vipin K Parashar, Yang Shi" * tag 'powerpc-4.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (214 commits) powerpc/64s: Power9 has no LPCR[VRMASD] field so don't set it powerpc/powernv: Fix TCE kill on NVLink2 powerpc/mm/radix: Drop support for CPUs without lockless tlbie powerpc/book3s/mce: Move add_taint() later in virtual mode powerpc/sysfs: Move #ifdef CONFIG_HOTPLUG_CPU out of the function body powerpc/smp: Document irq enable/disable after migrating IRQs powerpc/mpc52xx: Don't select user-visible RTAS_PROC powerpc/powernv: Document cxl dependency on special case in pnv_eeh_reset() powerpc/eeh: Clean up and document event handling functions powerpc/eeh: Avoid use after free in eeh_handle_special_event() cxl: Mask slice error interrupts after first occurrence cxl: Route eeh events to all drivers in cxl_pci_error_detected() cxl: Force context lock during EEH flow powerpc/64: Allow CONFIG_RELOCATABLE if COMPILE_TEST powerpc/xmon: Teach xmon oops about radix vectors powerpc/mm/hash: Fix off-by-one in comment about kernel contexts ids powerpc/pseries: Enable VFIO powerpc/powernv: Fix iommu table size calculation hook for small tables powerpc/powernv: Check kzalloc() return value in pnv_pci_table_alloc powerpc: Add arch/powerpc/tools directory ...
413 lines
11 KiB
C
413 lines
11 KiB
C
/*
|
|
* PowerPC version
|
|
* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
|
|
*
|
|
* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
|
|
* and Cort Dougan (PReP) (cort@cs.nmt.edu)
|
|
* Copyright (C) 1996 Paul Mackerras
|
|
*
|
|
* Derived from "arch/i386/mm/init.c"
|
|
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
|
|
*
|
|
* Dave Engebretsen <engebret@us.ibm.com>
|
|
* Rework for PPC64 port.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
*/
|
|
|
|
#undef DEBUG
|
|
|
|
#include <linux/signal.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/string.h>
|
|
#include <linux/types.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/init.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/idr.h>
|
|
#include <linux/nodemask.h>
|
|
#include <linux/module.h>
|
|
#include <linux/poison.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/hugetlb.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/of_fdt.h>
|
|
#include <linux/libfdt.h>
|
|
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/page.h>
|
|
#include <asm/prom.h>
|
|
#include <asm/rtas.h>
|
|
#include <asm/io.h>
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/mmu.h>
|
|
#include <linux/uaccess.h>
|
|
#include <asm/smp.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/tlb.h>
|
|
#include <asm/eeh.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/mmzone.h>
|
|
#include <asm/cputable.h>
|
|
#include <asm/sections.h>
|
|
#include <asm/iommu.h>
|
|
#include <asm/vdso.h>
|
|
|
|
#include "mmu_decl.h"
|
|
|
|
#ifdef CONFIG_PPC_STD_MMU_64
|
|
#if H_PGTABLE_RANGE > USER_VSID_RANGE
|
|
#warning Limited user VSID range means pagetable space is wasted
|
|
#endif
|
|
#endif /* CONFIG_PPC_STD_MMU_64 */
|
|
|
|
phys_addr_t memstart_addr = ~0;
|
|
EXPORT_SYMBOL_GPL(memstart_addr);
|
|
phys_addr_t kernstart_addr;
|
|
EXPORT_SYMBOL_GPL(kernstart_addr);
|
|
|
|
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
|
/*
|
|
* Given an address within the vmemmap, determine the pfn of the page that
|
|
* represents the start of the section it is within. Note that we have to
|
|
* do this by hand as the proffered address may not be correctly aligned.
|
|
* Subtraction of non-aligned pointers produces undefined results.
|
|
*/
|
|
static unsigned long __meminit vmemmap_section_start(unsigned long page)
|
|
{
|
|
unsigned long offset = page - ((unsigned long)(vmemmap));
|
|
|
|
/* Return the pfn of the start of the section. */
|
|
return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;
|
|
}
|
|
|
|
/*
|
|
* Check if this vmemmap page is already initialised. If any section
|
|
* which overlaps this vmemmap page is initialised then this page is
|
|
* initialised already.
|
|
*/
|
|
static int __meminit vmemmap_populated(unsigned long start, int page_size)
|
|
{
|
|
unsigned long end = start + page_size;
|
|
start = (unsigned long)(pfn_to_page(vmemmap_section_start(start)));
|
|
|
|
for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
|
|
if (pfn_valid(page_to_pfn((struct page *)start)))
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct vmemmap_backing *vmemmap_list;
|
|
static struct vmemmap_backing *next;
|
|
static int num_left;
|
|
static int num_freed;
|
|
|
|
static __meminit struct vmemmap_backing * vmemmap_list_alloc(int node)
|
|
{
|
|
struct vmemmap_backing *vmem_back;
|
|
/* get from freed entries first */
|
|
if (num_freed) {
|
|
num_freed--;
|
|
vmem_back = next;
|
|
next = next->list;
|
|
|
|
return vmem_back;
|
|
}
|
|
|
|
/* allocate a page when required and hand out chunks */
|
|
if (!num_left) {
|
|
next = vmemmap_alloc_block(PAGE_SIZE, node);
|
|
if (unlikely(!next)) {
|
|
WARN_ON(1);
|
|
return NULL;
|
|
}
|
|
num_left = PAGE_SIZE / sizeof(struct vmemmap_backing);
|
|
}
|
|
|
|
num_left--;
|
|
|
|
return next++;
|
|
}
|
|
|
|
static __meminit void vmemmap_list_populate(unsigned long phys,
|
|
unsigned long start,
|
|
int node)
|
|
{
|
|
struct vmemmap_backing *vmem_back;
|
|
|
|
vmem_back = vmemmap_list_alloc(node);
|
|
if (unlikely(!vmem_back)) {
|
|
WARN_ON(1);
|
|
return;
|
|
}
|
|
|
|
vmem_back->phys = phys;
|
|
vmem_back->virt_addr = start;
|
|
vmem_back->list = vmemmap_list;
|
|
|
|
vmemmap_list = vmem_back;
|
|
}
|
|
|
|
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
|
|
{
|
|
unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
|
|
|
|
/* Align to the page size of the linear mapping. */
|
|
start = _ALIGN_DOWN(start, page_size);
|
|
|
|
pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node);
|
|
|
|
for (; start < end; start += page_size) {
|
|
void *p;
|
|
int rc;
|
|
|
|
if (vmemmap_populated(start, page_size))
|
|
continue;
|
|
|
|
p = vmemmap_alloc_block(page_size, node);
|
|
if (!p)
|
|
return -ENOMEM;
|
|
|
|
vmemmap_list_populate(__pa(p), start, node);
|
|
|
|
pr_debug(" * %016lx..%016lx allocated at %p\n",
|
|
start, start + page_size, p);
|
|
|
|
rc = vmemmap_create_mapping(start, page_size, __pa(p));
|
|
if (rc < 0) {
|
|
pr_warning(
|
|
"vmemmap_populate: Unable to create vmemmap mapping: %d\n",
|
|
rc);
|
|
return -EFAULT;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
|
static unsigned long vmemmap_list_free(unsigned long start)
|
|
{
|
|
struct vmemmap_backing *vmem_back, *vmem_back_prev;
|
|
|
|
vmem_back_prev = vmem_back = vmemmap_list;
|
|
|
|
/* look for it with prev pointer recorded */
|
|
for (; vmem_back; vmem_back = vmem_back->list) {
|
|
if (vmem_back->virt_addr == start)
|
|
break;
|
|
vmem_back_prev = vmem_back;
|
|
}
|
|
|
|
if (unlikely(!vmem_back)) {
|
|
WARN_ON(1);
|
|
return 0;
|
|
}
|
|
|
|
/* remove it from vmemmap_list */
|
|
if (vmem_back == vmemmap_list) /* remove head */
|
|
vmemmap_list = vmem_back->list;
|
|
else
|
|
vmem_back_prev->list = vmem_back->list;
|
|
|
|
/* next point to this freed entry */
|
|
vmem_back->list = next;
|
|
next = vmem_back;
|
|
num_freed++;
|
|
|
|
return vmem_back->phys;
|
|
}
|
|
|
|
void __ref vmemmap_free(unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
|
|
|
|
start = _ALIGN_DOWN(start, page_size);
|
|
|
|
pr_debug("vmemmap_free %lx...%lx\n", start, end);
|
|
|
|
for (; start < end; start += page_size) {
|
|
unsigned long addr;
|
|
|
|
/*
|
|
* the section has already be marked as invalid, so
|
|
* vmemmap_populated() true means some other sections still
|
|
* in this page, so skip it.
|
|
*/
|
|
if (vmemmap_populated(start, page_size))
|
|
continue;
|
|
|
|
addr = vmemmap_list_free(start);
|
|
if (addr) {
|
|
struct page *page = pfn_to_page(addr >> PAGE_SHIFT);
|
|
|
|
if (PageReserved(page)) {
|
|
/* allocated from bootmem */
|
|
if (page_size < PAGE_SIZE) {
|
|
/*
|
|
* this shouldn't happen, but if it is
|
|
* the case, leave the memory there
|
|
*/
|
|
WARN_ON_ONCE(1);
|
|
} else {
|
|
unsigned int nr_pages =
|
|
1 << get_order(page_size);
|
|
while (nr_pages--)
|
|
free_reserved_page(page++);
|
|
}
|
|
} else
|
|
free_pages((unsigned long)(__va(addr)),
|
|
get_order(page_size));
|
|
|
|
vmemmap_remove_mapping(start, page_size);
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
void register_page_bootmem_memmap(unsigned long section_nr,
|
|
struct page *start_page, unsigned long size)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* We do not have access to the sparsemem vmemmap, so we fallback to
|
|
* walking the list of sparsemem blocks which we already maintain for
|
|
* the sake of crashdump. In the long run, we might want to maintain
|
|
* a tree if performance of that linear walk becomes a problem.
|
|
*
|
|
* realmode_pfn_to_page functions can fail due to:
|
|
* 1) As real sparsemem blocks do not lay in RAM continously (they
|
|
* are in virtual address space which is not available in the real mode),
|
|
* the requested page struct can be split between blocks so get_page/put_page
|
|
* may fail.
|
|
* 2) When huge pages are used, the get_page/put_page API will fail
|
|
* in real mode as the linked addresses in the page struct are virtual
|
|
* too.
|
|
*/
|
|
struct page *realmode_pfn_to_page(unsigned long pfn)
|
|
{
|
|
struct vmemmap_backing *vmem_back;
|
|
struct page *page;
|
|
unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
|
|
unsigned long pg_va = (unsigned long) pfn_to_page(pfn);
|
|
|
|
for (vmem_back = vmemmap_list; vmem_back; vmem_back = vmem_back->list) {
|
|
if (pg_va < vmem_back->virt_addr)
|
|
continue;
|
|
|
|
/* After vmemmap_list entry free is possible, need check all */
|
|
if ((pg_va + sizeof(struct page)) <=
|
|
(vmem_back->virt_addr + page_size)) {
|
|
page = (struct page *) (vmem_back->phys + pg_va -
|
|
vmem_back->virt_addr);
|
|
return page;
|
|
}
|
|
}
|
|
|
|
/* Probably that page struct is split between real pages */
|
|
return NULL;
|
|
}
|
|
EXPORT_SYMBOL_GPL(realmode_pfn_to_page);
|
|
|
|
#elif defined(CONFIG_FLATMEM)
|
|
|
|
struct page *realmode_pfn_to_page(unsigned long pfn)
|
|
{
|
|
struct page *page = pfn_to_page(pfn);
|
|
return page;
|
|
}
|
|
EXPORT_SYMBOL_GPL(realmode_pfn_to_page);
|
|
|
|
#endif /* CONFIG_SPARSEMEM_VMEMMAP/CONFIG_FLATMEM */
|
|
|
|
#ifdef CONFIG_PPC_STD_MMU_64
|
|
static bool disable_radix;
|
|
static int __init parse_disable_radix(char *p)
|
|
{
|
|
disable_radix = true;
|
|
return 0;
|
|
}
|
|
early_param("disable_radix", parse_disable_radix);
|
|
|
|
/*
|
|
* If we're running under a hypervisor, we need to check the contents of
|
|
* /chosen/ibm,architecture-vec-5 to see if the hypervisor is willing to do
|
|
* radix. If not, we clear the radix feature bit so we fall back to hash.
|
|
*/
|
|
static void early_check_vec5(void)
|
|
{
|
|
unsigned long root, chosen;
|
|
int size;
|
|
const u8 *vec5;
|
|
u8 mmu_supported;
|
|
|
|
root = of_get_flat_dt_root();
|
|
chosen = of_get_flat_dt_subnode_by_name(root, "chosen");
|
|
if (chosen == -FDT_ERR_NOTFOUND) {
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
|
|
return;
|
|
}
|
|
vec5 = of_get_flat_dt_prop(chosen, "ibm,architecture-vec-5", &size);
|
|
if (!vec5) {
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
|
|
return;
|
|
}
|
|
if (size <= OV5_INDX(OV5_MMU_SUPPORT)) {
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
|
|
return;
|
|
}
|
|
|
|
/* Check for supported configuration */
|
|
mmu_supported = vec5[OV5_INDX(OV5_MMU_SUPPORT)] &
|
|
OV5_FEAT(OV5_MMU_SUPPORT);
|
|
if (mmu_supported == OV5_FEAT(OV5_MMU_RADIX)) {
|
|
/* Hypervisor only supports radix - check enabled && GTSE */
|
|
if (!early_radix_enabled()) {
|
|
pr_warn("WARNING: Ignoring cmdline option disable_radix\n");
|
|
}
|
|
if (!(vec5[OV5_INDX(OV5_RADIX_GTSE)] &
|
|
OV5_FEAT(OV5_RADIX_GTSE))) {
|
|
pr_warn("WARNING: Hypervisor doesn't support RADIX with GTSE\n");
|
|
}
|
|
/* Do radix anyway - the hypervisor said we had to */
|
|
cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX;
|
|
} else if (mmu_supported == OV5_FEAT(OV5_MMU_HASH)) {
|
|
/* Hypervisor only supports hash - disable radix */
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
|
|
}
|
|
}
|
|
|
|
void __init mmu_early_init_devtree(void)
|
|
{
|
|
/* Disable radix mode based on kernel command line. */
|
|
if (disable_radix)
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
|
|
|
|
/*
|
|
* Check /chosen/ibm,architecture-vec-5 if running as a guest.
|
|
* When running bare-metal, we can use radix if we like
|
|
* even though the ibm,architecture-vec-5 property created by
|
|
* skiboot doesn't have the necessary bits set.
|
|
*/
|
|
if (!(mfmsr() & MSR_HV))
|
|
early_check_vec5();
|
|
|
|
if (early_radix_enabled())
|
|
radix__early_init_devtree();
|
|
else
|
|
hash__early_init_devtree();
|
|
}
|
|
#endif /* CONFIG_PPC_STD_MMU_64 */
|