2019-05-27 06:55:05 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
*
|
|
|
|
* Copyright (C) 2000, 2001 Kanoj Sarcar
|
|
|
|
* Copyright (C) 2000, 2001 Ralf Baechle
|
|
|
|
* Copyright (C) 2000, 2001 Silicon Graphics, Inc.
|
|
|
|
* Copyright (C) 2000, 2001, 2003 Broadcom Corporation
|
|
|
|
*/
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/interrupt.h>
|
2009-06-19 13:05:26 +00:00
|
|
|
#include <linux/smp.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/threads.h>
|
2016-08-21 19:58:13 +00:00
|
|
|
#include <linux/export.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/time.h>
|
|
|
|
#include <linux/timex.h>
|
2017-02-03 23:16:44 +00:00
|
|
|
#include <linux/sched/mm.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/cpumask.h>
|
2006-02-20 13:35:27 +00:00
|
|
|
#include <linux/cpu.h>
|
2007-07-29 22:36:13 +00:00
|
|
|
#include <linux/err.h>
|
2009-11-20 12:34:33 +00:00
|
|
|
#include <linux/ftrace.h>
|
2015-12-08 13:20:27 +00:00
|
|
|
#include <linux/irqdomain.h>
|
|
|
|
#include <linux/of.h>
|
|
|
|
#include <linux/of_irq.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-07-26 23:09:06 +00:00
|
|
|
#include <linux/atomic.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/cpu.h>
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
#include <asm/ginvt.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/processor.h>
|
2013-05-21 14:59:19 +00:00
|
|
|
#include <asm/idle.h>
|
2008-04-28 16:14:26 +00:00
|
|
|
#include <asm/r4k-timer.h>
|
2017-10-16 10:06:49 +00:00
|
|
|
#include <asm/mips-cps.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/mmu_context.h>
|
2007-10-11 22:46:09 +00:00
|
|
|
#include <asm/time.h>
|
2012-03-28 17:30:02 +00:00
|
|
|
#include <asm/setup.h>
|
2015-09-25 15:59:38 +00:00
|
|
|
#include <asm/maar.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-09-28 17:34:04 +00:00
|
|
|
int __cpu_number_map[CONFIG_MIPS_NR_CPU_NR_MAP]; /* Map physical to logical */
|
2010-07-24 01:41:45 +00:00
|
|
|
EXPORT_SYMBOL(__cpu_number_map);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
|
2010-07-24 01:41:45 +00:00
|
|
|
EXPORT_SYMBOL(__cpu_logical_map);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-02 20:42:04 +00:00
|
|
|
/* Number of TCs (or siblings in Intel speak) per CPU core */
|
|
|
|
int smp_num_siblings = 1;
|
|
|
|
EXPORT_SYMBOL(smp_num_siblings);
|
|
|
|
|
|
|
|
/* representing the TCs (or siblings in Intel speak) of each logical CPU */
|
|
|
|
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
|
|
|
|
EXPORT_SYMBOL(cpu_sibling_map);
|
|
|
|
|
2014-06-26 03:41:26 +00:00
|
|
|
/* representing the core map of multi-core chips of each logical CPU */
|
|
|
|
cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
|
|
|
|
EXPORT_SYMBOL(cpu_core_map);
|
|
|
|
|
2017-09-27 09:13:25 +00:00
|
|
|
static DECLARE_COMPLETION(cpu_starting);
|
2016-11-04 09:28:56 +00:00
|
|
|
static DECLARE_COMPLETION(cpu_running);
|
|
|
|
|
2015-07-10 08:29:10 +00:00
|
|
|
/*
|
|
|
|
* A logcal cpu mask containing only one VPE per core to
|
|
|
|
* reduce the number of IPIs on large MT systems.
|
|
|
|
*/
|
2016-07-13 13:12:52 +00:00
|
|
|
cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
|
2015-07-10 08:29:10 +00:00
|
|
|
EXPORT_SYMBOL(cpu_foreign_map);
|
|
|
|
|
2007-03-02 20:42:04 +00:00
|
|
|
/* representing cpus for which sibling maps can be computed */
|
|
|
|
static cpumask_t cpu_sibling_setup_map;
|
|
|
|
|
2014-06-26 03:41:26 +00:00
|
|
|
/* representing cpus for which core maps can be computed */
|
|
|
|
static cpumask_t cpu_core_setup_map;
|
|
|
|
|
2014-02-14 16:30:52 +00:00
|
|
|
cpumask_t cpu_coherent_mask;
|
|
|
|
|
2015-12-08 13:20:27 +00:00
|
|
|
#ifdef CONFIG_GENERIC_IRQ_IPI
|
|
|
|
static struct irq_desc *call_desc;
|
|
|
|
static struct irq_desc *sched_desc;
|
|
|
|
#endif
|
|
|
|
|
2007-03-02 20:42:04 +00:00
|
|
|
static inline void set_cpu_sibling_map(int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
|
2007-03-02 20:42:04 +00:00
|
|
|
|
|
|
|
if (smp_num_siblings > 1) {
|
2015-03-05 00:19:17 +00:00
|
|
|
for_each_cpu(i, &cpu_sibling_setup_map) {
|
2017-08-13 02:49:37 +00:00
|
|
|
if (cpus_are_siblings(cpu, i)) {
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
|
|
|
|
cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
|
2007-03-02 20:42:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} else
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
|
2007-03-02 20:42:04 +00:00
|
|
|
}
|
|
|
|
|
2014-06-26 03:41:26 +00:00
|
|
|
static inline void set_cpu_core_map(int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(cpu, &cpu_core_setup_map);
|
2014-06-26 03:41:26 +00:00
|
|
|
|
2015-03-05 00:19:17 +00:00
|
|
|
for_each_cpu(i, &cpu_core_setup_map) {
|
2014-06-26 03:41:26 +00:00
|
|
|
if (cpu_data[cpu].package == cpu_data[i].package) {
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(i, &cpu_core_map[cpu]);
|
|
|
|
cpumask_set_cpu(cpu, &cpu_core_map[i]);
|
2014-06-26 03:41:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-10 08:29:10 +00:00
|
|
|
/*
|
|
|
|
* Calculate a new cpu_foreign_map mask whenever a
|
|
|
|
* new cpu appears or disappears.
|
|
|
|
*/
|
2016-07-13 13:12:45 +00:00
|
|
|
void calculate_cpu_foreign_map(void)
|
2015-07-10 08:29:10 +00:00
|
|
|
{
|
|
|
|
int i, k, core_present;
|
|
|
|
cpumask_t temp_foreign_map;
|
|
|
|
|
|
|
|
/* Re-calculate the mask */
|
2016-03-04 10:10:51 +00:00
|
|
|
cpumask_clear(&temp_foreign_map);
|
2015-07-10 08:29:10 +00:00
|
|
|
for_each_online_cpu(i) {
|
|
|
|
core_present = 0;
|
|
|
|
for_each_cpu(k, &temp_foreign_map)
|
2017-08-13 02:49:37 +00:00
|
|
|
if (cpus_are_siblings(i, k))
|
2015-07-10 08:29:10 +00:00
|
|
|
core_present = 1;
|
|
|
|
if (!core_present)
|
|
|
|
cpumask_set_cpu(i, &temp_foreign_map);
|
|
|
|
}
|
|
|
|
|
2016-07-13 13:12:52 +00:00
|
|
|
for_each_online_cpu(i)
|
|
|
|
cpumask_andnot(&cpu_foreign_map[i],
|
|
|
|
&temp_foreign_map, &cpu_sibling_map[i]);
|
2015-07-10 08:29:10 +00:00
|
|
|
}
|
|
|
|
|
2017-07-19 08:21:03 +00:00
|
|
|
const struct plat_smp_ops *mp_ops;
|
2012-11-22 02:34:14 +00:00
|
|
|
EXPORT_SYMBOL(mp_ops);
|
2007-11-19 12:23:51 +00:00
|
|
|
|
2017-07-19 08:21:03 +00:00
|
|
|
void register_smp_ops(const struct plat_smp_ops *ops)
|
2007-11-19 12:23:51 +00:00
|
|
|
{
|
2008-05-06 10:21:22 +00:00
|
|
|
if (mp_ops)
|
|
|
|
printk(KERN_WARNING "Overriding previously set SMP ops\n");
|
2007-11-19 12:23:51 +00:00
|
|
|
|
|
|
|
mp_ops = ops;
|
|
|
|
}
|
|
|
|
|
2015-12-08 13:20:27 +00:00
|
|
|
#ifdef CONFIG_GENERIC_IRQ_IPI
|
|
|
|
void mips_smp_send_ipi_single(int cpu, unsigned int action)
|
|
|
|
{
|
|
|
|
mips_smp_send_ipi_mask(cpumask_of(cpu), action);
|
|
|
|
}
|
|
|
|
|
|
|
|
void mips_smp_send_ipi_mask(const struct cpumask *mask, unsigned int action)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
unsigned int core;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case SMP_CALL_FUNCTION:
|
|
|
|
__ipi_send_mask(call_desc, mask);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case SMP_RESCHEDULE_YOURSELF:
|
|
|
|
__ipi_send_mask(sched_desc, mask);
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (mips_cpc_present()) {
|
|
|
|
for_each_cpu(cpu, mask) {
|
2017-08-13 02:49:37 +00:00
|
|
|
if (cpus_are_siblings(cpu, smp_processor_id()))
|
2015-12-08 13:20:27 +00:00
|
|
|
continue;
|
|
|
|
|
2017-08-13 02:49:37 +00:00
|
|
|
core = cpu_core(&cpu_data[cpu]);
|
|
|
|
|
2015-12-08 13:20:27 +00:00
|
|
|
while (!cpumask_test_cpu(cpu, &cpu_coherent_mask)) {
|
2017-08-13 02:49:39 +00:00
|
|
|
mips_cm_lock_other_cpu(cpu, CM_GCR_Cx_OTHER_BLOCK_LOCAL);
|
2015-12-08 13:20:27 +00:00
|
|
|
mips_cpc_lock_other(core);
|
|
|
|
write_cpc_co_cmd(CPC_Cx_CMD_PWRUP);
|
|
|
|
mips_cpc_unlock_other();
|
2016-09-07 09:45:19 +00:00
|
|
|
mips_cm_unlock_other();
|
2015-12-08 13:20:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static irqreturn_t ipi_resched_interrupt(int irq, void *dev_id)
|
|
|
|
{
|
|
|
|
scheduler_ipi();
|
|
|
|
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t ipi_call_interrupt(int irq, void *dev_id)
|
|
|
|
{
|
|
|
|
generic_smp_call_function_interrupt();
|
|
|
|
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct irqaction irq_resched = {
|
|
|
|
.handler = ipi_resched_interrupt,
|
|
|
|
.flags = IRQF_PERCPU,
|
|
|
|
.name = "IPI resched"
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct irqaction irq_call = {
|
|
|
|
.handler = ipi_call_interrupt,
|
|
|
|
.flags = IRQF_PERCPU,
|
|
|
|
.name = "IPI call"
|
|
|
|
};
|
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
static void smp_ipi_init_one(unsigned int virq,
|
2015-12-08 13:20:27 +00:00
|
|
|
struct irqaction *action)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
irq_set_handler(virq, handle_percpu_irq);
|
|
|
|
ret = setup_irq(virq, action);
|
|
|
|
BUG_ON(ret);
|
|
|
|
}
|
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
static unsigned int call_virq, sched_virq;
|
|
|
|
|
|
|
|
int mips_smp_ipi_allocate(const struct cpumask *mask)
|
2015-12-08 13:20:27 +00:00
|
|
|
{
|
2016-09-20 08:47:26 +00:00
|
|
|
int virq;
|
2015-12-08 13:20:27 +00:00
|
|
|
struct irq_domain *ipidomain;
|
|
|
|
struct device_node *node;
|
|
|
|
|
|
|
|
node = of_irq_find_parent(of_root);
|
|
|
|
ipidomain = irq_find_matching_host(node, DOMAIN_BUS_IPI);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some platforms have half DT setup. So if we found irq node but
|
|
|
|
* didn't find an ipidomain, try to search for one that is not in the
|
|
|
|
* DT.
|
|
|
|
*/
|
|
|
|
if (node && !ipidomain)
|
|
|
|
ipidomain = irq_find_matching_host(NULL, DOMAIN_BUS_IPI);
|
|
|
|
|
2016-04-04 09:04:52 +00:00
|
|
|
/*
|
2017-03-30 19:06:13 +00:00
|
|
|
* There are systems which use IPI IRQ domains, but only have one
|
|
|
|
* registered when some runtime condition is met. For example a Malta
|
|
|
|
* kernel may include support for GIC & CPU interrupt controller IPI
|
|
|
|
* IRQ domains, but if run on a system with no GIC & no MT ASE then
|
|
|
|
* neither will be supported or registered.
|
|
|
|
*
|
|
|
|
* We only have a problem if we're actually using multiple CPUs so fail
|
|
|
|
* loudly if that is the case. Otherwise simply return, skipping IPI
|
|
|
|
* setup, if we're running with only a single CPU.
|
2016-04-04 09:04:52 +00:00
|
|
|
*/
|
2017-03-30 19:06:13 +00:00
|
|
|
if (!ipidomain) {
|
|
|
|
BUG_ON(num_present_cpus() > 1);
|
2016-04-04 09:04:52 +00:00
|
|
|
return 0;
|
2017-03-30 19:06:13 +00:00
|
|
|
}
|
2015-12-08 13:20:27 +00:00
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
virq = irq_reserve_ipi(ipidomain, mask);
|
|
|
|
BUG_ON(!virq);
|
|
|
|
if (!call_virq)
|
|
|
|
call_virq = virq;
|
2015-12-08 13:20:27 +00:00
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
virq = irq_reserve_ipi(ipidomain, mask);
|
|
|
|
BUG_ON(!virq);
|
|
|
|
if (!sched_virq)
|
|
|
|
sched_virq = virq;
|
2015-12-08 13:20:27 +00:00
|
|
|
|
|
|
|
if (irq_domain_is_ipi_per_cpu(ipidomain)) {
|
|
|
|
int cpu;
|
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
for_each_cpu(cpu, mask) {
|
2015-12-08 13:20:27 +00:00
|
|
|
smp_ipi_init_one(call_virq + cpu, &irq_call);
|
|
|
|
smp_ipi_init_one(sched_virq + cpu, &irq_resched);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
smp_ipi_init_one(call_virq, &irq_call);
|
|
|
|
smp_ipi_init_one(sched_virq, &irq_resched);
|
|
|
|
}
|
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int mips_smp_ipi_free(const struct cpumask *mask)
|
|
|
|
{
|
|
|
|
struct irq_domain *ipidomain;
|
|
|
|
struct device_node *node;
|
|
|
|
|
|
|
|
node = of_irq_find_parent(of_root);
|
|
|
|
ipidomain = irq_find_matching_host(node, DOMAIN_BUS_IPI);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some platforms have half DT setup. So if we found irq node but
|
|
|
|
* didn't find an ipidomain, try to search for one that is not in the
|
|
|
|
* DT.
|
|
|
|
*/
|
|
|
|
if (node && !ipidomain)
|
|
|
|
ipidomain = irq_find_matching_host(NULL, DOMAIN_BUS_IPI);
|
|
|
|
|
|
|
|
BUG_ON(!ipidomain);
|
|
|
|
|
|
|
|
if (irq_domain_is_ipi_per_cpu(ipidomain)) {
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for_each_cpu(cpu, mask) {
|
|
|
|
remove_irq(call_virq + cpu, &irq_call);
|
|
|
|
remove_irq(sched_virq + cpu, &irq_resched);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
irq_destroy_ipi(call_virq, mask);
|
|
|
|
irq_destroy_ipi(sched_virq, mask);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static int __init mips_smp_ipi_init(void)
|
|
|
|
{
|
2017-06-02 21:48:49 +00:00
|
|
|
if (num_possible_cpus() == 1)
|
|
|
|
return 0;
|
|
|
|
|
2016-09-20 08:47:26 +00:00
|
|
|
mips_smp_ipi_allocate(cpu_possible_mask);
|
|
|
|
|
2015-12-08 13:20:27 +00:00
|
|
|
call_desc = irq_to_desc(call_virq);
|
|
|
|
sched_desc = irq_to_desc(sched_virq);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_initcall(mips_smp_ipi_init);
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* First C code run on the secondary CPUs after being started up by
|
|
|
|
* the master.
|
|
|
|
*/
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 13:38:59 +00:00
|
|
|
asmlinkage void start_secondary(void)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2005-11-09 05:39:01 +00:00
|
|
|
unsigned int cpu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
cpu_probe();
|
2012-05-15 07:04:50 +00:00
|
|
|
per_cpu_trap_init(false);
|
2007-10-11 22:46:09 +00:00
|
|
|
mips_clockevent_init();
|
2007-11-19 12:23:51 +00:00
|
|
|
mp_ops->init_secondary();
|
2015-01-15 21:01:59 +00:00
|
|
|
cpu_report();
|
2015-09-25 15:59:38 +00:00
|
|
|
maar_init();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* XXX parity protection should be folded in here when it's converted
|
|
|
|
* to an option instead of something based on .cputype
|
|
|
|
*/
|
|
|
|
|
|
|
|
calibrate_delay();
|
2005-11-09 05:39:01 +00:00
|
|
|
preempt_disable();
|
|
|
|
cpu = smp_processor_id();
|
2005-04-16 22:20:36 +00:00
|
|
|
cpu_data[cpu].udelay_val = loops_per_jiffy;
|
|
|
|
|
2015-03-05 00:19:17 +00:00
|
|
|
cpumask_set_cpu(cpu, &cpu_coherent_mask);
|
2008-09-07 14:57:22 +00:00
|
|
|
notify_cpu_starting(cpu);
|
|
|
|
|
2017-09-27 09:13:25 +00:00
|
|
|
/* Notify boot CPU that we're starting & ready to sync counters */
|
|
|
|
complete(&cpu_starting);
|
|
|
|
|
|
|
|
synchronise_count_slave(cpu);
|
|
|
|
|
|
|
|
/* The CPU is running and counters synchronised, now mark it online */
|
2012-07-19 07:13:53 +00:00
|
|
|
set_cpu_online(cpu, true);
|
|
|
|
|
2007-03-02 20:42:04 +00:00
|
|
|
set_cpu_sibling_map(cpu);
|
2014-06-26 03:41:26 +00:00
|
|
|
set_cpu_core_map(cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-07-10 08:29:10 +00:00
|
|
|
calculate_cpu_foreign_map();
|
|
|
|
|
2017-09-27 09:13:25 +00:00
|
|
|
/*
|
|
|
|
* Notify boot CPU that we're up & online and it can safely return
|
|
|
|
* from __cpu_up
|
|
|
|
*/
|
MIPS: Fix race on setting and getting cpu_online_mask
While testing cpu hoptlug (cpu down and up in loops) on kernel 4.4, it was
observed that occasionally check for cpu online will fail in kernel/cpu.c,
_cpu_up:
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/tree/kernel/cpu.c?h=v4.4.79#n485
518 /* Arch-specific enabling code. */
519 ret = __cpu_up(cpu, idle);
520
521 if (ret != 0)
522 goto out_notify;
523 BUG_ON(!cpu_online(cpu));
Reason is race between start_secondary and _cpu_up. cpu_callin_map is set
before cpu_online_mask. In __cpu_up, cpu_callin_map is waited for, but cpu
online mask is not, resulting in race in which secondary processor started
and set cpu_callin_map, but not yet set the online mask,resulting in above
BUG being hit.
Upstream differs in the area. cpu_online check is in bringup_wait_for_ap,
which is after cpu reached AP_ONLINE_IDLE,where secondary passed its start
function. Nonetheless, fix makes start_secondary safe and not depending on
other locks throughout the code. It protects as well against cpu_online
checks put in between sometimes in the future.
Fix this by moving completion after all flags are set.
Signed-off-by: Matija Glavinic Pecotic <matija.glavinic-pecotic.ext@nokia.com>
Cc: Alexander Sverdlin <alexander.sverdlin@nokia.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/16925/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2017-08-03 06:20:22 +00:00
|
|
|
complete(&cpu_running);
|
|
|
|
|
2012-07-19 07:13:53 +00:00
|
|
|
/*
|
|
|
|
* irq will be enabled in ->smp_finish(), enabling it too early
|
|
|
|
* is dangerous.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(!irqs_disabled());
|
2012-07-19 07:13:53 +00:00
|
|
|
mp_ops->smp_finish();
|
|
|
|
|
2016-02-26 18:43:40 +00:00
|
|
|
cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void stop_this_cpu(void *dummy)
|
|
|
|
{
|
|
|
|
/*
|
2016-07-13 13:12:46 +00:00
|
|
|
* Remove this CPU:
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2015-07-10 08:29:10 +00:00
|
|
|
|
2012-03-29 05:08:30 +00:00
|
|
|
set_cpu_online(smp_processor_id(), false);
|
2015-07-10 08:29:10 +00:00
|
|
|
calculate_cpu_foreign_map();
|
2015-03-25 17:25:43 +00:00
|
|
|
local_irq_disable();
|
|
|
|
while (1);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void smp_send_stop(void)
|
|
|
|
{
|
2008-06-06 09:18:06 +00:00
|
|
|
smp_call_function(stop_this_cpu, NULL, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __init smp_cpus_done(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
/* called from main before smp_init() */
|
|
|
|
void __init smp_prepare_cpus(unsigned int max_cpus)
|
|
|
|
{
|
|
|
|
init_new_context(current, &init_mm);
|
|
|
|
current_thread_info()->cpu = 0;
|
2007-11-19 12:23:51 +00:00
|
|
|
mp_ops->prepare_cpus(max_cpus);
|
2007-03-02 20:42:04 +00:00
|
|
|
set_cpu_sibling_map(0);
|
2014-06-26 03:41:26 +00:00
|
|
|
set_cpu_core_map(0);
|
2015-07-10 08:29:10 +00:00
|
|
|
calculate_cpu_foreign_map();
|
2006-05-22 13:24:04 +00:00
|
|
|
#ifndef CONFIG_HOTPLUG_CPU
|
2012-03-29 05:08:30 +00:00
|
|
|
init_cpu_present(cpu_possible_mask);
|
2006-05-22 13:24:04 +00:00
|
|
|
#endif
|
2014-02-14 16:30:52 +00:00
|
|
|
cpumask_copy(&cpu_coherent_mask, cpu_possible_mask);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* preload SMP state for boot cpu */
|
2012-12-21 22:04:39 +00:00
|
|
|
void smp_prepare_boot_cpu(void)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2019-02-19 15:57:19 +00:00
|
|
|
if (mp_ops->prepare_boot_cpu)
|
|
|
|
mp_ops->prepare_boot_cpu();
|
2009-09-24 15:34:47 +00:00
|
|
|
set_cpu_possible(0, true);
|
|
|
|
set_cpu_online(0, true);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
MIPS: Delete __cpuinit/__CPUINIT usage from MIPS code
commit 3747069b25e419f6b51395f48127e9812abc3596 upstream.
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
and are flagged as __cpuinit -- so if we remove the __cpuinit from
the arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
related content into no-ops as early as possible, since that will get
rid of these warnings. In any case, they are temporary and harmless.
Here, we remove all the MIPS __cpuinit from C code and __CPUINIT
from asm files. MIPS is interesting in this respect, because there
are also uasm users hiding behind their own renamed versions of the
__cpuinit macros.
[1] https://lkml.org/lkml/2013/5/20/589
[ralf@linux-mips.org: Folded in Paul's followup fix.]
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/5494/
Patchwork: https://patchwork.linux-mips.org/patch/5495/
Patchwork: https://patchwork.linux-mips.org/patch/5509/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2013-06-18 13:38:59 +00:00
|
|
|
int __cpu_up(unsigned int cpu, struct task_struct *tidle)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2017-08-13 02:49:40 +00:00
|
|
|
int err;
|
|
|
|
|
|
|
|
err = mp_ops->boot_secondary(cpu, tidle);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-09-27 09:13:25 +00:00
|
|
|
/* Wait for CPU to start and be ready to sync counters */
|
|
|
|
if (!wait_for_completion_timeout(&cpu_starting,
|
2016-11-04 09:28:56 +00:00
|
|
|
msecs_to_jiffies(1000))) {
|
|
|
|
pr_crit("CPU%u: failed to start\n", cpu);
|
|
|
|
return -EIO;
|
MIPS: SMP: Fix build error.
CC arch/mips/kernel/smp.o
arch/mips/kernel/smp.c: In function ‘start_secondary’:
arch/mips/kernel/smp.c:149:2: error: passing argument 2 of ‘cpumask_set_cpu’ discards ‘volatile’ qualifier from pointer target type [-Werror]
cpumask_set_cpu(cpu, &cpu_callin_map);
^
In file included from ./arch/mips/include/asm/processor.h:14:0,
from ./arch/mips/include/asm/thread_info.h:15,
from include/linux/thread_info.h:54,
from include/asm-generic/preempt.h:4,
from arch/mips/include/generated/asm/preempt.h:1,
from include/linux/preempt.h:18,
from include/linux/interrupt.h:8,
from arch/mips/kernel/smp.c:24:
include/linux/cpumask.h:272:91: note: expected ‘struct cpumask *’ but argument is of type ‘volatile struct cpumask_t *’
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
^
arch/mips/kernel/smp.c: In function ‘smp_prepare_boot_cpu’:
arch/mips/kernel/smp.c:211:2: error: passing argument 2 of ‘cpumask_set_cpu’ discards ‘volatile’ qualifier from pointer target type [-Werror]
cpumask_set_cpu(0, &cpu_callin_map);
^
In file included from ./arch/mips/include/asm/processor.h:14:0,
from ./arch/mips/include/asm/thread_info.h:15,
from include/linux/thread_info.h:54,
from include/asm-generic/preempt.h:4,
from arch/mips/include/generated/asm/preempt.h:1,
from include/linux/preempt.h:18,
from include/linux/interrupt.h:8,
from arch/mips/kernel/smp.c:24:
include/linux/cpumask.h:272:91: note: expected ‘struct cpumask *’ but argument is of type ‘volatile struct cpumask_t *’
static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
^
arch/mips/kernel/smp.c: In function ‘__cpu_up’:
arch/mips/kernel/smp.c:221:10: error: passing argument 2 of ‘cpumask_test_cpu’ discards ‘volatile’ qualifier from pointer target type [-Werror]
while (!cpumask_test_cpu(cpu, &cpu_callin_map))
^
In file included from ./arch/mips/include/asm/processor.h:14:0,
from ./arch/mips/include/asm/thread_info.h:15,
from include/linux/thread_info.h:54,
from include/asm-generic/preempt.h:4,
from arch/mips/include/generated/asm/preempt.h:1,
from include/linux/preempt.h:18,
from include/linux/interrupt.h:8,
from arch/mips/kernel/smp.c:24:
include/linux/cpumask.h:294:90: note: expected ‘const struct cpumask *’ but argument is of type ‘volatile struct cpumask_t *’
static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
^
cc1: all warnings being treated as errors
make[2]: *** [arch/mips/kernel/smp.o] Error 1
make[1]: *** [arch/mips/kernel] Error 2
make: *** [arch/mips] Error 2
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2015-05-12 04:43:04 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-08-14 13:26:13 +00:00
|
|
|
synchronise_count_master(cpu);
|
2017-09-27 09:13:25 +00:00
|
|
|
|
|
|
|
/* Wait for CPU to finish startup & mark itself online before return */
|
|
|
|
wait_for_completion(&cpu_running);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Not really SMP stuff ... */
|
|
|
|
int setup_profiling_timer(unsigned int multiplier)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flush_tlb_all_ipi(void *info)
|
|
|
|
{
|
|
|
|
local_flush_tlb_all();
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tlb_all(void)
|
|
|
|
{
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
if (cpu_has_mmid) {
|
|
|
|
htw_stop();
|
|
|
|
ginvt_full();
|
|
|
|
sync_ginv();
|
|
|
|
instruction_hazard();
|
|
|
|
htw_start();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2008-05-09 07:39:44 +00:00
|
|
|
on_each_cpu(flush_tlb_all_ipi, NULL, 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flush_tlb_mm_ipi(void *mm)
|
|
|
|
{
|
2019-02-02 01:43:22 +00:00
|
|
|
drop_mmu_context((struct mm_struct *)mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-06-22 21:42:32 +00:00
|
|
|
/*
|
|
|
|
* Special Variant of smp_call_function for use by TLB functions:
|
|
|
|
*
|
|
|
|
* o No return value
|
|
|
|
* o collapses to normal function call on UP kernels
|
|
|
|
* o collapses to normal function call on systems with a single shared
|
|
|
|
* primary cache.
|
|
|
|
*/
|
|
|
|
static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
|
|
|
|
{
|
2008-06-06 09:18:06 +00:00
|
|
|
smp_call_function(func, info, 1);
|
2006-06-22 21:42:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
|
|
|
|
smp_on_other_tlbs(func, info);
|
|
|
|
func(info);
|
|
|
|
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* The following tlb flush calls are invoked when old translations are
|
|
|
|
* being torn down, or pte attributes are changing. For single threaded
|
|
|
|
* address spaces, a new context is obtained on the current cpu, and tlb
|
|
|
|
* context on other cpus are invalidated to force a new context allocation
|
|
|
|
* at switch_mm time, should the mm ever be used on other cpus. For
|
|
|
|
* multithreaded address spaces, intercpu interrupts have to be sent.
|
|
|
|
* Another case where intercpu interrupts are required is when the target
|
|
|
|
* mm might be active on another cpu (eg debuggers doing the flushes on
|
|
|
|
* behalf of debugees, kswapd stealing pages from another process etc).
|
|
|
|
* Kanoj 07/00.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void flush_tlb_mm(struct mm_struct *mm)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
if (cpu_has_mmid) {
|
|
|
|
/*
|
|
|
|
* No need to worry about other CPUs - the ginvt in
|
|
|
|
* drop_mmu_context() will be globalized.
|
|
|
|
*/
|
|
|
|
} else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
|
2007-10-04 15:57:08 +00:00
|
|
|
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2007-10-03 18:16:57 +00:00
|
|
|
unsigned int cpu;
|
|
|
|
|
2012-03-29 05:08:30 +00:00
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
if (cpu != smp_processor_id() && cpu_context(cpu, mm))
|
2019-02-02 01:43:25 +00:00
|
|
|
set_cpu_context(cpu, mm, 0);
|
2012-03-29 05:08:30 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2019-02-02 01:43:22 +00:00
|
|
|
drop_mmu_context(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
struct flush_tlb_data {
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
unsigned long addr1;
|
|
|
|
unsigned long addr2;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void flush_tlb_range_ipi(void *info)
|
|
|
|
{
|
2007-10-04 15:57:08 +00:00
|
|
|
struct flush_tlb_data *fd = info;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
unsigned long addr;
|
|
|
|
u32 old_mmid;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
preempt_disable();
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
if (cpu_has_mmid) {
|
|
|
|
htw_stop();
|
|
|
|
old_mmid = read_c0_memorymapid();
|
|
|
|
write_c0_memorymapid(cpu_asid(0, mm));
|
|
|
|
mtc0_tlbw_hazard();
|
|
|
|
addr = round_down(start, PAGE_SIZE * 2);
|
|
|
|
end = round_up(end, PAGE_SIZE * 2);
|
|
|
|
do {
|
|
|
|
ginvt_va_mmid(addr);
|
|
|
|
sync_ginv();
|
|
|
|
addr += PAGE_SIZE * 2;
|
|
|
|
} while (addr < end);
|
|
|
|
write_c0_memorymapid(old_mmid);
|
|
|
|
instruction_hazard();
|
|
|
|
htw_start();
|
|
|
|
} else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
|
2007-10-04 17:18:52 +00:00
|
|
|
struct flush_tlb_data fd = {
|
|
|
|
.vma = vma,
|
|
|
|
.addr1 = start,
|
|
|
|
.addr2 = end,
|
|
|
|
};
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-04 15:57:08 +00:00
|
|
|
smp_on_other_tlbs(flush_tlb_range_ipi, &fd);
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
local_flush_tlb_range(vma, start, end);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2007-10-03 18:16:57 +00:00
|
|
|
unsigned int cpu;
|
MIPS: SMP: Clear ASID without confusing has_valid_asid()
The SMP flush_tlb_*() functions may clear the memory map's ASIDs for
other CPUs if the mm has only a single user (the current CPU) in order
to avoid SMP calls. However this makes it appear to has_valid_asid(),
which is used by various cache flush functions, as if the CPUs have
never run in the mm, and therefore can't have cached any of its memory.
For flush_tlb_mm() this doesn't sound unreasonable.
flush_tlb_range() corresponds to flush_cache_range() which does do full
indexed cache flushes, but only on the icache if the specified mapping
is executable, otherwise it doesn't guarantee that there are no cache
contents left for the mm.
flush_tlb_page() corresponds to flush_cache_page(), which will perform
address based cache ops on the specified page only, and also only
touches the icache if the page is executable. It does not guarantee that
there are no cache contents left for the mm.
For example, this affects flush_cache_range() which uses the
has_valid_asid() optimisation. It is required to flush the icache when
mappings are made executable (e.g. using mprotect) so they are
immediately usable. If some code is changed to non executable in order
to be modified then it will not be flushed from the icache during that
time, but the ASID on other CPUs may still be cleared for TLB flushing.
When the code is changed back to executable, flush_cache_range() will
assume the code hasn't run on those other CPUs due to the zero ASID, and
won't invalidate the icache on them.
This is fixed by clearing the other CPUs ASIDs to 1 instead of 0 for the
above two flush_tlb_*() functions when the corresponding cache flushes
are likely to be incomplete (non executable range flush, or any page
flush). This ASID appears valid to has_valid_asid(), but still triggers
ASID regeneration due to the upper ASID version bits being 0, which is
less than the minimum ASID version of 1 and so always treated as stale.
Signed-off-by: James Hogan <james.hogan@imgtec.com>
Cc: Paul Burton <paul.burton@imgtec.com>
Cc: Leonid Yegoshin <leonid.yegoshin@imgtec.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/13795/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2016-07-13 13:12:44 +00:00
|
|
|
int exec = vma->vm_flags & VM_EXEC;
|
2007-10-03 18:16:57 +00:00
|
|
|
|
2012-03-29 05:08:30 +00:00
|
|
|
for_each_online_cpu(cpu) {
|
MIPS: SMP: Clear ASID without confusing has_valid_asid()
The SMP flush_tlb_*() functions may clear the memory map's ASIDs for
other CPUs if the mm has only a single user (the current CPU) in order
to avoid SMP calls. However this makes it appear to has_valid_asid(),
which is used by various cache flush functions, as if the CPUs have
never run in the mm, and therefore can't have cached any of its memory.
For flush_tlb_mm() this doesn't sound unreasonable.
flush_tlb_range() corresponds to flush_cache_range() which does do full
indexed cache flushes, but only on the icache if the specified mapping
is executable, otherwise it doesn't guarantee that there are no cache
contents left for the mm.
flush_tlb_page() corresponds to flush_cache_page(), which will perform
address based cache ops on the specified page only, and also only
touches the icache if the page is executable. It does not guarantee that
there are no cache contents left for the mm.
For example, this affects flush_cache_range() which uses the
has_valid_asid() optimisation. It is required to flush the icache when
mappings are made executable (e.g. using mprotect) so they are
immediately usable. If some code is changed to non executable in order
to be modified then it will not be flushed from the icache during that
time, but the ASID on other CPUs may still be cleared for TLB flushing.
When the code is changed back to executable, flush_cache_range() will
assume the code hasn't run on those other CPUs due to the zero ASID, and
won't invalidate the icache on them.
This is fixed by clearing the other CPUs ASIDs to 1 instead of 0 for the
above two flush_tlb_*() functions when the corresponding cache flushes
are likely to be incomplete (non executable range flush, or any page
flush). This ASID appears valid to has_valid_asid(), but still triggers
ASID regeneration due to the upper ASID version bits being 0, which is
less than the minimum ASID version of 1 and so always treated as stale.
Signed-off-by: James Hogan <james.hogan@imgtec.com>
Cc: Paul Burton <paul.burton@imgtec.com>
Cc: Leonid Yegoshin <leonid.yegoshin@imgtec.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/13795/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2016-07-13 13:12:44 +00:00
|
|
|
/*
|
|
|
|
* flush_cache_range() will only fully flush icache if
|
|
|
|
* the VMA is executable, otherwise we must invalidate
|
|
|
|
* ASID without it appearing to has_valid_asid() as if
|
|
|
|
* mm has been completely unused by that CPU.
|
|
|
|
*/
|
2012-03-29 05:08:30 +00:00
|
|
|
if (cpu != smp_processor_id() && cpu_context(cpu, mm))
|
2019-02-02 01:43:25 +00:00
|
|
|
set_cpu_context(cpu, mm, !exec);
|
2012-03-29 05:08:30 +00:00
|
|
|
}
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
local_flush_tlb_range(vma, start, end);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flush_tlb_kernel_range_ipi(void *info)
|
|
|
|
{
|
2007-10-04 15:57:08 +00:00
|
|
|
struct flush_tlb_data *fd = info;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
|
|
|
|
{
|
2007-10-04 17:18:52 +00:00
|
|
|
struct flush_tlb_data fd = {
|
|
|
|
.addr1 = start,
|
|
|
|
.addr2 = end,
|
|
|
|
};
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-05-09 07:39:44 +00:00
|
|
|
on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flush_tlb_page_ipi(void *info)
|
|
|
|
{
|
2007-10-04 15:57:08 +00:00
|
|
|
struct flush_tlb_data *fd = info;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
local_flush_tlb_page(fd->vma, fd->addr1);
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
|
|
|
|
{
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
u32 old_mmid;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
preempt_disable();
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
if (cpu_has_mmid) {
|
|
|
|
htw_stop();
|
|
|
|
old_mmid = read_c0_memorymapid();
|
|
|
|
write_c0_memorymapid(cpu_asid(0, vma->vm_mm));
|
|
|
|
mtc0_tlbw_hazard();
|
|
|
|
ginvt_va_mmid(page);
|
|
|
|
sync_ginv();
|
|
|
|
write_c0_memorymapid(old_mmid);
|
|
|
|
instruction_hazard();
|
|
|
|
htw_start();
|
|
|
|
} else if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
|
|
|
|
(current->mm != vma->vm_mm)) {
|
2007-10-04 17:18:52 +00:00
|
|
|
struct flush_tlb_data fd = {
|
|
|
|
.vma = vma,
|
|
|
|
.addr1 = page,
|
|
|
|
};
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-10-04 15:57:08 +00:00
|
|
|
smp_on_other_tlbs(flush_tlb_page_ipi, &fd);
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
local_flush_tlb_page(vma, page);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2007-10-03 18:16:57 +00:00
|
|
|
unsigned int cpu;
|
|
|
|
|
2012-03-29 05:08:30 +00:00
|
|
|
for_each_online_cpu(cpu) {
|
MIPS: SMP: Clear ASID without confusing has_valid_asid()
The SMP flush_tlb_*() functions may clear the memory map's ASIDs for
other CPUs if the mm has only a single user (the current CPU) in order
to avoid SMP calls. However this makes it appear to has_valid_asid(),
which is used by various cache flush functions, as if the CPUs have
never run in the mm, and therefore can't have cached any of its memory.
For flush_tlb_mm() this doesn't sound unreasonable.
flush_tlb_range() corresponds to flush_cache_range() which does do full
indexed cache flushes, but only on the icache if the specified mapping
is executable, otherwise it doesn't guarantee that there are no cache
contents left for the mm.
flush_tlb_page() corresponds to flush_cache_page(), which will perform
address based cache ops on the specified page only, and also only
touches the icache if the page is executable. It does not guarantee that
there are no cache contents left for the mm.
For example, this affects flush_cache_range() which uses the
has_valid_asid() optimisation. It is required to flush the icache when
mappings are made executable (e.g. using mprotect) so they are
immediately usable. If some code is changed to non executable in order
to be modified then it will not be flushed from the icache during that
time, but the ASID on other CPUs may still be cleared for TLB flushing.
When the code is changed back to executable, flush_cache_range() will
assume the code hasn't run on those other CPUs due to the zero ASID, and
won't invalidate the icache on them.
This is fixed by clearing the other CPUs ASIDs to 1 instead of 0 for the
above two flush_tlb_*() functions when the corresponding cache flushes
are likely to be incomplete (non executable range flush, or any page
flush). This ASID appears valid to has_valid_asid(), but still triggers
ASID regeneration due to the upper ASID version bits being 0, which is
less than the minimum ASID version of 1 and so always treated as stale.
Signed-off-by: James Hogan <james.hogan@imgtec.com>
Cc: Paul Burton <paul.burton@imgtec.com>
Cc: Leonid Yegoshin <leonid.yegoshin@imgtec.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/13795/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
2016-07-13 13:12:44 +00:00
|
|
|
/*
|
|
|
|
* flush_cache_page() only does partial flushes, so
|
|
|
|
* invalidate ASID without it appearing to
|
|
|
|
* has_valid_asid() as if mm has been completely unused
|
|
|
|
* by that CPU.
|
|
|
|
*/
|
2012-03-29 05:08:30 +00:00
|
|
|
if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
|
2019-02-02 01:43:25 +00:00
|
|
|
set_cpu_context(cpu, vma->vm_mm, 1);
|
2012-03-29 05:08:30 +00:00
|
|
|
}
|
MIPS: MemoryMapID (MMID) Support
Introduce support for using MemoryMapIDs (MMIDs) as an alternative to
Address Space IDs (ASIDs). The major difference between the two is that
MMIDs are global - ie. an MMID uniquely identifies an address space
across all coherent CPUs. In contrast ASIDs are non-global per-CPU IDs,
wherein each address space is allocated a separate ASID for each CPU
upon which it is used. This global namespace allows a new GINVT
instruction be used to globally invalidate TLB entries associated with a
particular MMID across all coherent CPUs in the system, removing the
need for IPIs to invalidate entries with separate ASIDs on each CPU.
The allocation scheme used here is largely borrowed from arm64 (see
arch/arm64/mm/context.c). In essence we maintain a bitmap to track
available MMIDs, and MMIDs in active use at the time of a rollover to a
new MMID version are preserved in the new version. The allocation scheme
requires efficient 64 bit atomics in order to perform reasonably, so
this support depends upon CONFIG_GENERIC_ATOMIC64=n (ie. currently it
will only be included in MIPS64 kernels).
The first, and currently only, available CPU with support for MMIDs is
the MIPS I6500. This CPU supports 16 bit MMIDs, and so for now we cap
our MMIDs to 16 bits wide in order to prevent the bitmap growing to
absurd sizes if any future CPU does implement 32 bit MMIDs as the
architecture manuals suggest is recommended.
When MMIDs are in use we also make use of GINVT instruction which is
available due to the global nature of MMIDs. By executing a sequence of
GINVT & SYNC 0x14 instructions we can avoid the overhead of an IPI to
each remote CPU in many cases. One complication is that GINVT will
invalidate wired entries (in all cases apart from type 0, which targets
the entire TLB). In order to avoid GINVT invalidating any wired TLB
entries we set up, we make sure to create those entries using a reserved
MMID (0) that we never associate with any address space.
Also of note is that KVM will require further work in order to support
MMIDs & GINVT, since KVM is involved in allocating IDs for guests & in
configuring the MMU. That work is not part of this patch, so for now
when MMIDs are in use KVM is disabled.
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: linux-mips@vger.kernel.org
2019-02-02 01:43:28 +00:00
|
|
|
local_flush_tlb_page(vma, page);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flush_tlb_one_ipi(void *info)
|
|
|
|
{
|
|
|
|
unsigned long vaddr = (unsigned long) info;
|
|
|
|
|
|
|
|
local_flush_tlb_one(vaddr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tlb_one(unsigned long vaddr)
|
|
|
|
{
|
2006-06-22 21:42:32 +00:00
|
|
|
smp_on_each_tlb(flush_tlb_one_ipi, (void *) vaddr);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(flush_tlb_page);
|
|
|
|
EXPORT_SYMBOL(flush_tlb_one);
|
2012-10-11 16:14:58 +00:00
|
|
|
|
2014-02-14 09:24:58 +00:00
|
|
|
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(atomic_t, tick_broadcast_count);
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 04:30:00 +00:00
|
|
|
static DEFINE_PER_CPU(call_single_data_t, tick_broadcast_csd);
|
2014-02-14 09:24:58 +00:00
|
|
|
|
|
|
|
void tick_broadcast(const struct cpumask *mask)
|
|
|
|
{
|
|
|
|
atomic_t *count;
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 04:30:00 +00:00
|
|
|
call_single_data_t *csd;
|
2014-02-14 09:24:58 +00:00
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for_each_cpu(cpu, mask) {
|
|
|
|
count = &per_cpu(tick_broadcast_count, cpu);
|
|
|
|
csd = &per_cpu(tick_broadcast_csd, cpu);
|
|
|
|
|
|
|
|
if (atomic_inc_return(count) == 1)
|
|
|
|
smp_call_function_single_async(cpu, csd);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tick_broadcast_callee(void *info)
|
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
tick_receive_broadcast();
|
|
|
|
atomic_set(&per_cpu(tick_broadcast_count, cpu), 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init tick_broadcast_init(void)
|
|
|
|
{
|
smp: Avoid using two cache lines for struct call_single_data
struct call_single_data is used in IPIs to transfer information between
CPUs. Its size is bigger than sizeof(unsigned long) and less than
cache line size. Currently it is not allocated with any explicit alignment
requirements. This makes it possible for allocated call_single_data to
cross two cache lines, which results in double the number of the cache lines
that need to be transferred among CPUs.
This can be fixed by requiring call_single_data to be aligned with the
size of call_single_data. Currently the size of call_single_data is the
power of 2. If we add new fields to call_single_data, we may need to
add padding to make sure the size of new definition is the power of 2
as well.
Fortunately, this is enforced by GCC, which will report bad sizes.
To set alignment requirements of call_single_data to the size of
call_single_data, a struct definition and a typedef is used.
To test the effect of the patch, I used the vm-scalability multiple
thread swap test case (swap-w-seq-mt). The test will create multiple
threads and each thread will eat memory until all RAM and part of swap
is used, so that huge number of IPIs are triggered when unmapping
memory. In the test, the throughput of memory writing improves ~5%
compared with misaligned call_single_data, because of faster IPIs.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Huang, Ying <ying.huang@intel.com>
[ Add call_single_data_t and align with size of call_single_data. ]
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/87bmnqd6lz.fsf@yhuang-mobile.sh.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-08-08 04:30:00 +00:00
|
|
|
call_single_data_t *csd;
|
2014-02-14 09:24:58 +00:00
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for (cpu = 0; cpu < NR_CPUS; cpu++) {
|
|
|
|
csd = &per_cpu(tick_broadcast_csd, cpu);
|
|
|
|
csd->func = tick_broadcast_callee;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_initcall(tick_broadcast_init);
|
|
|
|
|
|
|
|
#endif /* CONFIG_GENERIC_CLOCKEVENTS_BROADCAST */
|