mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-30 14:19:16 +00:00
x86: Unify CPU -> NUMA node mapping between 32 and 64bit
Unlike 64bit, 32bit has been using its own cpu_to_node_map[] for CPU -> NUMA node mapping. Replace it with early_percpu variable x86_cpu_to_node_map and share the mapping code with 64bit. * USE_PERCPU_NUMA_NODE_ID is now enabled for 32bit too. * x86_cpu_to_node_map and numa_set/clear_node() are moved from numa_64 to numa. For now, on 32bit, x86_cpu_to_node_map is initialized with 0 instead of NUMA_NO_NODE. This is to avoid introducing unexpected behavior change and will be updated once init path is unified. * srat_detect_node() is now enabled for x86_32 too. It calls numa_set_node() and initializes the mapping making explicit cpu_to_node_map[] updates from map/unmap_cpu_to_node() unnecessary. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: eric.dumazet@gmail.com Cc: yinghai@kernel.org Cc: brgerst@gmail.com Cc: gorcunov@gmail.com Cc: penberg@kernel.org Cc: shaohui.zheng@intel.com Cc: rientjes@google.com LKML-Reference: <1295789862-25482-15-git-send-email-tj@kernel.org> Signed-off-by: Ingo Molnar <mingo@elte.hu> Cc: David Rientjes <rientjes@google.com>
This commit is contained in:
parent
bbc9e2f452
commit
645a79195f
11 changed files with 85 additions and 104 deletions
|
@ -1705,7 +1705,7 @@ config HAVE_ARCH_EARLY_PFN_TO_NID
|
||||||
depends on NUMA
|
depends on NUMA
|
||||||
|
|
||||||
config USE_PERCPU_NUMA_NODE_ID
|
config USE_PERCPU_NUMA_NODE_ID
|
||||||
def_bool X86_64
|
def_bool y
|
||||||
depends on NUMA
|
depends on NUMA
|
||||||
|
|
||||||
menu "Power management and ACPI options"
|
menu "Power management and ACPI options"
|
||||||
|
|
|
@ -30,4 +30,12 @@ static inline void set_apicid_to_node(int apicid, s16 node)
|
||||||
# include "numa_64.h"
|
# include "numa_64.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef CONFIG_NUMA
|
||||||
|
extern void __cpuinit numa_set_node(int cpu, int node);
|
||||||
|
extern void __cpuinit numa_clear_node(int cpu);
|
||||||
|
#else /* CONFIG_NUMA */
|
||||||
|
static inline void numa_set_node(int cpu, int node) { }
|
||||||
|
static inline void numa_clear_node(int cpu) { }
|
||||||
|
#endif /* CONFIG_NUMA */
|
||||||
|
|
||||||
#endif /* _ASM_X86_NUMA_H */
|
#endif /* _ASM_X86_NUMA_H */
|
||||||
|
|
|
@ -30,8 +30,6 @@ extern void setup_node_bootmem(int nodeid, unsigned long start,
|
||||||
|
|
||||||
extern void __init init_cpu_to_node(void);
|
extern void __init init_cpu_to_node(void);
|
||||||
extern int __cpuinit numa_cpu_node(int cpu);
|
extern int __cpuinit numa_cpu_node(int cpu);
|
||||||
extern void __cpuinit numa_set_node(int cpu, int node);
|
|
||||||
extern void __cpuinit numa_clear_node(int cpu);
|
|
||||||
extern void __cpuinit numa_add_cpu(int cpu);
|
extern void __cpuinit numa_add_cpu(int cpu);
|
||||||
extern void __cpuinit numa_remove_cpu(int cpu);
|
extern void __cpuinit numa_remove_cpu(int cpu);
|
||||||
|
|
||||||
|
@ -43,8 +41,6 @@ void numa_emu_cmdline(char *);
|
||||||
#else
|
#else
|
||||||
static inline void init_cpu_to_node(void) { }
|
static inline void init_cpu_to_node(void) { }
|
||||||
static inline int numa_cpu_node(int cpu) { return NUMA_NO_NODE; }
|
static inline int numa_cpu_node(int cpu) { return NUMA_NO_NODE; }
|
||||||
static inline void numa_set_node(int cpu, int node) { }
|
|
||||||
static inline void numa_clear_node(int cpu) { }
|
|
||||||
static inline void numa_add_cpu(int cpu, int node) { }
|
static inline void numa_add_cpu(int cpu, int node) { }
|
||||||
static inline void numa_remove_cpu(int cpu) { }
|
static inline void numa_remove_cpu(int cpu) { }
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -47,21 +47,6 @@
|
||||||
|
|
||||||
#include <asm/mpspec.h>
|
#include <asm/mpspec.h>
|
||||||
|
|
||||||
#ifdef CONFIG_X86_32
|
|
||||||
|
|
||||||
/* Mappings between logical cpu number and node number */
|
|
||||||
extern int cpu_to_node_map[];
|
|
||||||
|
|
||||||
/* Returns the number of the node containing CPU 'cpu' */
|
|
||||||
static inline int __cpu_to_node(int cpu)
|
|
||||||
{
|
|
||||||
return cpu_to_node_map[cpu];
|
|
||||||
}
|
|
||||||
#define early_cpu_to_node __cpu_to_node
|
|
||||||
#define cpu_to_node __cpu_to_node
|
|
||||||
|
|
||||||
#else /* CONFIG_X86_64 */
|
|
||||||
|
|
||||||
/* Mappings between logical cpu number and node number */
|
/* Mappings between logical cpu number and node number */
|
||||||
DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
|
DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
|
||||||
|
|
||||||
|
@ -84,8 +69,6 @@ static inline int early_cpu_to_node(int cpu)
|
||||||
|
|
||||||
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
||||||
|
|
||||||
#endif /* CONFIG_X86_64 */
|
|
||||||
|
|
||||||
/* Mappings between node number and cpus on that node. */
|
/* Mappings between node number and cpus on that node. */
|
||||||
extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
||||||
|
|
||||||
|
|
|
@ -590,12 +590,7 @@ static void acpi_map_cpu2node(acpi_handle handle, int cpu, int physid)
|
||||||
if (nid == -1 || !node_online(nid))
|
if (nid == -1 || !node_online(nid))
|
||||||
return;
|
return;
|
||||||
set_apicid_to_node(physid, nid);
|
set_apicid_to_node(physid, nid);
|
||||||
#ifdef CONFIG_X86_64
|
|
||||||
numa_set_node(cpu, nid);
|
numa_set_node(cpu, nid);
|
||||||
#else /* CONFIG_X86_32 */
|
|
||||||
cpu_to_node_map[cpu] = nid;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -233,7 +233,7 @@ static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
|
#ifdef CONFIG_NUMA
|
||||||
/*
|
/*
|
||||||
* To workaround broken NUMA config. Read the comment in
|
* To workaround broken NUMA config. Read the comment in
|
||||||
* srat_detect_node().
|
* srat_detect_node().
|
||||||
|
@ -338,7 +338,7 @@ EXPORT_SYMBOL_GPL(amd_get_nb_id);
|
||||||
|
|
||||||
static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
|
static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
|
||||||
{
|
{
|
||||||
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
|
#ifdef CONFIG_NUMA
|
||||||
int cpu = smp_processor_id();
|
int cpu = smp_processor_id();
|
||||||
int node;
|
int node;
|
||||||
unsigned apicid = c->apicid;
|
unsigned apicid = c->apicid;
|
||||||
|
|
|
@ -276,7 +276,7 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
|
||||||
|
|
||||||
static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
|
static void __cpuinit srat_detect_node(struct cpuinfo_x86 *c)
|
||||||
{
|
{
|
||||||
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
|
#ifdef CONFIG_NUMA
|
||||||
unsigned node;
|
unsigned node;
|
||||||
int cpu = smp_processor_id();
|
int cpu = smp_processor_id();
|
||||||
|
|
||||||
|
|
|
@ -233,6 +233,7 @@ void __init setup_per_cpu_areas(void)
|
||||||
per_cpu(irq_stack_ptr, cpu) =
|
per_cpu(irq_stack_ptr, cpu) =
|
||||||
per_cpu(irq_stack_union.irq_stack, cpu) +
|
per_cpu(irq_stack_union.irq_stack, cpu) +
|
||||||
IRQ_STACK_SIZE - 64;
|
IRQ_STACK_SIZE - 64;
|
||||||
|
#endif
|
||||||
#ifdef CONFIG_NUMA
|
#ifdef CONFIG_NUMA
|
||||||
per_cpu(x86_cpu_to_node_map, cpu) =
|
per_cpu(x86_cpu_to_node_map, cpu) =
|
||||||
early_per_cpu_map(x86_cpu_to_node_map, cpu);
|
early_per_cpu_map(x86_cpu_to_node_map, cpu);
|
||||||
|
@ -245,7 +246,6 @@ void __init setup_per_cpu_areas(void)
|
||||||
* So set them all (boot cpu and all APs).
|
* So set them all (boot cpu and all APs).
|
||||||
*/
|
*/
|
||||||
set_cpu_numa_node(cpu, early_cpu_to_node(cpu));
|
set_cpu_numa_node(cpu, early_cpu_to_node(cpu));
|
||||||
#endif
|
|
||||||
#endif
|
#endif
|
||||||
/*
|
/*
|
||||||
* Up to this point, the boot CPU has been using .init.data
|
* Up to this point, the boot CPU has been using .init.data
|
||||||
|
@ -263,7 +263,7 @@ void __init setup_per_cpu_areas(void)
|
||||||
#ifdef CONFIG_X86_32
|
#ifdef CONFIG_X86_32
|
||||||
early_per_cpu_ptr(x86_cpu_to_logical_apicid) = NULL;
|
early_per_cpu_ptr(x86_cpu_to_logical_apicid) = NULL;
|
||||||
#endif
|
#endif
|
||||||
#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
|
#ifdef CONFIG_NUMA
|
||||||
early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
|
early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
|
@ -133,16 +133,11 @@ EXPORT_PER_CPU_SYMBOL(cpu_info);
|
||||||
atomic_t init_deasserted;
|
atomic_t init_deasserted;
|
||||||
|
|
||||||
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
|
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
|
||||||
/* which node each logical CPU is on */
|
|
||||||
int cpu_to_node_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
|
|
||||||
EXPORT_SYMBOL(cpu_to_node_map);
|
|
||||||
|
|
||||||
/* set up a mapping between cpu and node. */
|
/* set up a mapping between cpu and node. */
|
||||||
static void map_cpu_to_node(int cpu, int node)
|
static void map_cpu_to_node(int cpu, int node)
|
||||||
{
|
{
|
||||||
printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
|
printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
|
||||||
cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
|
cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
|
||||||
cpu_to_node_map[cpu] = node;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* undo a mapping between cpu and node. */
|
/* undo a mapping between cpu and node. */
|
||||||
|
@ -153,7 +148,6 @@ static void unmap_cpu_to_node(int cpu)
|
||||||
printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
|
printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
|
||||||
for (node = 0; node < MAX_NUMNODES; node++)
|
for (node = 0; node < MAX_NUMNODES; node++)
|
||||||
cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
|
||||||
cpu_to_node_map[cpu] = 0;
|
|
||||||
}
|
}
|
||||||
#else /* !(CONFIG_NUMA && CONFIG_X86_32) */
|
#else /* !(CONFIG_NUMA && CONFIG_X86_32) */
|
||||||
#define map_cpu_to_node(cpu, node) ({})
|
#define map_cpu_to_node(cpu, node) ({})
|
||||||
|
|
|
@ -35,6 +35,44 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
|
||||||
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
||||||
EXPORT_SYMBOL(node_to_cpumask_map);
|
EXPORT_SYMBOL(node_to_cpumask_map);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Map cpu index to node index
|
||||||
|
*/
|
||||||
|
#ifdef CONFIG_X86_32
|
||||||
|
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, 0);
|
||||||
|
#else
|
||||||
|
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
|
||||||
|
#endif
|
||||||
|
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
||||||
|
|
||||||
|
void __cpuinit numa_set_node(int cpu, int node)
|
||||||
|
{
|
||||||
|
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
|
||||||
|
|
||||||
|
/* early setting, no percpu area yet */
|
||||||
|
if (cpu_to_node_map) {
|
||||||
|
cpu_to_node_map[cpu] = node;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
||||||
|
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
|
||||||
|
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
|
||||||
|
dump_stack();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
||||||
|
|
||||||
|
if (node != NUMA_NO_NODE)
|
||||||
|
set_cpu_numa_node(cpu, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
void __cpuinit numa_clear_node(int cpu)
|
||||||
|
{
|
||||||
|
numa_set_node(cpu, NUMA_NO_NODE);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Allocate node_to_cpumask_map based on number of available nodes
|
* Allocate node_to_cpumask_map based on number of available nodes
|
||||||
* Requires node_possible_map to be valid.
|
* Requires node_possible_map to be valid.
|
||||||
|
@ -62,6 +100,37 @@ void __init setup_node_to_cpumask_map(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
||||||
|
|
||||||
|
int __cpu_to_node(int cpu)
|
||||||
|
{
|
||||||
|
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
|
||||||
|
printk(KERN_WARNING
|
||||||
|
"cpu_to_node(%d): usage too early!\n", cpu);
|
||||||
|
dump_stack();
|
||||||
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
||||||
|
}
|
||||||
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(__cpu_to_node);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Same function as cpu_to_node() but used if called before the
|
||||||
|
* per_cpu areas are setup.
|
||||||
|
*/
|
||||||
|
int early_cpu_to_node(int cpu)
|
||||||
|
{
|
||||||
|
if (early_per_cpu_ptr(x86_cpu_to_node_map))
|
||||||
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
||||||
|
|
||||||
|
if (!cpu_possible(cpu)) {
|
||||||
|
printk(KERN_WARNING
|
||||||
|
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
|
||||||
|
dump_stack();
|
||||||
|
return NUMA_NO_NODE;
|
||||||
|
}
|
||||||
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Returns a pointer to the bitmask of CPUs on Node 'node'.
|
* Returns a pointer to the bitmask of CPUs on Node 'node'.
|
||||||
*/
|
*/
|
||||||
|
@ -84,4 +153,5 @@ const struct cpumask *cpumask_of_node(int node)
|
||||||
return node_to_cpumask_map[node];
|
return node_to_cpumask_map[node];
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(cpumask_of_node);
|
EXPORT_SYMBOL(cpumask_of_node);
|
||||||
#endif
|
|
||||||
|
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
|
||||||
|
|
|
@ -29,12 +29,6 @@ struct memnode memnode;
|
||||||
static unsigned long __initdata nodemap_addr;
|
static unsigned long __initdata nodemap_addr;
|
||||||
static unsigned long __initdata nodemap_size;
|
static unsigned long __initdata nodemap_size;
|
||||||
|
|
||||||
/*
|
|
||||||
* Map cpu index to node index
|
|
||||||
*/
|
|
||||||
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
|
|
||||||
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Given a shift value, try to populate memnodemap[]
|
* Given a shift value, try to populate memnodemap[]
|
||||||
* Returns :
|
* Returns :
|
||||||
|
@ -732,34 +726,6 @@ int __cpuinit numa_cpu_node(int cpu)
|
||||||
return NUMA_NO_NODE;
|
return NUMA_NO_NODE;
|
||||||
}
|
}
|
||||||
|
|
||||||
void __cpuinit numa_set_node(int cpu, int node)
|
|
||||||
{
|
|
||||||
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
|
|
||||||
|
|
||||||
/* early setting, no percpu area yet */
|
|
||||||
if (cpu_to_node_map) {
|
|
||||||
cpu_to_node_map[cpu] = node;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
||||||
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
|
|
||||||
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
|
|
||||||
dump_stack();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
|
||||||
|
|
||||||
if (node != NUMA_NO_NODE)
|
|
||||||
set_cpu_numa_node(cpu, node);
|
|
||||||
}
|
|
||||||
|
|
||||||
void __cpuinit numa_clear_node(int cpu)
|
|
||||||
{
|
|
||||||
numa_set_node(cpu, NUMA_NO_NODE);
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
|
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
|
||||||
|
|
||||||
#ifndef CONFIG_NUMA_EMU
|
#ifndef CONFIG_NUMA_EMU
|
||||||
|
@ -887,37 +853,6 @@ void __cpuinit numa_remove_cpu(int cpu)
|
||||||
{
|
{
|
||||||
numa_set_cpumask(cpu, 0);
|
numa_set_cpumask(cpu, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
int __cpu_to_node(int cpu)
|
|
||||||
{
|
|
||||||
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
|
|
||||||
printk(KERN_WARNING
|
|
||||||
"cpu_to_node(%d): usage too early!\n", cpu);
|
|
||||||
dump_stack();
|
|
||||||
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
||||||
}
|
|
||||||
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL(__cpu_to_node);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Same function as cpu_to_node() but used if called before the
|
|
||||||
* per_cpu areas are setup.
|
|
||||||
*/
|
|
||||||
int early_cpu_to_node(int cpu)
|
|
||||||
{
|
|
||||||
if (early_per_cpu_ptr(x86_cpu_to_node_map))
|
|
||||||
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
||||||
|
|
||||||
if (!cpu_possible(cpu)) {
|
|
||||||
printk(KERN_WARNING
|
|
||||||
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
|
|
||||||
dump_stack();
|
|
||||||
return NUMA_NO_NODE;
|
|
||||||
}
|
|
||||||
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* --------- end of debug versions of the numa functions ---------
|
* --------- end of debug versions of the numa functions ---------
|
||||||
*/
|
*/
|
||||||
|
|
Loading…
Reference in a new issue