2019-05-19 12:08:55 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2011-05-26 16:22:53 +00:00
|
|
|
#include <linux/export.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/bitops.h>
|
2011-08-10 01:49:56 +00:00
|
|
|
#include <linux/elf.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/mm.h>
|
2008-09-08 00:58:53 +00:00
|
|
|
|
2009-07-03 23:35:45 +00:00
|
|
|
#include <linux/io.h>
|
2012-02-07 12:08:52 +00:00
|
|
|
#include <linux/sched.h>
|
2017-02-01 15:36:40 +00:00
|
|
|
#include <linux/sched/clock.h>
|
2015-03-27 11:38:21 +00:00
|
|
|
#include <linux/random.h>
|
2019-08-08 19:53:01 +00:00
|
|
|
#include <linux/topology.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/processor.h>
|
2007-08-10 20:31:07 +00:00
|
|
|
#include <asm/apic.h>
|
2018-04-27 21:34:37 +00:00
|
|
|
#include <asm/cacheinfo.h>
|
2009-03-08 07:46:26 +00:00
|
|
|
#include <asm/cpu.h>
|
2018-04-29 13:01:37 +00:00
|
|
|
#include <asm/spec-ctrl.h>
|
2014-06-24 11:25:04 +00:00
|
|
|
#include <asm/smp.h>
|
2020-08-06 12:35:11 +00:00
|
|
|
#include <asm/numa.h>
|
2009-06-08 13:55:09 +00:00
|
|
|
#include <asm/pci-direct.h>
|
2015-08-10 10:19:54 +00:00
|
|
|
#include <asm/delay.h>
|
2018-12-04 23:34:56 +00:00
|
|
|
#include <asm/debugreg.h>
|
2020-05-05 22:36:15 +00:00
|
|
|
#include <asm/resctrl.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-09-08 00:58:53 +00:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
# include <asm/mmconfig.h>
|
|
|
|
#endif
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#include "cpu.h"
|
|
|
|
|
2015-06-15 08:28:15 +00:00
|
|
|
/*
|
|
|
|
* nodes_per_socket: Stores the number of nodes per socket.
|
|
|
|
* Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX
|
|
|
|
* Node Identifiers[10:8]
|
|
|
|
*/
|
|
|
|
static u32 nodes_per_socket = 1;
|
|
|
|
|
2012-06-01 14:52:38 +00:00
|
|
|
static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
|
|
|
|
{
|
|
|
|
u32 gprs[8] = { 0 };
|
|
|
|
int err;
|
|
|
|
|
2013-04-08 15:57:45 +00:00
|
|
|
WARN_ONCE((boot_cpu_data.x86 != 0xf),
|
|
|
|
"%s should only be used on K8!\n", __func__);
|
2012-06-01 14:52:38 +00:00
|
|
|
|
|
|
|
gprs[1] = msr;
|
|
|
|
gprs[7] = 0x9c5a203a;
|
|
|
|
|
|
|
|
err = rdmsr_safe_regs(gprs);
|
|
|
|
|
|
|
|
*p = gprs[0] | ((u64)gprs[2] << 32);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
|
|
|
|
{
|
|
|
|
u32 gprs[8] = { 0 };
|
|
|
|
|
2013-04-08 15:57:45 +00:00
|
|
|
WARN_ONCE((boot_cpu_data.x86 != 0xf),
|
|
|
|
"%s should only be used on K8!\n", __func__);
|
2012-06-01 14:52:38 +00:00
|
|
|
|
|
|
|
gprs[0] = (u32)val;
|
|
|
|
gprs[1] = msr;
|
|
|
|
gprs[2] = val >> 32;
|
|
|
|
gprs[7] = 0x9c5a203a;
|
|
|
|
|
|
|
|
return wrmsr_safe_regs(gprs);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* B step AMD K6 before B 9730xxxx have hardware bugs that can cause
|
|
|
|
* misexecution of code under Linux. Owners of such processors should
|
|
|
|
* contact AMD for precise details and a CPU swap.
|
|
|
|
*
|
|
|
|
* See http://www.multimania.com/poulot/k6bug.html
|
2012-04-11 15:12:38 +00:00
|
|
|
* and section 2.6.2 of "AMD-K6 Processor Revision Guide - Model 6"
|
|
|
|
* (Publication # 21266 Issue Date: August 1998)
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* The following test is erm.. interesting. AMD neglected to up
|
|
|
|
* the chip setting when fixing the bug but they also tweaked some
|
|
|
|
* performance at the same time..
|
|
|
|
*/
|
2008-02-22 22:10:33 +00:00
|
|
|
|
2019-03-30 00:47:36 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2013-08-05 22:02:43 +00:00
|
|
|
extern __visible void vide(void);
|
2019-03-30 00:47:35 +00:00
|
|
|
__asm__(".text\n"
|
|
|
|
".globl vide\n"
|
2016-01-21 22:49:14 +00:00
|
|
|
".type vide, @function\n"
|
|
|
|
".align 4\n"
|
|
|
|
"vide: ret\n");
|
2019-03-30 00:47:36 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void init_amd_k5(struct cpuinfo_x86 *c)
|
2008-09-08 00:58:50 +00:00
|
|
|
{
|
2014-06-24 11:25:04 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-09-08 00:58:50 +00:00
|
|
|
/*
|
|
|
|
* General Systems BIOSen alias the cpu frequency registers
|
2016-02-23 23:34:30 +00:00
|
|
|
* of the Elan at 0x000df000. Unfortunately, one of the Linux
|
2008-09-08 00:58:50 +00:00
|
|
|
* drivers subsequently pokes it, and changes the CPU speed.
|
|
|
|
* Workaround : Remove the unneeded alias.
|
|
|
|
*/
|
|
|
|
#define CBAR (0xfffc) /* Configuration Base Address (32-bit) */
|
|
|
|
#define CBAR_ENB (0x80000000)
|
|
|
|
#define CBAR_KEY (0X000000CB)
|
|
|
|
if (c->x86_model == 9 || c->x86_model == 10) {
|
2009-07-03 23:35:45 +00:00
|
|
|
if (inl(CBAR) & CBAR_ENB)
|
|
|
|
outl(0 | CBAR_KEY, CBAR);
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
2014-06-24 11:25:04 +00:00
|
|
|
#endif
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void init_amd_k6(struct cpuinfo_x86 *c)
|
2008-09-08 00:58:50 +00:00
|
|
|
{
|
2014-06-24 11:25:04 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-09-08 00:58:50 +00:00
|
|
|
u32 l, h;
|
2013-07-03 22:04:19 +00:00
|
|
|
int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
|
2008-09-08 00:58:50 +00:00
|
|
|
|
|
|
|
if (c->x86_model < 6) {
|
|
|
|
/* Based on AMD doc 20734R - June 2000 */
|
|
|
|
if (c->x86_model == 0) {
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_APIC);
|
|
|
|
set_cpu_cap(c, X86_FEATURE_PGE);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-01-01 01:52:10 +00:00
|
|
|
if (c->x86_model == 6 && c->x86_stepping == 1) {
|
2008-09-08 00:58:50 +00:00
|
|
|
const int K6_BUG_LOOP = 1000000;
|
|
|
|
int n;
|
|
|
|
void (*f_vide)(void);
|
2015-06-25 16:44:01 +00:00
|
|
|
u64 d, d2;
|
2008-09-08 00:58:50 +00:00
|
|
|
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_info("AMD K6 stepping B detected - ");
|
2008-09-08 00:58:50 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* It looks like AMD fixed the 2.6.2 bug and improved indirect
|
|
|
|
* calls at the same time.
|
|
|
|
*/
|
|
|
|
|
|
|
|
n = K6_BUG_LOOP;
|
|
|
|
f_vide = vide;
|
2017-07-11 11:44:05 +00:00
|
|
|
OPTIMIZER_HIDE_VAR(f_vide);
|
2015-06-25 16:44:07 +00:00
|
|
|
d = rdtsc();
|
2008-09-08 00:58:50 +00:00
|
|
|
while (n--)
|
|
|
|
f_vide();
|
2015-06-25 16:44:07 +00:00
|
|
|
d2 = rdtsc();
|
2008-09-08 00:58:50 +00:00
|
|
|
d = d2-d;
|
|
|
|
|
|
|
|
if (d > 20*K6_BUG_LOOP)
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_cont("system stability may be impaired when more than 32 MB are used.\n");
|
2008-09-08 00:58:50 +00:00
|
|
|
else
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_cont("probably OK (after B9730xxxx).\n");
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* K6 with old style WHCR */
|
|
|
|
if (c->x86_model < 8 ||
|
2018-01-01 01:52:10 +00:00
|
|
|
(c->x86_model == 8 && c->x86_stepping < 8)) {
|
2008-09-08 00:58:50 +00:00
|
|
|
/* We can only write allocate on the low 508Mb */
|
|
|
|
if (mbytes > 508)
|
|
|
|
mbytes = 508;
|
|
|
|
|
|
|
|
rdmsr(MSR_K6_WHCR, l, h);
|
|
|
|
if ((l&0x0000FFFF) == 0) {
|
|
|
|
unsigned long flags;
|
|
|
|
l = (1<<0)|((mbytes/4)<<1);
|
|
|
|
local_irq_save(flags);
|
|
|
|
wbinvd();
|
|
|
|
wrmsr(MSR_K6_WHCR, l, h);
|
|
|
|
local_irq_restore(flags);
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_info("Enabling old style K6 write allocation for %d Mb\n",
|
2008-09-08 00:58:50 +00:00
|
|
|
mbytes);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-01-01 01:52:10 +00:00
|
|
|
if ((c->x86_model == 8 && c->x86_stepping > 7) ||
|
2008-09-08 00:58:50 +00:00
|
|
|
c->x86_model == 9 || c->x86_model == 13) {
|
|
|
|
/* The more serious chips .. */
|
|
|
|
|
|
|
|
if (mbytes > 4092)
|
|
|
|
mbytes = 4092;
|
|
|
|
|
|
|
|
rdmsr(MSR_K6_WHCR, l, h);
|
|
|
|
if ((l&0xFFFF0000) == 0) {
|
|
|
|
unsigned long flags;
|
|
|
|
l = ((mbytes>>2)<<22)|(1<<16);
|
|
|
|
local_irq_save(flags);
|
|
|
|
wbinvd();
|
|
|
|
wrmsr(MSR_K6_WHCR, l, h);
|
|
|
|
local_irq_restore(flags);
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_info("Enabling new style K6 write allocation for %d Mb\n",
|
2008-09-08 00:58:50 +00:00
|
|
|
mbytes);
|
|
|
|
}
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c->x86_model == 10) {
|
|
|
|
/* AMD Geode LX is model 10 */
|
|
|
|
/* placeholder for any needed mods */
|
|
|
|
return;
|
|
|
|
}
|
2014-06-24 11:25:04 +00:00
|
|
|
#endif
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
|
|
|
|
2014-06-24 11:25:04 +00:00
|
|
|
static void init_amd_k7(struct cpuinfo_x86 *c)
|
2009-03-08 07:46:26 +00:00
|
|
|
{
|
2014-06-24 11:25:04 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
u32 l, h;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Bit 15 of Athlon specific MSR 15, needs to be 0
|
|
|
|
* to enable SSE on Palomino/Morgan/Barton CPU's.
|
|
|
|
* If the BIOS didn't enable it already, enable it here.
|
|
|
|
*/
|
|
|
|
if (c->x86_model >= 6 && c->x86_model <= 10) {
|
|
|
|
if (!cpu_has(c, X86_FEATURE_XMM)) {
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_info("Enabling disabled K7/SSE Support.\n");
|
2014-06-24 11:25:04 +00:00
|
|
|
msr_clear_bit(MSR_K7_HWCR, 15);
|
|
|
|
set_cpu_cap(c, X86_FEATURE_XMM);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It's been determined by AMD that Athlons since model 8 stepping 1
|
|
|
|
* are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
|
|
|
|
* As per AMD technical note 27212 0.2
|
|
|
|
*/
|
2018-01-01 01:52:10 +00:00
|
|
|
if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) {
|
2014-06-24 11:25:04 +00:00
|
|
|
rdmsr(MSR_K7_CLK_CTL, l, h);
|
|
|
|
if ((l & 0xfff00000) != 0x20000000) {
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
|
|
|
|
l, ((l & 0x000fffff)|0x20000000));
|
2014-06-24 11:25:04 +00:00
|
|
|
wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-03-08 07:46:26 +00:00
|
|
|
/* calling is from identify_secondary_cpu() ? */
|
2010-07-21 17:03:58 +00:00
|
|
|
if (!c->cpu_index)
|
2009-03-08 07:46:26 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Certain Athlons might work (for various values of 'work') in SMP
|
|
|
|
* but they are not certified as MP capable.
|
|
|
|
*/
|
|
|
|
/* Athlon 660/661 is valid. */
|
2018-01-01 01:52:10 +00:00
|
|
|
if ((c->x86_model == 6) && ((c->x86_stepping == 0) ||
|
|
|
|
(c->x86_stepping == 1)))
|
2013-04-08 15:57:46 +00:00
|
|
|
return;
|
2009-03-08 07:46:26 +00:00
|
|
|
|
|
|
|
/* Duron 670 is valid */
|
2018-01-01 01:52:10 +00:00
|
|
|
if ((c->x86_model == 7) && (c->x86_stepping == 0))
|
2013-04-08 15:57:46 +00:00
|
|
|
return;
|
2009-03-08 07:46:26 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Athlon 662, Duron 671, and Athlon >model 7 have capability
|
|
|
|
* bit. It's worth noting that the A5 stepping (662) of some
|
|
|
|
* Athlon XP's have the MP bit set.
|
|
|
|
* See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
|
|
|
|
* more.
|
|
|
|
*/
|
2018-01-01 01:52:10 +00:00
|
|
|
if (((c->x86_model == 6) && (c->x86_stepping >= 2)) ||
|
|
|
|
((c->x86_model == 7) && (c->x86_stepping >= 1)) ||
|
2009-03-08 07:46:26 +00:00
|
|
|
(c->x86_model > 7))
|
2014-06-24 11:25:04 +00:00
|
|
|
if (cpu_has(c, X86_FEATURE_MP))
|
2013-04-08 15:57:46 +00:00
|
|
|
return;
|
2009-03-08 07:46:26 +00:00
|
|
|
|
|
|
|
/* If we get here, not a certified SMP capable AMD system. */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't taint if we are running SMP kernel on a single non-MP
|
|
|
|
* approved Athlon
|
|
|
|
*/
|
|
|
|
WARN_ONCE(1, "WARNING: This combination of AMD"
|
2009-07-22 13:50:23 +00:00
|
|
|
" processors is not suitable for SMP.\n");
|
2014-02-26 15:49:49 +00:00
|
|
|
add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
|
2008-09-08 00:58:54 +00:00
|
|
|
#endif
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
2008-09-08 00:58:54 +00:00
|
|
|
|
2011-01-23 13:37:40 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
2011-01-23 13:37:39 +00:00
|
|
|
/*
|
|
|
|
* To workaround broken NUMA config. Read the comment in
|
|
|
|
* srat_detect_node().
|
|
|
|
*/
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static int nearby_node(int apicid)
|
2008-09-08 00:58:54 +00:00
|
|
|
{
|
|
|
|
int i, node;
|
|
|
|
|
|
|
|
for (i = apicid - 1; i >= 0; i--) {
|
2011-01-23 13:37:39 +00:00
|
|
|
node = __apicid_to_node[i];
|
2008-09-08 00:58:54 +00:00
|
|
|
if (node != NUMA_NO_NODE && node_online(node))
|
|
|
|
return node;
|
|
|
|
}
|
|
|
|
for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
|
2011-01-23 13:37:39 +00:00
|
|
|
node = __apicid_to_node[i];
|
2008-09-08 00:58:54 +00:00
|
|
|
if (node != NUMA_NO_NODE && node_online(node))
|
|
|
|
return node;
|
|
|
|
}
|
|
|
|
return first_node(node_online_map); /* Shouldn't happen */
|
|
|
|
}
|
|
|
|
#endif
|
2008-09-08 00:58:50 +00:00
|
|
|
|
2017-07-31 08:51:58 +00:00
|
|
|
/*
|
2023-08-14 08:18:34 +00:00
|
|
|
* Fix up topo::core_id for pre-F17h systems to be in the
|
2017-07-31 08:51:58 +00:00
|
|
|
* [0 .. cores_per_node - 1] range. Not really needed but
|
|
|
|
* kept so as not to break existing setups.
|
|
|
|
*/
|
|
|
|
static void legacy_fixup_core_id(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
u32 cus_per_node;
|
|
|
|
|
|
|
|
if (c->x86 >= 0x17)
|
|
|
|
return;
|
|
|
|
|
|
|
|
cus_per_node = c->x86_max_cores / nodes_per_socket;
|
2023-08-14 08:18:34 +00:00
|
|
|
c->topo.core_id %= cus_per_node;
|
2017-07-31 08:51:58 +00:00
|
|
|
}
|
|
|
|
|
2009-09-03 07:40:21 +00:00
|
|
|
/*
|
2010-09-30 12:36:28 +00:00
|
|
|
* Fixup core topology information for
|
|
|
|
* (1) AMD multi-node processors
|
|
|
|
* Assumption: Number of cores in each internal node is the same.
|
2010-09-30 12:38:57 +00:00
|
|
|
* (2) AMD processors supporting compute units
|
2009-09-03 07:40:21 +00:00
|
|
|
*/
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void amd_get_topology(struct cpuinfo_x86 *c)
|
2009-09-03 07:40:21 +00:00
|
|
|
{
|
2010-09-30 12:36:28 +00:00
|
|
|
/* get information required for multi-node processors */
|
2015-12-07 09:39:41 +00:00
|
|
|
if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
|
2018-04-27 21:48:01 +00:00
|
|
|
int err;
|
2017-02-05 10:50:21 +00:00
|
|
|
u32 eax, ebx, ecx, edx;
|
2010-09-30 12:38:57 +00:00
|
|
|
|
2017-02-05 10:50:21 +00:00
|
|
|
cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
|
|
|
|
|
2023-08-14 08:18:31 +00:00
|
|
|
c->topo.die_id = ecx & 0xff;
|
2017-02-05 10:50:21 +00:00
|
|
|
|
|
|
|
if (c->x86 == 0x15)
|
2023-08-14 08:18:35 +00:00
|
|
|
c->topo.cu_id = ebx & 0xff;
|
2016-11-08 15:30:54 +00:00
|
|
|
|
2017-02-05 10:50:22 +00:00
|
|
|
if (c->x86 >= 0x17) {
|
2023-08-14 08:18:34 +00:00
|
|
|
c->topo.core_id = ebx & 0xff;
|
2017-02-05 10:50:22 +00:00
|
|
|
|
|
|
|
if (smp_num_siblings > 1)
|
|
|
|
c->x86_max_cores /= smp_num_siblings;
|
|
|
|
}
|
|
|
|
|
2016-11-08 15:30:54 +00:00
|
|
|
/*
|
2018-04-27 21:48:01 +00:00
|
|
|
* In case leaf B is available, use it to derive
|
|
|
|
* topology information.
|
2016-11-08 15:30:54 +00:00
|
|
|
*/
|
2018-04-27 21:48:01 +00:00
|
|
|
err = detect_extended_topology(c);
|
|
|
|
if (!err)
|
|
|
|
c->x86_coreid_bits = get_count_order(c->x86_max_cores);
|
|
|
|
|
2023-08-14 08:18:38 +00:00
|
|
|
cacheinfo_amd_init_llc_id(c);
|
2018-04-27 21:34:37 +00:00
|
|
|
|
2010-09-30 12:36:28 +00:00
|
|
|
} else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
|
2010-09-30 12:38:57 +00:00
|
|
|
u64 value;
|
|
|
|
|
2010-09-30 12:36:28 +00:00
|
|
|
rdmsrl(MSR_FAM10H_NODE_ID, value);
|
2023-08-14 08:18:31 +00:00
|
|
|
c->topo.die_id = value & 7;
|
2023-08-14 08:18:38 +00:00
|
|
|
c->topo.llc_id = c->topo.die_id;
|
2010-09-30 12:36:28 +00:00
|
|
|
} else
|
2009-09-03 07:40:21 +00:00
|
|
|
return;
|
|
|
|
|
2015-06-15 08:28:15 +00:00
|
|
|
if (nodes_per_socket > 1) {
|
2010-09-30 12:36:28 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_AMD_DCM);
|
2017-07-31 08:51:58 +00:00
|
|
|
legacy_fixup_core_id(c);
|
2010-09-30 12:36:28 +00:00
|
|
|
}
|
2009-09-03 07:40:21 +00:00
|
|
|
}
|
|
|
|
|
2008-09-08 00:58:50 +00:00
|
|
|
/*
|
2013-09-18 04:00:43 +00:00
|
|
|
* On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
|
2008-09-08 00:58:50 +00:00
|
|
|
* Assumes number of cores is a power of two.
|
|
|
|
*/
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void amd_detect_cmp(struct cpuinfo_x86 *c)
|
2008-09-08 00:58:50 +00:00
|
|
|
{
|
|
|
|
unsigned bits;
|
|
|
|
|
|
|
|
bits = c->x86_coreid_bits;
|
|
|
|
/* Low order bits define the core id (index of core in socket) */
|
2023-08-14 08:18:34 +00:00
|
|
|
c->topo.core_id = c->topo.initial_apicid & ((1 << bits)-1);
|
2008-09-08 00:58:50 +00:00
|
|
|
/* Convert the initial APIC ID into the socket ID */
|
2023-08-14 08:18:30 +00:00
|
|
|
c->topo.pkg_id = c->topo.initial_apicid >> bits;
|
2009-06-19 08:59:09 +00:00
|
|
|
/* use socket ID also for last level cache */
|
2023-08-14 08:18:38 +00:00
|
|
|
c->topo.llc_id = c->topo.die_id = c->topo.pkg_id;
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
|
|
|
|
2015-06-15 08:28:15 +00:00
|
|
|
u32 amd_get_nodes_per_socket(void)
|
|
|
|
{
|
|
|
|
return nodes_per_socket;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket);
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void srat_detect_node(struct cpuinfo_x86 *c)
|
2008-09-08 00:58:54 +00:00
|
|
|
{
|
2011-01-23 13:37:40 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
2008-09-08 00:58:54 +00:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int node;
|
2023-08-14 08:18:29 +00:00
|
|
|
unsigned apicid = c->topo.apicid;
|
2008-09-08 00:58:54 +00:00
|
|
|
|
2011-01-23 13:37:39 +00:00
|
|
|
node = numa_cpu_node(cpu);
|
|
|
|
if (node == NUMA_NO_NODE)
|
2023-08-14 08:18:38 +00:00
|
|
|
node = per_cpu_llc_id(cpu);
|
2008-09-08 00:58:54 +00:00
|
|
|
|
2011-12-05 08:20:37 +00:00
|
|
|
/*
|
2012-04-02 16:06:48 +00:00
|
|
|
* On multi-fabric platform (e.g. Numascale NumaChip) a
|
|
|
|
* platform-specific handler needs to be called to fixup some
|
|
|
|
* IDs of the CPU.
|
2011-12-05 08:20:37 +00:00
|
|
|
*/
|
2012-04-02 16:06:48 +00:00
|
|
|
if (x86_cpuinit.fixup_cpu_id)
|
2011-12-05 08:20:37 +00:00
|
|
|
x86_cpuinit.fixup_cpu_id(c, node);
|
|
|
|
|
2008-09-08 00:58:54 +00:00
|
|
|
if (!node_online(node)) {
|
2011-01-23 13:37:39 +00:00
|
|
|
/*
|
|
|
|
* Two possibilities here:
|
|
|
|
*
|
|
|
|
* - The CPU is missing memory and no node was created. In
|
|
|
|
* that case try picking one from a nearby CPU.
|
|
|
|
*
|
|
|
|
* - The APIC IDs differ from the HyperTransport node IDs
|
|
|
|
* which the K8 northbridge parsing fills in. Assume
|
|
|
|
* they are all increased by a constant offset, but in
|
|
|
|
* the same order as the HT nodeids. If that doesn't
|
|
|
|
* result in a usable node fall back to the path for the
|
|
|
|
* previous case.
|
|
|
|
*
|
|
|
|
* This workaround operates directly on the mapping between
|
|
|
|
* APIC ID and NUMA node, assuming certain relationship
|
|
|
|
* between APIC ID, HT node ID and NUMA topology. As going
|
|
|
|
* through CPU mapping may alter the outcome, directly
|
|
|
|
* access __apicid_to_node[].
|
|
|
|
*/
|
2023-08-14 08:18:29 +00:00
|
|
|
int ht_nodeid = c->topo.initial_apicid;
|
2008-09-08 00:58:54 +00:00
|
|
|
|
2016-01-13 12:39:40 +00:00
|
|
|
if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
|
2011-01-23 13:37:39 +00:00
|
|
|
node = __apicid_to_node[ht_nodeid];
|
2008-09-08 00:58:54 +00:00
|
|
|
/* Pick a nearby node */
|
|
|
|
if (!node_online(node))
|
|
|
|
node = nearby_node(apicid);
|
|
|
|
}
|
|
|
|
numa_set_node(cpu, node);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void early_init_amd_mc(struct cpuinfo_x86 *c)
|
2008-09-08 00:58:50 +00:00
|
|
|
{
|
2015-06-04 16:55:25 +00:00
|
|
|
#ifdef CONFIG_SMP
|
2008-09-08 00:58:50 +00:00
|
|
|
unsigned bits, ecx;
|
|
|
|
|
|
|
|
/* Multi core CPU? */
|
|
|
|
if (c->extended_cpuid_level < 0x80000008)
|
|
|
|
return;
|
|
|
|
|
|
|
|
ecx = cpuid_ecx(0x80000008);
|
|
|
|
|
|
|
|
c->x86_max_cores = (ecx & 0xff) + 1;
|
|
|
|
|
|
|
|
/* CPU telling us the core id bits shift? */
|
|
|
|
bits = (ecx >> 12) & 0xF;
|
|
|
|
|
|
|
|
/* Otherwise recompute */
|
|
|
|
if (bits == 0) {
|
|
|
|
while ((1 << bits) < c->x86_max_cores)
|
|
|
|
bits++;
|
|
|
|
}
|
|
|
|
|
|
|
|
c->x86_coreid_bits = bits;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void bsp_init_amd(struct cpuinfo_x86 *c)
|
2011-08-05 18:04:09 +00:00
|
|
|
{
|
|
|
|
if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
|
|
|
|
|
|
|
|
if (c->x86 > 0x10 ||
|
|
|
|
(c->x86 == 0x10 && c->x86_model >= 0x2)) {
|
|
|
|
u64 val;
|
|
|
|
|
|
|
|
rdmsrl(MSR_K7_HWCR, val);
|
|
|
|
if (!(val & BIT(24)))
|
2016-02-02 03:45:02 +00:00
|
|
|
pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n");
|
2011-08-05 18:04:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (c->x86 == 0x15) {
|
|
|
|
unsigned long upperbit;
|
|
|
|
u32 cpuid, assoc;
|
|
|
|
|
|
|
|
cpuid = cpuid_edx(0x80000005);
|
|
|
|
assoc = cpuid >> 16 & 0xff;
|
|
|
|
upperbit = ((cpuid >> 24) << 10) / assoc;
|
|
|
|
|
|
|
|
va_align.mask = (upperbit - 1) & PAGE_MASK;
|
|
|
|
va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
|
2015-03-27 11:38:21 +00:00
|
|
|
|
|
|
|
/* A random value per boot for bit slice [12:upper_bit) */
|
2022-10-05 15:43:22 +00:00
|
|
|
va_align.bits = get_random_u32() & va_align.mask;
|
2011-08-05 18:04:09 +00:00
|
|
|
}
|
2015-08-10 10:19:54 +00:00
|
|
|
|
|
|
|
if (cpu_has(c, X86_FEATURE_MWAITX))
|
|
|
|
use_mwaitx_delay();
|
2016-01-14 02:50:04 +00:00
|
|
|
|
|
|
|
if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
|
|
|
|
u32 ecx;
|
|
|
|
|
|
|
|
ecx = cpuid_ecx(0x8000001e);
|
2021-01-11 10:04:29 +00:00
|
|
|
__max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1;
|
2016-01-14 02:50:04 +00:00
|
|
|
} else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
|
|
|
|
u64 value;
|
|
|
|
|
|
|
|
rdmsrl(MSR_FAM10H_NODE_ID, value);
|
2021-01-11 10:04:29 +00:00
|
|
|
__max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1;
|
2016-01-14 02:50:04 +00:00
|
|
|
}
|
2018-04-26 02:04:24 +00:00
|
|
|
|
2018-07-02 21:35:53 +00:00
|
|
|
if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
|
|
|
|
!boot_cpu_has(X86_FEATURE_VIRT_SSBD) &&
|
|
|
|
c->x86 >= 0x15 && c->x86 <= 0x17) {
|
2018-04-26 02:04:24 +00:00
|
|
|
unsigned int bit;
|
|
|
|
|
|
|
|
switch (c->x86) {
|
|
|
|
case 0x15: bit = 54; break;
|
|
|
|
case 0x16: bit = 33; break;
|
|
|
|
case 0x17: bit = 10; break;
|
|
|
|
default: return;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Try to cache the base value so further operations can
|
2018-05-09 19:41:38 +00:00
|
|
|
* avoid RMW. If that faults, do not enable SSBD.
|
2018-04-26 02:04:24 +00:00
|
|
|
*/
|
|
|
|
if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {
|
2018-05-10 18:21:36 +00:00
|
|
|
setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);
|
2018-05-09 19:41:38 +00:00
|
|
|
setup_force_cpu_cap(X86_FEATURE_SSBD);
|
|
|
|
x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
|
2018-04-26 02:04:24 +00:00
|
|
|
}
|
|
|
|
}
|
2020-05-05 22:36:15 +00:00
|
|
|
|
|
|
|
resctrl_cpu_detect(c);
|
2023-10-31 22:30:59 +00:00
|
|
|
|
|
|
|
/* Figure out Zen generations: */
|
|
|
|
switch (c->x86) {
|
|
|
|
case 0x17: {
|
|
|
|
switch (c->x86_model) {
|
|
|
|
case 0x00 ... 0x2f:
|
|
|
|
case 0x50 ... 0x5f:
|
2023-12-02 11:50:23 +00:00
|
|
|
setup_force_cpu_cap(X86_FEATURE_ZEN1);
|
2023-10-31 22:30:59 +00:00
|
|
|
break;
|
|
|
|
case 0x30 ... 0x4f:
|
|
|
|
case 0x60 ... 0x7f:
|
|
|
|
case 0x90 ... 0x91:
|
|
|
|
case 0xa0 ... 0xaf:
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_ZEN2);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
goto warn;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 0x19: {
|
|
|
|
switch (c->x86_model) {
|
|
|
|
case 0x00 ... 0x0f:
|
|
|
|
case 0x20 ... 0x5f:
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_ZEN3);
|
|
|
|
break;
|
|
|
|
case 0x10 ... 0x1f:
|
|
|
|
case 0x60 ... 0xaf:
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_ZEN4);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
goto warn;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
warn:
|
|
|
|
WARN_ONCE(1, "Family 0x%x, model: 0x%x??\n", c->x86, c->x86_model);
|
2011-08-05 18:04:09 +00:00
|
|
|
}
|
|
|
|
|
2017-12-04 16:57:23 +00:00
|
|
|
static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
u64 msr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BIOS support is required for SME and SEV.
|
|
|
|
* For SME: If BIOS has enabled SME then adjust x86_phys_bits by
|
|
|
|
* the SME physical address space reduction value.
|
|
|
|
* If BIOS has not enabled SME then don't advertise the
|
|
|
|
* SME feature (set in scattered.c).
|
2022-02-16 03:44:46 +00:00
|
|
|
* If the kernel has not enabled SME via any means then
|
|
|
|
* don't advertise the SME feature.
|
2017-12-04 16:57:23 +00:00
|
|
|
* For SEV: If BIOS has not enabled SEV then don't advertise the
|
2020-09-07 13:15:06 +00:00
|
|
|
* SEV and SEV_ES feature (set in scattered.c).
|
2017-12-04 16:57:23 +00:00
|
|
|
*
|
|
|
|
* In all cases, since support for SME and SEV requires long mode,
|
|
|
|
* don't advertise the feature under CONFIG_X86_32.
|
|
|
|
*/
|
|
|
|
if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {
|
|
|
|
/* Check if memory encryption is enabled */
|
2021-04-27 11:16:36 +00:00
|
|
|
rdmsrl(MSR_AMD64_SYSCFG, msr);
|
|
|
|
if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
|
2017-12-04 16:57:23 +00:00
|
|
|
goto clear_all;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Always adjust physical address bits. Even though this
|
|
|
|
* will be a value above 32-bits this is still done for
|
|
|
|
* CONFIG_X86_32 so that accurate values are reported.
|
|
|
|
*/
|
|
|
|
c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f;
|
|
|
|
|
|
|
|
if (IS_ENABLED(CONFIG_X86_32))
|
|
|
|
goto clear_all;
|
|
|
|
|
2022-02-16 03:44:46 +00:00
|
|
|
if (!sme_me_mask)
|
|
|
|
setup_clear_cpu_cap(X86_FEATURE_SME);
|
|
|
|
|
2017-12-04 16:57:23 +00:00
|
|
|
rdmsrl(MSR_K7_HWCR, msr);
|
|
|
|
if (!(msr & MSR_K7_HWCR_SMMLOCK))
|
|
|
|
goto clear_sev;
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
clear_all:
|
2020-01-15 22:05:16 +00:00
|
|
|
setup_clear_cpu_cap(X86_FEATURE_SME);
|
2017-12-04 16:57:23 +00:00
|
|
|
clear_sev:
|
2020-01-15 22:05:16 +00:00
|
|
|
setup_clear_cpu_cap(X86_FEATURE_SEV);
|
2020-09-07 13:15:06 +00:00
|
|
|
setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
|
2017-12-04 16:57:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void early_init_amd(struct cpuinfo_x86 *c)
|
2008-01-30 12:32:40 +00:00
|
|
|
{
|
2018-06-22 09:34:11 +00:00
|
|
|
u64 value;
|
2017-07-17 21:10:23 +00:00
|
|
|
u32 dummy;
|
|
|
|
|
2008-09-08 00:58:50 +00:00
|
|
|
early_init_amd_mc(c);
|
|
|
|
|
2018-07-19 20:55:28 +00:00
|
|
|
if (c->x86 >= 0xf)
|
|
|
|
set_cpu_cap(c, X86_FEATURE_K8);
|
|
|
|
|
2017-07-17 21:10:23 +00:00
|
|
|
rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
|
|
|
|
|
2008-11-18 00:11:37 +00:00
|
|
|
/*
|
|
|
|
* c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
|
|
|
|
* with P/T states and does not stop in deep C-states
|
|
|
|
*/
|
|
|
|
if (c->x86_power & (1 << 8)) {
|
2008-09-06 08:52:28 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
|
2008-11-18 00:11:37 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
|
|
|
|
}
|
2008-09-04 19:09:43 +00:00
|
|
|
|
2016-01-14 02:50:06 +00:00
|
|
|
/* Bit 12 of 8000_0007 edx is accumulated power mechanism. */
|
|
|
|
if (c->x86_power & BIT(12))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_ACC_POWER);
|
|
|
|
|
2021-05-14 13:59:20 +00:00
|
|
|
/* Bit 14 indicates the Runtime Average Power Limit interface. */
|
|
|
|
if (c->x86_power & BIT(14))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_RAPL);
|
|
|
|
|
2008-09-08 00:58:54 +00:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
set_cpu_cap(c, X86_FEATURE_SYSCALL32);
|
|
|
|
#else
|
2008-09-04 19:09:43 +00:00
|
|
|
/* Set MTRR capability flag if appropriate */
|
2008-09-08 00:58:54 +00:00
|
|
|
if (c->x86 == 5)
|
|
|
|
if (c->x86_model == 13 || c->x86_model == 9 ||
|
2018-01-01 01:52:10 +00:00
|
|
|
(c->x86_model == 8 && c->x86_stepping >= 8))
|
2008-09-08 00:58:54 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_K6_MTRR);
|
|
|
|
#endif
|
2009-06-08 13:55:09 +00:00
|
|
|
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
|
2015-04-27 15:25:51 +00:00
|
|
|
/*
|
|
|
|
* ApicID can always be treated as an 8-bit value for AMD APIC versions
|
|
|
|
* >= 0x10, but even old K8s came out of reset with version 0x10. So, we
|
|
|
|
* can safely set X86_FEATURE_EXTD_APICID unconditionally for families
|
|
|
|
* after 16h.
|
|
|
|
*/
|
2016-04-05 06:29:51 +00:00
|
|
|
if (boot_cpu_has(X86_FEATURE_APIC)) {
|
|
|
|
if (c->x86 > 0x16)
|
2009-06-08 13:55:09 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
|
2016-04-05 06:29:51 +00:00
|
|
|
else if (c->x86 >= 0xf) {
|
|
|
|
/* check CPU config space for extended APIC ID */
|
|
|
|
unsigned int val;
|
|
|
|
|
|
|
|
val = read_pci_config(0, 24, 0, 0x68);
|
|
|
|
if ((val >> 17 & 0x3) == 0x3)
|
|
|
|
set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
|
|
|
|
}
|
2009-06-08 13:55:09 +00:00
|
|
|
}
|
|
|
|
#endif
|
2014-01-14 23:07:11 +00:00
|
|
|
|
2014-09-22 11:17:48 +00:00
|
|
|
/*
|
|
|
|
* This is only needed to tell the kernel whether to use VMCALL
|
|
|
|
* and VMMCALL. VMMCALL is never executed except under virt, so
|
|
|
|
* we can set it unconditionally.
|
|
|
|
*/
|
|
|
|
set_cpu_cap(c, X86_FEATURE_VMMCALL);
|
|
|
|
|
2014-01-14 23:07:11 +00:00
|
|
|
/* F16h erratum 793, CVE-2013-6885 */
|
2014-03-09 17:05:24 +00:00
|
|
|
if (c->x86 == 0x16 && c->x86_model <= 0xf)
|
|
|
|
msr_set_bit(MSR_AMD64_LS_CFG, 15);
|
2008-01-30 12:32:40 +00:00
|
|
|
|
2017-12-04 16:57:23 +00:00
|
|
|
early_detect_mem_encrypt(c);
|
2018-06-05 22:57:38 +00:00
|
|
|
|
2018-06-22 09:34:11 +00:00
|
|
|
/* Re-enable TopologyExtensions if switched off by BIOS */
|
|
|
|
if (c->x86 == 0x15 &&
|
|
|
|
(c->x86_model >= 0x10 && c->x86_model <= 0x6f) &&
|
|
|
|
!cpu_has(c, X86_FEATURE_TOPOEXT)) {
|
|
|
|
|
|
|
|
if (msr_set_bit(0xc0011005, 54) > 0) {
|
|
|
|
rdmsrl(0xc0011005, value);
|
|
|
|
if (value & BIT_64(54)) {
|
|
|
|
set_cpu_cap(c, X86_FEATURE_TOPOEXT);
|
|
|
|
pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-23 16:54:33 +00:00
|
|
|
if (cpu_has(c, X86_FEATURE_TOPOEXT))
|
|
|
|
smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
|
2023-09-05 05:04:46 +00:00
|
|
|
|
2023-09-05 05:04:47 +00:00
|
|
|
if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) {
|
2023-09-05 05:04:46 +00:00
|
|
|
if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB))
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
|
|
|
|
else if (c->x86 >= 0x19 && !wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) {
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
|
|
|
|
setup_force_cpu_cap(X86_FEATURE_SBPB);
|
|
|
|
}
|
|
|
|
}
|
2016-12-09 18:29:09 +00:00
|
|
|
}
|
2013-03-20 14:07:27 +00:00
|
|
|
|
2014-06-24 11:25:04 +00:00
|
|
|
static void init_amd_k8(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
u32 level;
|
|
|
|
u64 value;
|
|
|
|
|
|
|
|
/* On C+ stepping K8 rep microcode works well for copy/memset */
|
|
|
|
level = cpuid_eax(1);
|
|
|
|
if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
|
|
|
|
set_cpu_cap(c, X86_FEATURE_REP_GOOD);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some BIOSes incorrectly force this feature, but only K8 revision D
|
|
|
|
* (model = 0x14) and later actually support it.
|
|
|
|
* (AMD Erratum #110, docId: 25759).
|
|
|
|
*/
|
|
|
|
if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
|
|
|
|
if (!rdmsrl_amd_safe(0xc001100d, &value)) {
|
|
|
|
value &= ~BIT_64(32);
|
|
|
|
wrmsrl_amd_safe(0xc001100d, value);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!c->x86_model_id[0])
|
|
|
|
strcpy(c->x86_model_id, "Hammer");
|
2014-07-29 15:41:23 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/*
|
|
|
|
* Disable TLB flush filter by setting HWCR.FFDIS on K8
|
|
|
|
* bit 6 of msr C001_0015
|
|
|
|
*
|
|
|
|
* Errata 63 for SH-B3 steppings
|
|
|
|
* Errata 122 for all steppings (F+ have it disabled by default)
|
|
|
|
*/
|
|
|
|
msr_set_bit(MSR_K7_HWCR, 6);
|
|
|
|
#endif
|
2016-04-08 00:31:49 +00:00
|
|
|
set_cpu_bug(c, X86_BUG_SWAPGS_FENCE);
|
2023-11-03 22:20:11 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check models and steppings affected by erratum 400. This is
|
|
|
|
* used to select the proper idle routine and to enable the
|
|
|
|
* check whether the machine is affected in arch_post_acpi_subsys_init()
|
|
|
|
* which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
|
|
|
|
*/
|
|
|
|
if (c->x86_model > 0x41 ||
|
|
|
|
(c->x86_model == 0x41 && c->x86_stepping >= 0x2))
|
|
|
|
setup_force_cpu_bug(X86_BUG_AMD_E400);
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void init_amd_gh(struct cpuinfo_x86 *c)
|
|
|
|
{
|
2018-03-07 07:39:17 +00:00
|
|
|
#ifdef CONFIG_MMCONF_FAM10H
|
2014-06-24 11:25:04 +00:00
|
|
|
/* do this for boot cpu */
|
|
|
|
if (c == &boot_cpu_data)
|
|
|
|
check_enable_amd_mmconf_dmi();
|
|
|
|
|
|
|
|
fam10h_check_enable_mmcfg();
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable GART TLB Walk Errors on Fam10h. We do this here because this
|
|
|
|
* is always needed when GART is enabled, even in a kernel which has no
|
|
|
|
* MCE support built in. BIOS should disable GartTlbWlk Errors already.
|
|
|
|
* If it doesn't, we do it here as suggested by the BKDG.
|
|
|
|
*
|
|
|
|
* Fixes: https://bugzilla.kernel.org/show_bug.cgi?id=33012
|
|
|
|
*/
|
|
|
|
msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* On family 10h BIOS may not have properly enabled WC+ support, causing
|
|
|
|
* it to be converted to CD memtype. This may result in performance
|
|
|
|
* degradation for certain nested-paging guests. Prevent this conversion
|
|
|
|
* by clearing bit 24 in MSR_AMD64_BU_CFG2.
|
|
|
|
*
|
|
|
|
* NOTE: we want to use the _safe accessors so as not to #GP kvm
|
|
|
|
* guests on older kvm hosts.
|
|
|
|
*/
|
|
|
|
msr_clear_bit(MSR_AMD64_BU_CFG2, 24);
|
|
|
|
|
2023-11-03 18:58:53 +00:00
|
|
|
set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
|
2023-11-03 22:20:11 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check models and steppings affected by erratum 400. This is
|
|
|
|
* used to select the proper idle routine and to enable the
|
|
|
|
* check whether the machine is affected in arch_post_acpi_subsys_init()
|
|
|
|
* which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
|
|
|
|
*/
|
|
|
|
if (c->x86_model > 0x2 ||
|
|
|
|
(c->x86_model == 0x2 && c->x86_stepping >= 0x1))
|
|
|
|
setup_force_cpu_bug(X86_BUG_AMD_E400);
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
|
|
|
|
2016-09-02 05:35:50 +00:00
|
|
|
static void init_amd_ln(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Apply erratum 665 fix unconditionally so machines without a BIOS
|
|
|
|
* fix work.
|
|
|
|
*/
|
|
|
|
msr_set_bit(MSR_AMD64_DE_CFG, 31);
|
|
|
|
}
|
|
|
|
|
2019-08-19 15:52:35 +00:00
|
|
|
static bool rdrand_force;
|
|
|
|
|
|
|
|
static int __init rdrand_cmdline(char *str)
|
|
|
|
{
|
|
|
|
if (!str)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!strcmp(str, "force"))
|
|
|
|
rdrand_force = true;
|
|
|
|
else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("rdrand", rdrand_cmdline);
|
|
|
|
|
|
|
|
static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Saving of the MSR used to hide the RDRAND support during
|
|
|
|
* suspend/resume is done by arch/x86/power/cpu.c, which is
|
|
|
|
* dependent on CONFIG_PM_SLEEP.
|
|
|
|
*/
|
|
|
|
if (!IS_ENABLED(CONFIG_PM_SLEEP))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
x86/rdrand: Remove "nordrand" flag in favor of "random.trust_cpu"
The decision of whether or not to trust RDRAND is controlled by the
"random.trust_cpu" boot time parameter or the CONFIG_RANDOM_TRUST_CPU
compile time default. The "nordrand" flag was added during the early
days of RDRAND, when there were worries that merely using its values
could compromise the RNG. However, these days, RDRAND values are not
used directly but always go through the RNG's hash function, making
"nordrand" no longer useful.
Rather, the correct switch is "random.trust_cpu", which not only handles
the relevant trust issue directly, but also is general to multiple CPU
types, not just x86.
However, x86 RDRAND does have a history of being occasionally
problematic. Prior, when the kernel would notice something strange, it'd
warn in dmesg and suggest enabling "nordrand". We can improve on that by
making the test a little bit better and then taking the step of
automatically disabling RDRAND if we detect it's problematic.
Also disable RDSEED if the RDRAND test fails.
Cc: x86@kernel.org
Cc: Theodore Ts'o <tytso@mit.edu>
Suggested-by: H. Peter Anvin <hpa@zytor.com>
Suggested-by: Borislav Petkov <bp@suse.de>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-09 10:43:06 +00:00
|
|
|
* The self-test can clear X86_FEATURE_RDRAND, so check for
|
2019-08-19 15:52:35 +00:00
|
|
|
* RDRAND support using the CPUID function directly.
|
|
|
|
*/
|
|
|
|
if (!(cpuid_ecx(1) & BIT(30)) || rdrand_force)
|
|
|
|
return;
|
|
|
|
|
|
|
|
msr_clear_bit(MSR_AMD64_CPUID_FN_1, 62);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Verify that the CPUID change has occurred in case the kernel is
|
|
|
|
* running virtualized and the hypervisor doesn't support the MSR.
|
|
|
|
*/
|
|
|
|
if (cpuid_ecx(1) & BIT(30)) {
|
|
|
|
pr_info_once("BIOS may not properly restore RDRAND after suspend, but hypervisor does not support hiding RDRAND via CPUID.\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_RDRAND);
|
|
|
|
pr_info_once("BIOS may not properly restore RDRAND after suspend, hiding RDRAND via CPUID. Use rdrand=force to reenable.\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_amd_jg(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Some BIOS implementations do not restore proper RDRAND support
|
|
|
|
* across suspend and resume. Check on whether to hide the RDRAND
|
|
|
|
* instruction support via CPUID.
|
|
|
|
*/
|
|
|
|
clear_rdrand_cpuid_bit(c);
|
|
|
|
}
|
|
|
|
|
2014-06-24 11:25:04 +00:00
|
|
|
static void init_amd_bd(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
u64 value;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The way access filter has a performance penalty on some workloads.
|
|
|
|
* Disable it on the affected CPUs.
|
|
|
|
*/
|
|
|
|
if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
|
2015-11-23 10:12:23 +00:00
|
|
|
if (!rdmsrl_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) {
|
2014-06-24 11:25:04 +00:00
|
|
|
value |= 0x1E;
|
2015-11-23 10:12:23 +00:00
|
|
|
wrmsrl_safe(MSR_F15H_IC_CFG, value);
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
|
|
|
}
|
2019-08-19 15:52:35 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Some BIOS implementations do not restore proper RDRAND support
|
|
|
|
* across suspend and resume. Check on whether to hide the RDRAND
|
|
|
|
* instruction support via CPUID.
|
|
|
|
*/
|
|
|
|
clear_rdrand_cpuid_bit(c);
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
|
|
|
|
2023-11-01 10:14:59 +00:00
|
|
|
static void fix_erratum_1386(struct cpuinfo_x86 *c)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Work around Erratum 1386. The XSAVES instruction malfunctions in
|
|
|
|
* certain circumstances on Zen1/2 uarch, and not all parts have had
|
|
|
|
* updated microcode at the time of writing (March 2023).
|
|
|
|
*
|
|
|
|
* Affected parts all have no supervisor XSAVE states, meaning that
|
|
|
|
* the XSAVEC instruction (which works fine) is equivalent.
|
|
|
|
*/
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_XSAVES);
|
|
|
|
}
|
|
|
|
|
2022-06-14 21:16:04 +00:00
|
|
|
void init_spectral_chicken(struct cpuinfo_x86 *c)
|
|
|
|
{
|
2022-06-27 22:21:17 +00:00
|
|
|
#ifdef CONFIG_CPU_UNRET_ENTRY
|
2022-06-14 21:16:04 +00:00
|
|
|
u64 value;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* On Zen2 we offer this chicken (bit) on the altar of Speculation.
|
|
|
|
*
|
|
|
|
* This suppresses speculation from the middle of a basic block, i.e. it
|
|
|
|
* suppresses non-branch predictions.
|
|
|
|
*/
|
2023-11-01 10:20:01 +00:00
|
|
|
if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
|
2022-06-14 21:16:04 +00:00
|
|
|
if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) {
|
|
|
|
value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT;
|
|
|
|
wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value);
|
|
|
|
}
|
|
|
|
}
|
2022-06-27 22:21:17 +00:00
|
|
|
#endif
|
2022-06-14 21:16:04 +00:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:34:29 +00:00
|
|
|
static void init_amd_zen_common(void)
|
2017-09-07 17:08:21 +00:00
|
|
|
{
|
2023-12-02 11:50:23 +00:00
|
|
|
setup_force_cpu_cap(X86_FEATURE_ZEN);
|
2019-08-08 19:53:01 +00:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
node_reclaim_distance = 32;
|
|
|
|
#endif
|
2023-11-01 11:31:44 +00:00
|
|
|
}
|
|
|
|
|
2023-12-02 11:50:23 +00:00
|
|
|
static void init_amd_zen1(struct cpuinfo_x86 *c)
|
2023-11-01 11:31:44 +00:00
|
|
|
{
|
2023-11-01 11:34:29 +00:00
|
|
|
init_amd_zen_common();
|
2023-11-01 11:31:44 +00:00
|
|
|
fix_erratum_1386(c);
|
2019-08-08 19:53:01 +00:00
|
|
|
|
2022-06-24 13:41:21 +00:00
|
|
|
/* Fix up CPUID bits, but only if not virtualised. */
|
|
|
|
if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
|
|
|
|
|
|
|
|
/* Erratum 1076: CPB feature bit not being set in CPUID. */
|
|
|
|
if (!cpu_has(c, X86_FEATURE_CPB))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_CPB);
|
|
|
|
}
|
2023-11-01 11:52:01 +00:00
|
|
|
|
|
|
|
pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n");
|
|
|
|
setup_force_cpu_bug(X86_BUG_DIV0);
|
2017-09-07 17:08:21 +00:00
|
|
|
}
|
|
|
|
|
2023-07-15 11:41:28 +00:00
|
|
|
static bool cpu_has_zenbleed_microcode(void)
|
|
|
|
{
|
|
|
|
u32 good_rev = 0;
|
|
|
|
|
|
|
|
switch (boot_cpu_data.x86_model) {
|
|
|
|
case 0x30 ... 0x3f: good_rev = 0x0830107a; break;
|
|
|
|
case 0x60 ... 0x67: good_rev = 0x0860010b; break;
|
|
|
|
case 0x68 ... 0x6f: good_rev = 0x08608105; break;
|
|
|
|
case 0x70 ... 0x7f: good_rev = 0x08701032; break;
|
|
|
|
case 0xa0 ... 0xaf: good_rev = 0x08a00008; break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (boot_cpu_data.microcode < good_rev)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2023-11-01 11:38:35 +00:00
|
|
|
static void zen2_zenbleed_check(struct cpuinfo_x86 *c)
|
2023-07-15 11:41:28 +00:00
|
|
|
{
|
|
|
|
if (cpu_has(c, X86_FEATURE_HYPERVISOR))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!cpu_has(c, X86_FEATURE_AVX))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!cpu_has_zenbleed_microcode()) {
|
|
|
|
pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n");
|
|
|
|
msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
|
|
|
|
} else {
|
|
|
|
msr_clear_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-31 22:30:59 +00:00
|
|
|
static void init_amd_zen2(struct cpuinfo_x86 *c)
|
|
|
|
{
|
2023-11-01 11:34:29 +00:00
|
|
|
init_amd_zen_common();
|
2023-11-01 10:20:01 +00:00
|
|
|
init_spectral_chicken(c);
|
2023-11-01 10:14:59 +00:00
|
|
|
fix_erratum_1386(c);
|
2023-11-01 11:38:35 +00:00
|
|
|
zen2_zenbleed_check(c);
|
2023-10-31 22:30:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void init_amd_zen3(struct cpuinfo_x86 *c)
|
|
|
|
{
|
2023-11-01 11:34:29 +00:00
|
|
|
init_amd_zen_common();
|
|
|
|
|
2023-11-01 10:28:31 +00:00
|
|
|
if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
|
|
|
|
/*
|
|
|
|
* Zen3 (Fam19 model < 0x10) parts are not susceptible to
|
|
|
|
* Branch Type Confusion, but predate the allocation of the
|
|
|
|
* BTC_NO bit.
|
|
|
|
*/
|
|
|
|
if (!cpu_has(c, X86_FEATURE_BTC_NO))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_BTC_NO);
|
|
|
|
}
|
2023-10-31 22:30:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void init_amd_zen4(struct cpuinfo_x86 *c)
|
|
|
|
{
|
2023-11-01 11:34:29 +00:00
|
|
|
init_amd_zen_common();
|
2023-11-03 22:21:56 +00:00
|
|
|
|
|
|
|
if (!cpu_has(c, X86_FEATURE_HYPERVISOR))
|
|
|
|
msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT);
|
2023-10-31 22:30:59 +00:00
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void init_amd(struct cpuinfo_x86 *c)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2023-09-21 11:49:40 +00:00
|
|
|
u64 vm_cr;
|
|
|
|
|
2008-01-30 12:32:40 +00:00
|
|
|
early_init_amd(c);
|
|
|
|
|
2008-02-22 22:10:33 +00:00
|
|
|
/*
|
|
|
|
* Bit 31 in normal CPUID used for nonstandard 3DNow ID;
|
2008-02-26 07:49:57 +00:00
|
|
|
* 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
|
2008-02-22 22:10:33 +00:00
|
|
|
*/
|
2008-02-26 07:49:57 +00:00
|
|
|
clear_cpu_cap(c, 0*32+31);
|
2008-02-22 22:10:33 +00:00
|
|
|
|
2010-06-02 18:29:21 +00:00
|
|
|
if (c->x86 >= 0x10)
|
2008-09-08 00:58:54 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_REP_GOOD);
|
2009-08-29 20:17:14 +00:00
|
|
|
|
x86: set FSRS automatically on AMD CPUs that have FSRM
So Intel introduced the FSRS ("Fast Short REP STOS") CPU capability bit,
because they seem to have done the (much simpler) REP STOS optimizations
separately and later than the REP MOVS one.
In contrast, when AMD introduced support for FSRM ("Fast Short REP
MOVS"), in the Zen 3 core, it appears to have improved the REP STOS case
at the same time, and since the FSRS bit was added by Intel later, it
doesn't show up on those AMD Zen 3 cores.
And now that we made use of FSRS for the "rep stos" conditional, that
made those AMD machines unnecessarily slower. The Intel situation where
"rep movs" is fast, but "rep stos" isn't, is just odd. The 'stos' case
is a lot simpler with no aliasing, no mutual alignment issues, no
complicated cases.
So this just sets FSRS automatically when FSRM is available on AMD
machines, to get back all the nice REP STOS goodness in Zen 3.
Reported-and-tested-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2023-04-17 19:13:35 +00:00
|
|
|
/* AMD FSRM also implies FSRS */
|
|
|
|
if (cpu_has(c, X86_FEATURE_FSRM))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_FSRS);
|
|
|
|
|
2009-08-29 20:17:14 +00:00
|
|
|
/* get apicid instead of initial apic id from cpuid */
|
2023-08-14 08:18:29 +00:00
|
|
|
c->topo.apicid = read_apic_id();
|
2008-09-08 00:58:50 +00:00
|
|
|
|
|
|
|
/* K6s reports MCEs but don't actually have all the MSRs */
|
|
|
|
if (c->x86 < 6)
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_MCE);
|
2014-06-24 11:25:04 +00:00
|
|
|
|
|
|
|
switch (c->x86) {
|
|
|
|
case 4: init_amd_k5(c); break;
|
|
|
|
case 5: init_amd_k6(c); break;
|
|
|
|
case 6: init_amd_k7(c); break;
|
|
|
|
case 0xf: init_amd_k8(c); break;
|
|
|
|
case 0x10: init_amd_gh(c); break;
|
2016-09-02 05:35:50 +00:00
|
|
|
case 0x12: init_amd_ln(c); break;
|
2014-06-24 11:25:04 +00:00
|
|
|
case 0x15: init_amd_bd(c); break;
|
2019-08-19 15:52:35 +00:00
|
|
|
case 0x16: init_amd_jg(c); break;
|
2014-06-24 11:25:04 +00:00
|
|
|
}
|
2008-09-08 00:58:50 +00:00
|
|
|
|
2023-12-02 11:50:23 +00:00
|
|
|
if (boot_cpu_has(X86_FEATURE_ZEN1))
|
|
|
|
init_amd_zen1(c);
|
2023-10-31 22:30:59 +00:00
|
|
|
else if (boot_cpu_has(X86_FEATURE_ZEN2))
|
|
|
|
init_amd_zen2(c);
|
|
|
|
else if (boot_cpu_has(X86_FEATURE_ZEN3))
|
|
|
|
init_amd_zen3(c);
|
|
|
|
else if (boot_cpu_has(X86_FEATURE_ZEN4))
|
|
|
|
init_amd_zen4(c);
|
|
|
|
|
2017-11-28 21:01:06 +00:00
|
|
|
/*
|
|
|
|
* Enable workaround for FXSAVE leak on CPUs
|
|
|
|
* without a XSaveErPtr feature
|
|
|
|
*/
|
|
|
|
if ((c->x86 >= 6) && (!cpu_has(c, X86_FEATURE_XSAVEERPTR)))
|
2014-06-17 22:06:23 +00:00
|
|
|
set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-11-21 13:01:45 +00:00
|
|
|
cpu_detect_cache_sizes(c);
|
2005-04-16 22:25:15 +00:00
|
|
|
|
2018-06-15 18:48:39 +00:00
|
|
|
amd_detect_cmp(c);
|
|
|
|
amd_get_topology(c);
|
|
|
|
srat_detect_node(c);
|
2006-01-11 21:42:45 +00:00
|
|
|
|
2012-10-19 08:59:33 +00:00
|
|
|
init_amd_cacheinfo(c);
|
2007-04-02 10:14:12 +00:00
|
|
|
|
2023-09-21 11:49:40 +00:00
|
|
|
if (cpu_has(c, X86_FEATURE_SVM)) {
|
|
|
|
rdmsrl(MSR_VM_CR, vm_cr);
|
|
|
|
if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) {
|
|
|
|
pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n");
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_SVM);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-01-24 16:33:15 +00:00
|
|
|
if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) {
|
2018-01-08 22:09:21 +00:00
|
|
|
/*
|
x86: Remove X86_FEATURE_MFENCE_RDTSC
AMD and Intel both have serializing lfence (X86_FEATURE_LFENCE_RDTSC).
They've both had it for a long time, and AMD has had it enabled in Linux
since Spectre v1 was announced.
Back then, there was a proposal to remove the serializing mfence feature
bit (X86_FEATURE_MFENCE_RDTSC), since both AMD and Intel have
serializing lfence. At the time, it was (ahem) speculated that some
hypervisors might not yet support its removal, so it remained for the
time being.
Now a year-and-a-half later, it should be safe to remove.
I asked Andrew Cooper about whether it's still needed:
So if you're virtualised, you've got no choice in the matter. lfence
is either dispatch-serialising or not on AMD, and you won't be able to
change it.
Furthermore, you can't accurately tell what state the bit is in, because
the MSR might not be virtualised at all, or may not reflect the true
state in hardware. Worse still, attempting to set the bit may not be
successful even if there isn't a fault for doing so.
Xen sets the DE_CFG bit unconditionally, as does Linux by the looks of
things (see MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT). ISTR other hypervisor
vendors saying the same, but I don't have any information to hand.
If you are running under a hypervisor which has been updated, then
lfence will almost certainly be dispatch-serialising in practice, and
you'll almost certainly see the bit already set in DE_CFG. If you're
running under a hypervisor which hasn't been patched since Spectre,
you've already lost in many more ways.
I'd argue that X86_FEATURE_MFENCE_RDTSC is not worth keeping.
So remove it. This will reduce some code rot, and also make it easier
to hook barrier_nospec() up to a cmdline disable for performance
raisins, without having to need an alternative_3() macro.
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/d990aa51e40063acb9888e8c1b688e41355a9588.1562255067.git.jpoimboe@redhat.com
2019-07-04 15:46:37 +00:00
|
|
|
* Use LFENCE for execution serialization. On families which
|
2018-01-08 22:09:21 +00:00
|
|
|
* don't have that MSR, LFENCE is already serializing.
|
|
|
|
* msr_set_bit() uses the safe accessors, too, even if the MSR
|
|
|
|
* is not present.
|
|
|
|
*/
|
2022-11-14 11:44:01 +00:00
|
|
|
msr_set_bit(MSR_AMD64_DE_CFG,
|
|
|
|
MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT);
|
2018-01-08 22:09:21 +00:00
|
|
|
|
x86: Remove X86_FEATURE_MFENCE_RDTSC
AMD and Intel both have serializing lfence (X86_FEATURE_LFENCE_RDTSC).
They've both had it for a long time, and AMD has had it enabled in Linux
since Spectre v1 was announced.
Back then, there was a proposal to remove the serializing mfence feature
bit (X86_FEATURE_MFENCE_RDTSC), since both AMD and Intel have
serializing lfence. At the time, it was (ahem) speculated that some
hypervisors might not yet support its removal, so it remained for the
time being.
Now a year-and-a-half later, it should be safe to remove.
I asked Andrew Cooper about whether it's still needed:
So if you're virtualised, you've got no choice in the matter. lfence
is either dispatch-serialising or not on AMD, and you won't be able to
change it.
Furthermore, you can't accurately tell what state the bit is in, because
the MSR might not be virtualised at all, or may not reflect the true
state in hardware. Worse still, attempting to set the bit may not be
successful even if there isn't a fault for doing so.
Xen sets the DE_CFG bit unconditionally, as does Linux by the looks of
things (see MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT). ISTR other hypervisor
vendors saying the same, but I don't have any information to hand.
If you are running under a hypervisor which has been updated, then
lfence will almost certainly be dispatch-serialising in practice, and
you'll almost certainly see the bit already set in DE_CFG. If you're
running under a hypervisor which hasn't been patched since Spectre,
you've already lost in many more ways.
I'd argue that X86_FEATURE_MFENCE_RDTSC is not worth keeping.
So remove it. This will reduce some code rot, and also make it easier
to hook barrier_nospec() up to a cmdline disable for performance
raisins, without having to need an alternative_3() macro.
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/d990aa51e40063acb9888e8c1b688e41355a9588.1562255067.git.jpoimboe@redhat.com
2019-07-04 15:46:37 +00:00
|
|
|
/* A serializing LFENCE stops RDTSC speculation */
|
|
|
|
set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
|
2008-09-08 00:58:50 +00:00
|
|
|
}
|
2008-09-08 00:58:54 +00:00
|
|
|
|
2011-05-26 15:19:52 +00:00
|
|
|
/*
|
|
|
|
* Family 0x12 and above processors have APIC timer
|
|
|
|
* running in deep C states.
|
|
|
|
*/
|
|
|
|
if (c->x86 > 0x11)
|
2011-03-15 16:13:44 +00:00
|
|
|
set_cpu_cap(c, X86_FEATURE_ARAT);
|
2011-04-15 12:47:40 +00:00
|
|
|
|
x86/asm: Cleanup prefetch primitives
This is based on a patch originally by hpa.
With the current improvements to the alternatives, we can simply use %P1
as a mem8 operand constraint and rely on the toolchain to generate the
proper instruction sizes. For example, on 32-bit, where we use an empty
old instruction we get:
apply_alternatives: feat: 6*32+8, old: (c104648b, len: 4), repl: (c195566c, len: 4)
c104648b: alt_insn: 90 90 90 90
c195566c: rpl_insn: 0f 0d 4b 5c
...
apply_alternatives: feat: 6*32+8, old: (c18e09b4, len: 3), repl: (c1955948, len: 3)
c18e09b4: alt_insn: 90 90 90
c1955948: rpl_insn: 0f 0d 08
...
apply_alternatives: feat: 6*32+8, old: (c1190cf9, len: 7), repl: (c1955a79, len: 7)
c1190cf9: alt_insn: 90 90 90 90 90 90 90
c1955a79: rpl_insn: 0f 0d 0d a0 d4 85 c1
all with the proper padding done depending on the size of the
replacement instruction the compiler generates.
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: H. Peter Anvin <hpa@linux.intel.com>
2015-01-18 16:48:18 +00:00
|
|
|
/* 3DNow or LM implies PREFETCHW */
|
|
|
|
if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH))
|
|
|
|
if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM))
|
|
|
|
set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH);
|
2015-04-26 23:47:59 +00:00
|
|
|
|
2017-04-27 05:01:20 +00:00
|
|
|
/* AMD CPUs don't reset SS attributes on SYSRET, Xen does. */
|
2022-11-04 07:27:01 +00:00
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_XENPV))
|
2017-04-27 05:01:20 +00:00
|
|
|
set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
|
2020-02-19 17:52:43 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Turn on the Instructions Retired free counter on machines not
|
|
|
|
* susceptible to erratum #1054 "Instructions Retired Performance
|
|
|
|
* Counter May Be Inaccurate".
|
|
|
|
*/
|
|
|
|
if (cpu_has(c, X86_FEATURE_IRPERF) &&
|
2023-12-02 11:50:23 +00:00
|
|
|
(boot_cpu_has(X86_FEATURE_ZEN1) && c->x86_model > 0x2f))
|
2020-02-19 17:52:43 +00:00
|
|
|
msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
|
2021-10-21 10:47:44 +00:00
|
|
|
|
|
|
|
check_null_seg_clears_base(c);
|
2023-02-25 00:11:31 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure EFER[AIBRSE - Automatic IBRS Enable] is set. The APs are brought up
|
|
|
|
* using the trampoline code and as part of it, MSR_EFER gets prepared there in
|
|
|
|
* order to be replicated onto them. Regardless, set it here again, if not set,
|
|
|
|
* to protect against any future refactoring/code reorganization which might
|
|
|
|
* miss setting this important bit.
|
|
|
|
*/
|
|
|
|
if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
|
|
|
|
cpu_has(c, X86_FEATURE_AUTOIBRS))
|
|
|
|
WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS));
|
2023-07-15 11:41:28 +00:00
|
|
|
|
x86/barrier: Do not serialize MSR accesses on AMD
AMD does not have the requirement for a synchronization barrier when
acccessing a certain group of MSRs. Do not incur that unnecessary
penalty there.
There will be a CPUID bit which explicitly states that a MFENCE is not
needed. Once that bit is added to the APM, this will be extended with
it.
While at it, move to processor.h to avoid include hell. Untangling that
file properly is a matter for another day.
Some notes on the performance aspect of why this is relevant, courtesy
of Kishon VijayAbraham <Kishon.VijayAbraham@amd.com>:
On a AMD Zen4 system with 96 cores, a modified ipi-bench[1] on a VM
shows x2AVIC IPI rate is 3% to 4% lower than AVIC IPI rate. The
ipi-bench is modified so that the IPIs are sent between two vCPUs in the
same CCX. This also requires to pin the vCPU to a physical core to
prevent any latencies. This simulates the use case of pinning vCPUs to
the thread of a single CCX to avoid interrupt IPI latency.
In order to avoid run-to-run variance (for both x2AVIC and AVIC), the
below configurations are done:
1) Disable Power States in BIOS (to prevent the system from going to
lower power state)
2) Run the system at fixed frequency 2500MHz (to prevent the system
from increasing the frequency when the load is more)
With the above configuration:
*) Performance measured using ipi-bench for AVIC:
Average Latency: 1124.98ns [Time to send IPI from one vCPU to another vCPU]
Cumulative throughput: 42.6759M/s [Total number of IPIs sent in a second from
48 vCPUs simultaneously]
*) Performance measured using ipi-bench for x2AVIC:
Average Latency: 1172.42ns [Time to send IPI from one vCPU to another vCPU]
Cumulative throughput: 40.9432M/s [Total number of IPIs sent in a second from
48 vCPUs simultaneously]
From above, x2AVIC latency is ~4% more than AVIC. However, the expectation is
x2AVIC performance to be better or equivalent to AVIC. Upon analyzing
the perf captures, it is observed significant time is spent in
weak_wrmsr_fence() invoked by x2apic_send_IPI().
With the fix to skip weak_wrmsr_fence()
*) Performance measured using ipi-bench for x2AVIC:
Average Latency: 1117.44ns [Time to send IPI from one vCPU to another vCPU]
Cumulative throughput: 42.9608M/s [Total number of IPIs sent in a second from
48 vCPUs simultaneously]
Comparing the performance of x2AVIC with and without the fix, it can be seen
the performance improves by ~4%.
Performance captured using an unmodified ipi-bench using the 'mesh-ipi' option
with and without weak_wrmsr_fence() on a Zen4 system also showed significant
performance improvement without weak_wrmsr_fence(). The 'mesh-ipi' option ignores
CCX or CCD and just picks random vCPU.
Average throughput (10 iterations) with weak_wrmsr_fence(),
Cumulative throughput: 4933374 IPI/s
Average throughput (10 iterations) without weak_wrmsr_fence(),
Cumulative throughput: 6355156 IPI/s
[1] https://github.com/bytedance/kvm-utils/tree/master/microbenchmark/ipi-bench
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20230622095212.20940-1-bp@alien8.de
2023-10-27 12:24:16 +00:00
|
|
|
/* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
|
|
|
|
clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-09-08 00:58:54 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
/* AMD errata T13 (order #21922) */
|
2018-10-02 22:45:11 +00:00
|
|
|
if (c->x86 == 6) {
|
2009-07-03 23:35:45 +00:00
|
|
|
/* Duron Rev A0 */
|
2018-01-01 01:52:10 +00:00
|
|
|
if (c->x86_model == 3 && c->x86_stepping == 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
size = 64;
|
2009-07-03 23:35:45 +00:00
|
|
|
/* Tbird rev A1/A2 */
|
2005-04-16 22:20:36 +00:00
|
|
|
if (c->x86_model == 4 &&
|
2018-01-01 01:52:10 +00:00
|
|
|
(c->x86_stepping == 0 || c->x86_stepping == 1))
|
2005-04-16 22:20:36 +00:00
|
|
|
size = 256;
|
|
|
|
}
|
|
|
|
return size;
|
|
|
|
}
|
2008-09-08 00:58:54 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
|
2012-08-06 17:00:38 +00:00
|
|
|
{
|
|
|
|
u32 ebx, eax, ecx, edx;
|
|
|
|
u16 mask = 0xfff;
|
|
|
|
|
|
|
|
if (c->x86 < 0xf)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (c->extended_cpuid_level < 0x80000006)
|
|
|
|
return;
|
|
|
|
|
|
|
|
cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
|
|
|
|
|
|
|
|
tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
|
|
|
|
tlb_lli_4k[ENTRIES] = ebx & mask;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
|
|
|
|
* characteristics from the CPUID function 0x80000005 instead.
|
|
|
|
*/
|
|
|
|
if (c->x86 == 0xf) {
|
|
|
|
cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
|
|
|
|
mask = 0xff;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
|
2014-01-15 11:52:15 +00:00
|
|
|
if (!((eax >> 16) & mask))
|
|
|
|
tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
|
|
|
|
else
|
2012-08-06 17:00:38 +00:00
|
|
|
tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
|
|
|
|
|
|
|
|
/* a 4M entry uses two 2M entries */
|
|
|
|
tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
|
|
|
|
|
|
|
|
/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
|
|
|
|
if (!(eax & mask)) {
|
|
|
|
/* Erratum 658 */
|
|
|
|
if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
|
|
|
|
tlb_lli_2m[ENTRIES] = 1024;
|
|
|
|
} else {
|
|
|
|
cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
|
|
|
|
tlb_lli_2m[ENTRIES] = eax & 0xff;
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
tlb_lli_2m[ENTRIES] = eax & mask;
|
|
|
|
|
|
|
|
tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
|
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static const struct cpu_dev amd_cpu_dev = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.c_vendor = "AMD",
|
2008-02-22 22:10:33 +00:00
|
|
|
.c_ident = { "AuthenticAMD" },
|
2008-09-08 00:58:54 +00:00
|
|
|
#ifdef CONFIG_X86_32
|
2013-10-21 08:35:20 +00:00
|
|
|
.legacy_models = {
|
|
|
|
{ .family = 4, .model_names =
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
[3] = "486 DX/2",
|
|
|
|
[7] = "486 DX/2-WB",
|
2008-02-22 22:10:33 +00:00
|
|
|
[8] = "486 DX/4",
|
|
|
|
[9] = "486 DX/4-WB",
|
2005-04-16 22:20:36 +00:00
|
|
|
[14] = "Am5x86-WT",
|
2008-02-22 22:10:33 +00:00
|
|
|
[15] = "Am5x86-WB"
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
},
|
|
|
|
},
|
2013-10-21 08:35:20 +00:00
|
|
|
.legacy_cache_size = amd_size_cache,
|
2008-09-08 00:58:54 +00:00
|
|
|
#endif
|
x86: use ELF section to list CPU vendor specific code
Replace the hardcoded list of initialization functions for each CPU
vendor by a list in an ELF section, which is read at initialization in
arch/x86/kernel/cpu/cpu.c to fill the cpu_devs[] array. The ELF
section, named .x86cpuvendor.init, is reclaimed after boot, and
contains entries of type "struct cpu_vendor_dev" which associates a
vendor number with a pointer to a "struct cpu_dev" structure.
This first modification allows to remove all the VENDOR_init_cpu()
functions.
This patch also removes the hardcoded calls to early_init_amd() and
early_init_intel(). Instead, we add a "c_early_init" member to the
cpu_dev structure, which is then called if not NULL by the generic CPU
initialization code. Unfortunately, in early_cpu_detect(), this_cpu is
not yet set, so we have to use the cpu_devs[] array directly.
This patch is part of the Linux Tiny project, and is needed for
further patch that will allow to disable compilation of unused CPU
support code.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-02-15 11:00:23 +00:00
|
|
|
.c_early_init = early_init_amd,
|
2012-08-06 17:00:38 +00:00
|
|
|
.c_detect_tlb = cpu_detect_tlb_amd,
|
2011-08-05 18:04:09 +00:00
|
|
|
.c_bsp_init = bsp_init_amd,
|
2005-04-16 22:20:36 +00:00
|
|
|
.c_init = init_amd,
|
2008-09-04 19:09:45 +00:00
|
|
|
.c_x86_vendor = X86_VENDOR_AMD,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
2008-09-04 19:09:45 +00:00
|
|
|
cpu_dev_register(amd_cpu_dev);
|
2010-07-28 17:09:30 +00:00
|
|
|
|
2023-01-20 03:10:45 +00:00
|
|
|
static DEFINE_PER_CPU_READ_MOSTLY(unsigned long[4], amd_dr_addr_mask);
|
|
|
|
|
|
|
|
static unsigned int amd_msr_dr_addr_masks[] = {
|
|
|
|
MSR_F16H_DR0_ADDR_MASK,
|
|
|
|
MSR_F16H_DR1_ADDR_MASK,
|
|
|
|
MSR_F16H_DR1_ADDR_MASK + 1,
|
|
|
|
MSR_F16H_DR1_ADDR_MASK + 2
|
|
|
|
};
|
|
|
|
|
|
|
|
void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr)
|
2014-05-29 15:26:50 +00:00
|
|
|
{
|
2023-01-20 03:10:45 +00:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_BPEXT))
|
2014-05-29 15:26:50 +00:00
|
|
|
return;
|
|
|
|
|
2023-01-20 03:10:45 +00:00
|
|
|
if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks)))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask)
|
|
|
|
return;
|
|
|
|
|
|
|
|
wrmsr(amd_msr_dr_addr_masks[dr], mask, 0);
|
|
|
|
per_cpu(amd_dr_addr_mask, cpu)[dr] = mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long amd_get_dr_addr_mask(unsigned int dr)
|
|
|
|
{
|
|
|
|
if (!cpu_feature_enabled(X86_FEATURE_BPEXT))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (WARN_ON_ONCE(dr >= ARRAY_SIZE(amd_msr_dr_addr_masks)))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return per_cpu(amd_dr_addr_mask[dr], smp_processor_id());
|
2014-05-29 15:26:50 +00:00
|
|
|
}
|
2023-01-20 03:10:45 +00:00
|
|
|
EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask);
|
2021-04-25 07:34:51 +00:00
|
|
|
|
|
|
|
u32 amd_get_highest_perf(void)
|
|
|
|
{
|
|
|
|
struct cpuinfo_x86 *c = &boot_cpu_data;
|
|
|
|
|
|
|
|
if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) ||
|
|
|
|
(c->x86_model >= 0x70 && c->x86_model < 0x80)))
|
|
|
|
return 166;
|
|
|
|
|
|
|
|
if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) ||
|
|
|
|
(c->x86_model >= 0x40 && c->x86_model < 0x70)))
|
|
|
|
return 166;
|
|
|
|
|
|
|
|
return 255;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(amd_get_highest_perf);
|
2023-07-15 11:41:28 +00:00
|
|
|
|
|
|
|
static void zenbleed_check_cpu(void *unused)
|
|
|
|
{
|
|
|
|
struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
|
|
|
|
|
2023-11-01 11:38:35 +00:00
|
|
|
zen2_zenbleed_check(c);
|
2023-07-15 11:41:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void amd_check_microcode(void)
|
|
|
|
{
|
2023-12-01 18:37:27 +00:00
|
|
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
|
|
|
|
return;
|
|
|
|
|
2023-07-15 11:41:28 +00:00
|
|
|
on_each_cpu(zenbleed_check_cpu, NULL, 1);
|
|
|
|
}
|
2023-08-07 23:35:44 +00:00
|
|
|
|
2023-08-04 22:06:43 +00:00
|
|
|
/*
|
|
|
|
* Issue a DIV 0/1 insn to clear any division data from previous DIV
|
|
|
|
* operations.
|
|
|
|
*/
|
|
|
|
void noinstr amd_clear_divider(void)
|
|
|
|
{
|
|
|
|
asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)
|
|
|
|
:: "a" (0), "d" (0), "r" (1));
|
|
|
|
}
|
2023-08-11 21:38:24 +00:00
|
|
|
EXPORT_SYMBOL_GPL(amd_clear_divider);
|