mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-28 05:12:49 +00:00
Code Issues Releases Wiki Activity

* Rework apic callbacks, getting rid of unnecessary ones and

Browse source 
coalescing lots of silly duplicates.
  * Use static_calls() instead of indirect calls for apic->foo()
  * Tons of cleanups an crap removal along the way
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEV76QKkVc4xCGURexaDWVMHDJkrAFAmTvfO8ACgkQaDWVMHDJ
 krAP2A//ccii/LuvtTnNEIMMR5w2rwTdHv91ancgFkC8pOeNk37Z8sSLq8tKuLFA
 vgjBIysVIqunuRcNCJ+eqwIIxYfU+UGCWHppzLwO+DY3Q7o9EoTL0BgytdAqxpQQ
 ntEVarqWq25QYXKFoAqbUTJ1UXa42/8HfiXAX/jvP+ACXfilkGPZre6ASxlXeOhm
 XbgPuNQPmXi2WYQH9GCQEsz2Nh80hKap8upK2WbQzzJ3lXsm+xA//4klab0HCYwl
 Uc302uVZozyXRMKbAlwmgasTFOLiV8KKriJ0oHoktBpWgkpdR9uv/RDeSaFR3DAl
 aFmecD4k/Hqezg4yVl+4YpEn2KjxiwARCm4PMW5AV7lpWBPBHAOOai65yJlAi9U6
 bP8pM0+aIx9xg7oWfsTnQ7RkIJ+GZ0w+KZ9LXFM59iu3eV1pAJE3UVyUehe/J1q9
 n8OcH0UeHRlAb8HckqVm1AC7IPvfHw4OAPtUq7z3NFDwbq6i651Tu7f+i2bj31cX
 77Ames+fx6WjxUjyFbJwaK44E7Qez3waztdBfn91qw+m0b+gnKE3ieDNpJTqmm5b
 mKulV7KJwwS6cdqY3+Kr+pIlN+uuGAv7wGzVLcaEAXucDsVn/YAMJHY2+v97xv+n
 J9N+yeaYtmSXVlDsJ6dndMrTQMmcasK1CVXKxs+VYq5Lgf+A68w=
 =eoKm
 -----END PGP SIGNATURE-----

Merge tag 'x86_apic_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 apic updates from Dave Hansen:
 "This includes a very thorough rework of the 'struct apic' handlers.
  Quite a variety of them popped up over the years, especially in the
  32-bit days when odd apics were much more in vogue.

  The end result speaks for itself, which is a removal of a ton of code
  and static calls to replace indirect calls.

  If there's any breakage here, it's likely to be around the 32-bit
  museum pieces that get light to no testing these days.

  Summary:

   - Rework apic callbacks, getting rid of unnecessary ones and
     coalescing lots of silly duplicates.

   - Use static_calls() instead of indirect calls for apic->foo()

   - Tons of cleanups an crap removal along the way"

* tag 'x86_apic_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (64 commits)
  x86/apic: Turn on static calls
  x86/apic: Provide static call infrastructure for APIC callbacks
  x86/apic: Wrap IPI calls into helper functions
  x86/apic: Mark all hotpath APIC callback wrappers __always_inline
  x86/xen/apic: Mark apic __ro_after_init
  x86/apic: Convert other overrides to apic_update_callback()
  x86/apic: Replace acpi_wake_cpu_handler_update() and apic_set_eoi_cb()
  x86/apic: Provide apic_update_callback()
  x86/xen/apic: Use standard apic driver mechanism for Xen PV
  x86/apic: Provide common init infrastructure
  x86/apic: Wrap apic->native_eoi() into a helper
  x86/apic: Nuke ack_APIC_irq()
  x86/apic: Remove pointless arguments from [native_]eoi_write()
  x86/apic/noop: Tidy up the code
  x86/apic: Remove pointless NULL initializations
  x86/apic: Sanitize APIC ID range validation
  x86/apic: Prepare x2APIC for using apic::max_apic_id
  x86/apic: Simplify X2APIC ID validation
  x86/apic: Add max_apic_id member
  x86/apic: Wrap APIC ID validation into an inline
  ...
This commit is contained in:
Linus Torvalds 2023-08-30 10:44:46 -07:00
commit 1687d8aca5
69 changed files with 845 additions and 1424 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
arch/x86/hyperv/hv_apic.c
View file

@ -86,14 +86,14 @@ static void hv_apic_write(u32 reg, u32 val)
}
}
static void hv_apic_eoi_write(u32 reg, u32 val)
static void hv_apic_eoi_write(void)
{
struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()];
if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1))
return;
wrmsr(HV_X64_MSR_EOI, val, 0);
wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0);
}
static bool cpu_is_self(int cpu)
@ -286,12 +286,12 @@ void __init hv_apic_init(void)
*/
orig_apic = *apic;
apic->send_IPI = hv_send_ipi;
apic->send_IPI_mask = hv_send_ipi_mask;
apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;
apic->send_IPI_allbutself = hv_send_ipi_allbutself;
apic->send_IPI_all = hv_send_ipi_all;
apic->send_IPI_self = hv_send_ipi_self;
apic_update_callback(send_IPI, hv_send_ipi);
apic_update_callback(send_IPI_mask, hv_send_ipi_mask);
apic_update_callback(send_IPI_mask_allbutself, hv_send_ipi_mask_allbutself);
apic_update_callback(send_IPI_allbutself, hv_send_ipi_allbutself);
apic_update_callback(send_IPI_all, hv_send_ipi_all);
apic_update_callback(send_IPI_self, hv_send_ipi_self);
}
if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
@ -308,12 +308,12 @@ void __init hv_apic_init(void)
* lazy EOI when available, but the same accessor works for
* both xapic and x2apic because the field layout is the same.
*/
apic_set_eoi_write(hv_apic_eoi_write);
apic_update_callback(eoi, hv_apic_eoi_write);
if (!x2apic_enabled()) {
apic->read = hv_apic_read;
apic->write = hv_apic_write;
apic->icr_write = hv_apic_icr_write;
apic->icr_read = hv_apic_icr_read;
apic_update_callback(read, hv_apic_read);
apic_update_callback(write, hv_apic_write);
apic_update_callback(icr_write, hv_apic_icr_write);
apic_update_callback(icr_read, hv_apic_icr_read);
}
}
}

2
arch/x86/hyperv/hv_init.c
View file

@ -162,7 +162,7 @@ static inline bool hv_reenlightenment_available(void)
DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
{
ack_APIC_irq();
apic_eoi();
inc_irq_stat(irq_hv_reenlightenment_count);
schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
}

2
arch/x86/hyperv/hv_spinlock.c
View file

@ -20,7 +20,7 @@ static bool __initdata hv_pvspin = true;
static void hv_qlock_kick(int cpu)
{
apic->send_IPI(cpu, X86_PLATFORM_IPI_VECTOR);
__apic_send_IPI(cpu, X86_PLATFORM_IPI_VECTOR);
}
static void hv_qlock_wait(u8 *byte, u8 val)

2
arch/x86/hyperv/hv_vtl.c
View file

@ -226,7 +226,7 @@ static int __init hv_vtl_early_init(void)
"Please add 'noxsave' to the kernel command line.\n");
real_mode_header = &hv_vtl_real_mode_header;
apic->wakeup_secondary_cpu_64 = hv_vtl_wakeup_secondary_cpu;
apic_update_callback(wakeup_secondary_cpu_64, hv_vtl_wakeup_secondary_cpu);
return 0;
}

241
arch/x86/include/asm/apic.h
View file

@ -3,6 +3,7 @@
#define _ASM_X86_APIC_H
#include <linux/cpumask.h>
#include <linux/static_call.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
@ -40,11 +41,9 @@
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
extern void generic_apic_probe(void);
extern void x86_32_probe_apic(void);
#else
static inline void generic_apic_probe(void)
{
}
static inline void x86_32_probe_apic(void) { }
#endif
#ifdef CONFIG_X86_LOCAL_APIC
@ -52,7 +51,7 @@ static inline void generic_apic_probe(void)
extern int apic_verbosity;
extern int local_apic_timer_c2_ok;
extern int disable_apic;
extern bool apic_is_disabled;
extern unsigned int lapic_timer_period;
extern int cpuid_to_apicid[];
@ -66,20 +65,6 @@ enum apic_intr_mode_id {
APIC_SYMMETRIC_IO_NO_ROUTING
};
#ifdef CONFIG_SMP
extern void __inquire_remote_apic(int apicid);
#else /* CONFIG_SMP */
static inline void __inquire_remote_apic(int apicid)
{
}
#endif /* CONFIG_SMP */
static inline void default_inquire_remote_apic(int apicid)
{
if (apic_verbosity >= APIC_DEBUG)
__inquire_remote_apic(apicid);
}
/*
* With 82489DX we can't rely on apic feature bit
* retrieved via cpuid but still have to deal with
@ -90,7 +75,7 @@ static inline void default_inquire_remote_apic(int apicid)
*/
static inline bool apic_from_smp_config(void)
{
return smp_found_config && !disable_apic;
return smp_found_config && !apic_is_disabled;
}
/*
@ -114,8 +99,11 @@ static inline u32 native_apic_mem_read(u32 reg)
return *((volatile u32 *)(APIC_BASE + reg));
}
extern void native_apic_wait_icr_idle(void);
extern u32 native_safe_apic_wait_icr_idle(void);
static inline void native_apic_mem_eoi(void)
{
native_apic_mem_write(APIC_EOI, APIC_EOI_ACK);
}
extern void native_apic_icr_write(u32 low, u32 id);
extern u64 native_apic_icr_read(void);
@ -149,12 +137,12 @@ extern void setup_secondary_APIC_clock(void);
extern void lapic_update_tsc_freq(void);
#ifdef CONFIG_X86_64
static inline int apic_force_enable(unsigned long addr)
static inline bool apic_force_enable(unsigned long addr)
{
return -1;
return false;
}
#else
extern int apic_force_enable(unsigned long addr);
extern bool apic_force_enable(unsigned long addr);
#endif
extern void apic_ap_setup(void);
@ -207,7 +195,7 @@ static inline void native_apic_msr_write(u32 reg, u32 v)
wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0);
}
static inline void native_apic_msr_eoi_write(u32 reg, u32 v)
static inline void native_apic_msr_eoi(void)
{
__wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0);
}
@ -223,18 +211,6 @@ static inline u32 native_apic_msr_read(u32 reg)
return (u32)msr;
}
static inline void native_x2apic_wait_icr_idle(void)
{
/* no need to wait for icr idle in x2apic */
return;
}
static inline u32 native_safe_x2apic_wait_icr_idle(void)
{
/* no need to wait for icr idle in x2apic */
return 0;
}
static inline void native_x2apic_icr_write(u32 low, u32 id)
{
wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
@ -261,7 +237,7 @@ static inline int x2apic_enabled(void)
#else /* !CONFIG_X86_X2APIC */
static inline void x2apic_setup(void) { }
static inline int x2apic_enabled(void) { return 0; }
static inline u32 native_apic_msr_read(u32 reg) { BUG(); }
#define x2apic_mode (0)
#define x2apic_supported() (0)
#endif /* !CONFIG_X86_X2APIC */
@ -280,8 +256,8 @@ struct irq_data;
*/
struct apic {
/* Hotpath functions first */
void (*eoi_write)(u32 reg, u32 v);
void (*native_eoi_write)(u32 reg, u32 v);
void (*eoi)(void);
void (*native_eoi)(void);
void (*write)(u32 reg, u32 v);
u32 (*read)(u32 reg);
@ -296,10 +272,11 @@ struct apic {
void (*send_IPI_all)(int vector);
void (*send_IPI_self)(int vector);
u32 disable_esr;
enum apic_delivery_modes delivery_mode;
bool dest_mode_logical;
u32 disable_esr : 1,
dest_mode_logical : 1,
x2apic_set_max_apicid : 1;
u32 (*calc_dest_apicid)(unsigned int cpu);
@ -307,19 +284,18 @@ struct apic {
u64 (*icr_read)(void);
void (*icr_write)(u32 low, u32 high);
/* The limit of the APIC ID space. */
u32 max_apic_id;
/* Probe, setup and smpboot functions */
int (*probe)(void);
int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id);
int (*apic_id_valid)(u32 apicid);
int (*apic_id_registered)(void);
bool (*apic_id_registered)(void);
bool (*check_apicid_used)(physid_mask_t *map, int apicid);
void (*init_apic_ldr)(void);
void (*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap);
void (*setup_apic_routing)(void);
int (*cpu_present_to_apicid)(int mps_cpu);
void (*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap);
int (*check_phys_apicid_present)(int phys_apicid);
int (*phys_pkg_id)(int cpuid_apic, int index_msb);
u32 (*get_apic_id)(unsigned long x);
@ -330,24 +306,26 @@ struct apic {
/* wakeup secondary CPU using 64-bit wakeup point */
int (*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip);
void (*inquire_remote_apic)(int apicid);
#ifdef CONFIG_X86_32
/*
* Called very early during boot from get_smp_config(). It should
* return the logical apicid. x86_[bios]_cpu_to_apicid is
* initialized before this function is called.
*
* If logical apicid can't be determined that early, the function
* may return BAD_APICID. Logical apicid will be configured after
* init_apic_ldr() while bringing up CPUs. Note that NUMA affinity
* won't be applied properly during early boot in this case.
*/
int (*x86_32_early_logical_apicid)(int cpu);
#endif
char *name;
};
struct apic_override {
void (*eoi)(void);
void (*native_eoi)(void);
void (*write)(u32 reg, u32 v);
u32 (*read)(u32 reg);
void (*send_IPI)(int cpu, int vector);
void (*send_IPI_mask)(const struct cpumask *mask, int vector);
void (*send_IPI_mask_allbutself)(const struct cpumask *msk, int vec);
void (*send_IPI_allbutself)(int vector);
void (*send_IPI_all)(int vector);
void (*send_IPI_self)(int vector);
u64 (*icr_read)(void);
void (*icr_write)(u32 low, u32 high);
int (*wakeup_secondary_cpu)(int apicid, unsigned long start_eip);
int (*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip);
};
/*
* Pointer to the local APIC driver in use on this system (there's
* always just one such driver in use - the kernel decides via an
@ -383,43 +361,111 @@ extern int lapic_can_unplug_cpu(void);
#endif
#ifdef CONFIG_X86_LOCAL_APIC
extern struct apic_override __x86_apic_override;
static inline u32 apic_read(u32 reg)
{
return apic->read(reg);
void __init apic_setup_apic_calls(void);
void __init apic_install_driver(struct apic *driver);
#define apic_update_callback(_callback, _fn) { \
__x86_apic_override._callback = _fn; \
apic->_callback = _fn; \
static_call_update(apic_call_##_callback, _fn); \
pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn); \
}
static inline void apic_write(u32 reg, u32 val)
#define DECLARE_APIC_CALL(__cb) \
DECLARE_STATIC_CALL(apic_call_##__cb, *apic->__cb)
DECLARE_APIC_CALL(eoi);
DECLARE_APIC_CALL(native_eoi);
DECLARE_APIC_CALL(icr_read);
DECLARE_APIC_CALL(icr_write);
DECLARE_APIC_CALL(read);
DECLARE_APIC_CALL(send_IPI);
DECLARE_APIC_CALL(send_IPI_mask);
DECLARE_APIC_CALL(send_IPI_mask_allbutself);
DECLARE_APIC_CALL(send_IPI_allbutself);
DECLARE_APIC_CALL(send_IPI_all);
DECLARE_APIC_CALL(send_IPI_self);
DECLARE_APIC_CALL(wait_icr_idle);
DECLARE_APIC_CALL(wakeup_secondary_cpu);
DECLARE_APIC_CALL(wakeup_secondary_cpu_64);
DECLARE_APIC_CALL(write);
static __always_inline u32 apic_read(u32 reg)
{
apic->write(reg, val);
return static_call(apic_call_read)(reg);
}
static inline void apic_eoi(void)
static __always_inline void apic_write(u32 reg, u32 val)
{
apic->eoi_write(APIC_EOI, APIC_EOI_ACK);
static_call(apic_call_write)(reg, val);
}
static inline u64 apic_icr_read(void)
static __always_inline void apic_eoi(void)
{
return apic->icr_read();
static_call(apic_call_eoi)();
}
static inline void apic_icr_write(u32 low, u32 high)
static __always_inline void apic_native_eoi(void)
{
apic->icr_write(low, high);
static_call(apic_call_native_eoi)();
}
static inline void apic_wait_icr_idle(void)
static __always_inline u64 apic_icr_read(void)
{
apic->wait_icr_idle();
return static_call(apic_call_icr_read)();
}
static inline u32 safe_apic_wait_icr_idle(void)
static __always_inline void apic_icr_write(u32 low, u32 high)
{
return apic->safe_wait_icr_idle();
static_call(apic_call_icr_write)(low, high);
}
extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v));
static __always_inline void __apic_send_IPI(int cpu, int vector)
{
static_call(apic_call_send_IPI)(cpu, vector);
}
static __always_inline void __apic_send_IPI_mask(const struct cpumask *mask, int vector)
{
static_call_mod(apic_call_send_IPI_mask)(mask, vector);
}
static __always_inline void __apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
{
static_call(apic_call_send_IPI_mask_allbutself)(mask, vector);
}
static __always_inline void __apic_send_IPI_allbutself(int vector)
{
static_call(apic_call_send_IPI_allbutself)(vector);
}
static __always_inline void __apic_send_IPI_all(int vector)
{
static_call(apic_call_send_IPI_all)(vector);
}
static __always_inline void __apic_send_IPI_self(int vector)
{
static_call_mod(apic_call_send_IPI_self)(vector);
}
static __always_inline void apic_wait_icr_idle(void)
{
static_call_cond(apic_call_wait_icr_idle)();
}
static __always_inline u32 safe_apic_wait_icr_idle(void)
{
return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0;
}
static __always_inline bool apic_id_valid(u32 apic_id)
{
return apic_id <= apic->max_apic_id;
}
#else /* CONFIG_X86_LOCAL_APIC */
@ -430,22 +476,16 @@ static inline u64 apic_icr_read(void) { return 0; }
static inline void apic_icr_write(u32 low, u32 high) { }
static inline void apic_wait_icr_idle(void) { }
static inline u32 safe_apic_wait_icr_idle(void) { return 0; }
static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {}
static inline void apic_set_eoi_cb(void (*eoi)(void)) {}
static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); }
static inline void apic_setup_apic_calls(void) { }
#define apic_update_callback(_callback, _fn) do { } while (0)
#endif /* CONFIG_X86_LOCAL_APIC */
extern void apic_ack_irq(struct irq_data *data);
static inline void ack_APIC_irq(void)
{
/*
* ack_APIC_irq() actually gets compiled as a single instruction
* ... yummie.
*/
apic_eoi();
}
static inline bool lapic_vector_set_in_irr(unsigned int vector)
{
u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
@ -475,10 +515,6 @@ extern void generic_bigsmp_probe(void);
#include <asm/smp.h>
#define APIC_DFR_VALUE (APIC_DFR_FLAT)
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
extern struct apic apic_noop;
static inline unsigned int read_apic_id(void)
@ -490,12 +526,14 @@ static inline unsigned int read_apic_id(void)
#ifdef CONFIG_X86_64
typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip);
extern void acpi_wake_cpu_handler_update(wakeup_cpu_handler handler);
extern int default_acpi_madt_oem_check(char *, char *);
extern void x86_64_probe_apic(void);
#else
static inline int default_acpi_madt_oem_check(char *a, char *b) { return 0; }
static inline void x86_64_probe_apic(void) { }
#endif
extern int default_apic_id_valid(u32 apicid);
extern int default_acpi_madt_oem_check(char *, char *);
extern void default_setup_apic_routing(void);
extern u32 apic_default_calc_apicid(unsigned int cpu);
extern u32 apic_flat_calc_apicid(unsigned int cpu);
@ -503,9 +541,12 @@ extern u32 apic_flat_calc_apicid(unsigned int cpu);
extern bool default_check_apicid_used(physid_mask_t *map, int apicid);
extern void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap);
extern int default_cpu_present_to_apicid(int mps_cpu);
extern int default_check_phys_apicid_present(int phys_apicid);
#endif /* CONFIG_X86_LOCAL_APIC */
#else /* CONFIG_X86_LOCAL_APIC */
static inline unsigned int read_apic_id(void) { return 0; }
#endif /* !CONFIG_X86_LOCAL_APIC */
#ifdef CONFIG_SMP
void apic_smt_update(void);

4
arch/x86/include/asm/hw_irq.h
View file

@ -97,10 +97,10 @@ extern struct irq_cfg *irqd_cfg(struct irq_data *irq_data);
extern void lock_vector_lock(void);
extern void unlock_vector_lock(void);
#ifdef CONFIG_SMP
extern void send_cleanup_vector(struct irq_cfg *);
extern void vector_schedule_cleanup(struct irq_cfg *);
extern void irq_complete_move(struct irq_cfg *cfg);
#else
static inline void send_cleanup_vector(struct irq_cfg *c) { }
static inline void vector_schedule_cleanup(struct irq_cfg *c) { }
static inline void irq_complete_move(struct irq_cfg *c) { }
#endif

1
arch/x86/include/asm/idtentry.h
View file

@ -648,7 +648,6 @@ DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi);
#ifdef CONFIG_SMP
DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi);
DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR, sysvec_irq_move_cleanup);
DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot);
DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single);
DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function);

7
arch/x86/include/asm/io_apic.h
View file

@ -109,8 +109,8 @@ extern int mp_irq_entries;
/* MP IRQ source entries */
extern struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
/* 1 if "noapic" boot option passed */
extern int skip_ioapic_setup;
/* True if "noapic" boot option passed */
extern bool ioapic_is_disabled;
/* 1 if "noapic" boot option passed */
extern int noioapicquirk;
@ -129,7 +129,7 @@ extern unsigned long io_apic_irqs;
* assignment of PCI IRQ's.
*/
#define io_apic_assign_pci_irqs \
(mp_irq_entries && !skip_ioapic_setup && io_apic_irqs)
(mp_irq_entries && !ioapic_is_disabled && io_apic_irqs)
struct irq_cfg;
extern void ioapic_insert_resources(void);
@ -179,6 +179,7 @@ extern void print_IO_APICs(void);
#define IO_APIC_IRQ(x) 0
#define io_apic_assign_pci_irqs 0
#define setup_ioapic_ids_from_mpc x86_init_noop
#define nr_ioapics (0)
static inline void ioapic_insert_resources(void) { }
static inline int arch_early_ioapic_init(void) { return 0; }
static inline void print_IO_APICs(void) {}

7
arch/x86/include/asm/irq_vectors.h
View file

@ -35,13 +35,6 @@
*/
#define FIRST_EXTERNAL_VECTOR 0x20
/*
* Reserve the lowest usable vector (and hence lowest priority) 0x20 for
* triggering cleanup after irq migration. 0x21-0x2f will still be used
* for device interrupts.
*/
#define IRQ_MOVE_CLEANUP_VECTOR FIRST_EXTERNAL_VECTOR
#define IA32_SYSCALL_VECTOR 0x80
/*

31
arch/x86/include/asm/mpspec.h
View file

@ -23,8 +23,6 @@ extern int pic_mode;
#define MAX_IRQ_SOURCES 256
extern unsigned int def_to_bigsmp;
#else /* CONFIG_X86_64: */
#define MAX_MP_BUSSES 256
@ -41,7 +39,6 @@ extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
extern unsigned int boot_cpu_physical_apicid;
extern u8 boot_cpu_apic_version;
extern unsigned long mp_lapic_addr;
#ifdef CONFIG_X86_LOCAL_APIC
extern int smp_found_config;
@ -76,7 +73,7 @@ static inline void e820__memblock_alloc_reserved_mpc_new(void) { }
#define default_get_smp_config x86_init_uint_noop
#endif
int generic_processor_info(int apicid, int version);
int generic_processor_info(int apicid);
#define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_LOCAL_APIC)
@ -87,13 +84,7 @@ struct physid_mask {
typedef struct physid_mask physid_mask_t;
#define physid_set(physid, map) set_bit(physid, (map).mask)
#define physid_clear(physid, map) clear_bit(physid, (map).mask)
#define physid_isset(physid, map) test_bit(physid, (map).mask)
#define physid_test_and_set(physid, map) \
test_and_set_bit(physid, (map).mask)
#define physids_and(dst, src1, src2) \
bitmap_and((dst).mask, (src1).mask, (src2).mask, MAX_LOCAL_APIC)
#define physids_or(dst, src1, src2) \
bitmap_or((dst).mask, (src1).mask, (src2).mask, MAX_LOCAL_APIC)
@ -101,29 +92,9 @@ typedef struct physid_mask physid_mask_t;
#define physids_clear(map) \
bitmap_zero((map).mask, MAX_LOCAL_APIC)
#define physids_complement(dst, src) \
bitmap_complement((dst).mask, (src).mask, MAX_LOCAL_APIC)
#define physids_empty(map) \
bitmap_empty((map).mask, MAX_LOCAL_APIC)
#define physids_equal(map1, map2) \
bitmap_equal((map1).mask, (map2).mask, MAX_LOCAL_APIC)
#define physids_weight(map) \
bitmap_weight((map).mask, MAX_LOCAL_APIC)
#define physids_shift_right(d, s, n) \
bitmap_shift_right((d).mask, (s).mask, n, MAX_LOCAL_APIC)
#define physids_shift_left(d, s, n) \
bitmap_shift_left((d).mask, (s).mask, n, MAX_LOCAL_APIC)
static inline unsigned long physids_coerce(physid_mask_t *map)
{
return map->mask[0];
}
static inline void physids_promote(unsigned long physids, physid_mask_t *map)
{
physids_clear(*map);

1
arch/x86/include/asm/processor.h
View file

@ -190,7 +190,6 @@ static inline unsigned long long l1tf_pfn_limit(void)
}
extern void early_cpu_init(void);
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
extern void print_cpu_info(struct cpuinfo_x86 *);
void print_cpu_msr(struct cpuinfo_x86 *);

11
arch/x86/include/asm/smp.h
View file

@ -22,10 +22,6 @@ DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id);
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid);
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid);
#endif
struct task_struct;
@ -183,13 +179,6 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu)
extern unsigned disabled_cpus;
#ifdef CONFIG_X86_LOCAL_APIC
extern int hard_smp_processor_id(void);
#else /* CONFIG_X86_LOCAL_APIC */
#define hard_smp_processor_id() 0
#endif /* CONFIG_X86_LOCAL_APIC */
#ifdef CONFIG_DEBUG_NMI_SELFTEST
extern void nmi_selftest(void);
#else

12
arch/x86/kernel/acpi/boot.c
View file

@ -170,7 +170,6 @@ static int __init acpi_parse_madt(struct acpi_table_header *table)
*/
static int acpi_register_lapic(int id, u32 acpiid, u8 enabled)
{
unsigned int ver = 0;
int cpu;
if (id >= MAX_LOCAL_APIC) {
@ -183,10 +182,7 @@ static int acpi_register_lapic(int id, u32 acpiid, u8 enabled)
return -EINVAL;
}
if (boot_cpu_physical_apicid != -1U)
ver = boot_cpu_apic_version;
cpu = generic_processor_info(id, ver);
cpu = generic_processor_info(id);
if (cpu >= 0)
early_per_cpu(x86_cpu_to_acpiid, cpu) = acpiid;
@ -240,7 +236,7 @@ acpi_parse_x2apic(union acpi_subtable_headers *header, const unsigned long end)
* to not preallocating memory for all NR_CPUS
* when we use CPU hotplug.
*/
if (!apic->apic_id_valid(apic_id)) {
if (!apic_id_valid(apic_id)) {
if (enabled)
pr_warn("x2apic entry ignored\n");
return 0;
@ -1182,7 +1178,7 @@ static int __init acpi_parse_mp_wake(union acpi_subtable_headers *header,
acpi_mp_wake_mailbox_paddr = mp_wake->base_address;
acpi_wake_cpu_handler_update(acpi_wakeup_cpu);
apic_update_callback(wakeup_secondary_cpu_64, acpi_wakeup_cpu);
return 0;
}
@ -1279,7 +1275,7 @@ static int __init acpi_parse_madt_ioapic_entries(void)
/*
* if "noapic" boot option, don't look for IO-APICs
*/
if (skip_ioapic_setup) {
if (ioapic_is_disabled) {
pr_info("Skipping IOAPIC probe due to 'noapic' option.\n");
return -ENODEV;
}

2
arch/x86/kernel/apic/Makefile
View file

@ -7,7 +7,7 @@
# In particualr, smp_apic_timer_interrupt() is called in random places.
KCOV_INSTRUMENT := n
obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_common.o apic_noop.o ipi.o vector.o
obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_common.o apic_noop.o ipi.o vector.o init.o
obj-y += hw_nmi.o
obj-$(CONFIG_X86_IO_APIC) += io_apic.o

456
arch/x86/kernel/apic/apic.c
View file

@ -63,6 +63,8 @@
#include <asm/irq_regs.h>
#include <asm/cpu.h>
#include "local.h"
unsigned int num_processors;
unsigned disabled_cpus;
@ -73,11 +75,6 @@ EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
u8 boot_cpu_apic_version __ro_after_init;
/*
* The highest APIC ID seen during enumeration.
*/
static unsigned int max_physical_apicid;
/*
* Bitmask of physically existing CPUs:
*/
@ -104,26 +101,20 @@ static bool virt_ext_dest_id __ro_after_init;
/* For parallel bootup. */
unsigned long apic_mmio_base __ro_after_init;
static inline bool apic_accessible(void)
{
return x2apic_mode || apic_mmio_base;
}
/*
* Map cpu index to physical APIC ID
*/
DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid);
#ifdef CONFIG_X86_32
/*
* On x86_32, the mapping between cpu and logical apicid may vary
* depending on apic in use. The following early percpu variable is
* used for the mapping. This is where the behaviors of x86_64 and 32
* actually diverge. Let's keep it ugly for now.
*/
DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
/* Local APIC was disabled by the BIOS and enabled by the kernel */
static int enabled_via_apicbase __ro_after_init;
@ -179,8 +170,8 @@ static __init int setup_apicpmtimer(char *s)
__setup("apicpmtimer", setup_apicpmtimer);
#endif
unsigned long mp_lapic_addr __ro_after_init;
int disable_apic __ro_after_init;
static unsigned long mp_lapic_addr __ro_after_init;
bool apic_is_disabled __ro_after_init;
/* Disable local APIC timer from the kernel commandline or via dmi quirk */
static int disable_apic_timer __initdata;
/* Local APIC timer works in C2 */
@ -206,8 +197,6 @@ unsigned int lapic_timer_period = 0;
static void apic_pm_activate(void);
static unsigned long apic_phys __ro_after_init;
/*
* Get the LAPIC version
*/
@ -247,31 +236,7 @@ static int modern_apic(void)
*/
static void __init apic_disable(void)
{
pr_info("APIC: switched to apic NOOP\n");
apic = &apic_noop;
}
void native_apic_wait_icr_idle(void)
{
while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
cpu_relax();
}
u32 native_safe_apic_wait_icr_idle(void)
{
u32 send_status;
int timeout;
timeout = 0;
do {
send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
if (!send_status)
break;
inc_irq_stat(icr_read_retry_count);
udelay(100);
} while (timeout++ < 1000);
return send_status;
apic_install_driver(&apic_noop);
}
void native_apic_icr_write(u32 low, u32 id)
@ -537,7 +502,7 @@ static int lapic_timer_set_oneshot(struct clock_event_device *evt)
static void lapic_timer_broadcast(const struct cpumask *mask)
{
#ifdef CONFIG_SMP
apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
__apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
#endif
}
@ -810,7 +775,7 @@ bool __init apic_needs_pit(void)
return true;
/* Is there an APIC at all or is it disabled? */
if (!boot_cpu_has(X86_FEATURE_APIC) || disable_apic)
if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
return true;
/*
@ -1110,7 +1075,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
{
struct pt_regs *old_regs = set_irq_regs(regs);
ack_APIC_irq();
apic_eoi();
trace_local_timer_entry(LOCAL_TIMER_VECTOR);
local_apic_timer_interrupt();
trace_local_timer_exit(LOCAL_TIMER_VECTOR);
@ -1134,8 +1099,7 @@ void clear_local_APIC(void)
int maxlvt;
u32 v;
/* APIC hasn't been mapped yet */
if (!x2apic_mode && !apic_phys)
if (!apic_accessible())
return;
maxlvt = lapic_get_maxlvt();
@ -1225,8 +1189,7 @@ void apic_soft_disable(void)
*/
void disable_local_APIC(void)
{
/* APIC hasn't been mapped yet */
if (!x2apic_mode && !apic_phys)
if (!apic_accessible())
return;
apic_soft_disable();
@ -1299,7 +1262,7 @@ enum apic_intr_mode_id apic_intr_mode __ro_after_init;
static int __init __apic_intr_mode_select(void)
{
/* Check kernel option */
if (disable_apic) {
if (apic_is_disabled) {
pr_info("APIC disabled via kernel command line\n");
return APIC_PIC;
}
@ -1308,7 +1271,7 @@ static int __init __apic_intr_mode_select(void)
#ifdef CONFIG_X86_64
/* On 64-bit, the APIC must be integrated, Check local APIC only */
if (!boot_cpu_has(X86_FEATURE_APIC)) {
disable_apic = 1;
apic_is_disabled = true;
pr_info("APIC disabled by BIOS\n");
return APIC_PIC;
}
@ -1317,16 +1280,15 @@ static int __init __apic_intr_mode_select(void)
/* Neither 82489DX nor integrated APIC ? */
if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
disable_apic = 1;
apic_is_disabled = true;
return APIC_PIC;
}
/* If the BIOS pretends there is an integrated APIC ? */
if (!boot_cpu_has(X86_FEATURE_APIC) &&
APIC_INTEGRATED(boot_cpu_apic_version)) {
disable_apic = 1;
pr_err(FW_BUG "Local APIC %d not detected, force emulation\n",
boot_cpu_physical_apicid);
apic_is_disabled = true;
pr_err(FW_BUG "Local APIC not detected, force emulation\n");
return APIC_PIC;
}
#endif
@ -1347,12 +1309,6 @@ static int __init __apic_intr_mode_select(void)
pr_info("APIC: SMP mode deactivated\n");
return APIC_SYMMETRIC_IO_NO_ROUTING;
}
if (read_apic_id() != boot_cpu_physical_apicid) {
panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
read_apic_id(), boot_cpu_physical_apicid);
/* Or can we switch back to PIC here? */
}
#endif
return APIC_SYMMETRIC_IO;
@ -1439,7 +1395,9 @@ void __init apic_intr_mode_init(void)
break;
}
default_setup_apic_routing();
x86_64_probe_apic();
x86_32_install_bigsmp();
if (x86_platform.apic_post_init)
x86_platform.apic_post_init();
@ -1521,7 +1479,7 @@ static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
* per set bit.
*/
for_each_set_bit(bit, isr->map, APIC_IR_BITS)
ack_APIC_irq();
apic_eoi();
return true;
}
@ -1533,7 +1491,7 @@ static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
* interrupt from previous kernel might still have ISR bit set.
*
* Most probably by now the CPU has serviced that pending interrupt and it
* might not have done the ack_APIC_irq() because it thought, interrupt
* might not have done the apic_eoi() because it thought, interrupt
* came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
* the ISR bit and cpu thinks it has already serviced the interrupt. Hence
* a vector might get locked. It was noticed for timer irq (vector
@ -1567,7 +1525,7 @@ static void setup_local_APIC(void)
int cpu = smp_processor_id();
unsigned int value;
if (disable_apic) {
if (apic_is_disabled) {
disable_ioapic_support();
return;
}
@ -1589,36 +1547,18 @@ static void setup_local_APIC(void)
apic_write(APIC_ESR, 0);
}
#endif
/*
* Double-check whether this APIC is really registered.
* This is meaningless in clustered apic mode, so we skip it.
*/
BUG_ON(!apic->apic_id_registered());
/* Validate that the APIC is registered if required */
BUG_ON(apic->apic_id_registered && !apic->apic_id_registered());
/*
* Intel recommends to set DFR, LDR and TPR before enabling
* an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
* document number 292116). So here it goes...
* document number 292116).
*
* Except for APICs which operate in physical destination mode.
*/
apic->init_apic_ldr();
#ifdef CONFIG_X86_32
if (apic->dest_mode_logical) {
int logical_apicid, ldr_apicid;
/*
* APIC LDR is initialized. If logical_apicid mapping was
* initialized during get_smp_config(), make sure it matches
* the actual value.
*/
logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
if (logical_apicid != BAD_APICID)
WARN_ON(logical_apicid != ldr_apicid);
/* Always use the value from LDR. */
early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid;
}
#endif
if (apic->init_apic_ldr)
apic->init_apic_ldr();
/*
* Set Task Priority to 'accept all except vectors 0-31'. An APIC
@ -1691,7 +1631,7 @@ static void setup_local_APIC(void)
* TODO: set up through-local-APIC from through-I/O-APIC? --macro
*/
value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
if (!cpu && (pic_mode || !value || skip_ioapic_setup)) {
if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
value = APIC_DM_EXTINT;
apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
} else {
@ -1748,6 +1688,25 @@ void apic_ap_setup(void)
end_local_APIC_setup();
}
static __init void cpu_set_boot_apic(void);
static __init void apic_read_boot_cpu_id(bool x2apic)
{
/*
* This can be invoked from check_x2apic() before the APIC has been
* selected. But that code knows for sure that the BIOS enabled
* X2APIC.
*/
if (x2apic) {
boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
} else {
boot_cpu_physical_apicid = read_apic_id();
boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
}
cpu_set_boot_apic();
}
#ifdef CONFIG_X86_X2APIC
int x2apic_mode;
EXPORT_SYMBOL_GPL(x2apic_mode);
@ -1847,6 +1806,8 @@ void x2apic_setup(void)
__x2apic_enable();
}
static __init void apic_set_fixmap(void);
static __init void x2apic_disable(void)
{
u32 x2apic_id, state = x2apic_state;
@ -1867,7 +1828,7 @@ static __init void x2apic_disable(void)
}
__x2apic_disable();
register_lapic_address(mp_lapic_addr);
apic_set_fixmap();
}
static __init void x2apic_enable(void)
@ -1928,6 +1889,7 @@ void __init check_x2apic(void)
x2apic_state = X2APIC_ON_LOCKED;
else
x2apic_state = X2APIC_ON;
apic_read_boot_cpu_id(true);
} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
x2apic_state = X2APIC_DISABLED;
}
@ -1943,7 +1905,7 @@ void __init check_x2apic(void)
pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
pr_err("Disabling APIC, expect reduced performance and functionality.\n");
disable_apic = 1;
apic_is_disabled = true;
setup_clear_cpu_cap(X86_FEATURE_APIC);
}
@ -1956,7 +1918,7 @@ void __init enable_IR_x2apic(void)
unsigned long flags;
int ret, ir_stat;
if (skip_ioapic_setup) {
if (ioapic_is_disabled) {
pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
return;
}
@ -1994,19 +1956,19 @@ void __init enable_IR_x2apic(void)
* On AMD64 we trust the BIOS - if it says no APIC it is likely
* not correctly set up (usually the APIC timer won't work etc.)
*/
static int __init detect_init_APIC(void)
static bool __init detect_init_APIC(void)
{
if (!boot_cpu_has(X86_FEATURE_APIC)) {
pr_info("No local APIC present\n");
return -1;
return false;
}
mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
return 0;
register_lapic_address(APIC_DEFAULT_PHYS_BASE);
return true;
}
#else
static int __init apic_verify(void)
static bool __init apic_verify(unsigned long addr)
{
u32 features, h, l;
@ -2017,28 +1979,28 @@ static int __init apic_verify(void)
features = cpuid_edx(1);
if (!(features & (1 << X86_FEATURE_APIC))) {
pr_warn("Could not enable APIC!\n");
return -1;
return false;
}
set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
/* The BIOS may have set up the APIC at some other address */
if (boot_cpu_data.x86 >= 6) {
rdmsr(MSR_IA32_APICBASE, l, h);
if (l & MSR_IA32_APICBASE_ENABLE)
mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
addr = l & MSR_IA32_APICBASE_BASE;
}
register_lapic_address(addr);
pr_info("Found and enabled local APIC!\n");
return 0;
return true;
}
int __init apic_force_enable(unsigned long addr)
bool __init apic_force_enable(unsigned long addr)
{
u32 h, l;
if (disable_apic)
return -1;
if (apic_is_disabled)
return false;
/*
* Some BIOSes disable the local APIC in the APIC_BASE
@ -2055,17 +2017,17 @@ int __init apic_force_enable(unsigned long addr)
enabled_via_apicbase = 1;
}
}
return apic_verify();
return apic_verify(addr);
}
/*
* Detect and initialize APIC
*/
static int __init detect_init_APIC(void)
static bool __init detect_init_APIC(void)
{
/* Disabled by kernel option? */
if (disable_apic)
return -1;
if (apic_is_disabled)
return false;
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_AMD:
@ -2092,22 +2054,22 @@ static int __init detect_init_APIC(void)
if (!force_enable_local_apic) {
pr_info("Local APIC disabled by BIOS -- "
"you can enable it with \"lapic\"\n");
return -1;
return false;
}
if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
return -1;
if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
return false;
} else {
if (apic_verify())
return -1;
if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
return false;
}
apic_pm_activate();
return 0;
return true;
no_apic:
pr_info("No local APIC present or hardware disabled\n");
return -1;
return false;
}
#endif
@ -2116,64 +2078,38 @@ static int __init detect_init_APIC(void)
*/
void __init init_apic_mappings(void)
{
unsigned int new_apicid;
if (apic_validate_deadline_timer())
pr_info("TSC deadline timer available\n");
if (x2apic_mode) {
boot_cpu_physical_apicid = read_apic_id();
if (x2apic_mode)
return;
}
/* If no local APIC can be found return early */
if (!smp_found_config && detect_init_APIC()) {
/* lets NOP'ify apic operations */
pr_info("APIC: disable apic facility\n");
apic_disable();
} else {
apic_phys = mp_lapic_addr;
/*
* If the system has ACPI MADT tables or MP info, the LAPIC
* address is already registered.
*/
if (!acpi_lapic && !smp_found_config)
register_lapic_address(apic_phys);
if (!smp_found_config) {
if (!detect_init_APIC()) {
pr_info("APIC: disable apic facility\n");
apic_disable();
}
num_processors = 1;
}
}
/*
* Fetch the APIC ID of the BSP in case we have a
* default configuration (or the MP table is broken).
*/
new_apicid = read_apic_id();
if (boot_cpu_physical_apicid != new_apicid) {
boot_cpu_physical_apicid = new_apicid;
/*
* yeah -- we lie about apic_version
* in case if apic was disabled via boot option
* but it's not a problem for SMP compiled kernel
* since apic_intr_mode_select is prepared for such
* a case and disable smp mode
*/
boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
}
static __init void apic_set_fixmap(void)
{
set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
apic_mmio_base = APIC_BASE;
apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
apic_mmio_base, mp_lapic_addr);
apic_read_boot_cpu_id(false);
}
void __init register_lapic_address(unsigned long address)
{
/* This should only happen once */
WARN_ON_ONCE(mp_lapic_addr);
mp_lapic_addr = address;
if (!x2apic_mode) {
set_fixmap_nocache(FIX_APIC_BASE, address);
apic_mmio_base = APIC_BASE;
apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
APIC_BASE, address);
}
if (boot_cpu_physical_apicid == -1U) {
boot_cpu_physical_apicid = read_apic_id();
boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
}
if (!x2apic_mode)
apic_set_fixmap();
}
/*
@ -2210,7 +2146,7 @@ static noinline void handle_spurious_interrupt(u8 vector)
if (v & (1 << (vector & 0x1f))) {
pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
vector, smp_processor_id());
ack_APIC_irq();
apic_eoi();
} else {
pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
vector, smp_processor_id());
@ -2261,7 +2197,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
apic_write(APIC_ESR, 0);
v = apic_read(APIC_ESR);
ack_APIC_irq();
apic_eoi();
atomic_inc(&irq_err_count);
apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
@ -2446,54 +2382,43 @@ static int allocate_logical_cpuid(int apicid)
return nr_logical_cpuids++;
}
int generic_processor_info(int apicid, int version)
static void cpu_update_apic(int cpu, int apicid)
{
#if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
#endif
set_cpu_possible(cpu, true);
physid_set(apicid, phys_cpu_present_map);
set_cpu_present(cpu, true);
num_processors++;
if (system_state != SYSTEM_BOOTING)
cpu_mark_primary_thread(cpu, apicid);
}
static __init void cpu_set_boot_apic(void)
{
cpuid_to_apicid[0] = boot_cpu_physical_apicid;
cpu_update_apic(0, boot_cpu_physical_apicid);
x86_32_probe_bigsmp_early();
}
int generic_processor_info(int apicid)
{
int cpu, max = nr_cpu_ids;
bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
phys_cpu_present_map);
/*
* boot_cpu_physical_apicid is designed to have the apicid
* returned by read_apic_id(), i.e, the apicid of the
* currently booting-up processor. However, on some platforms,
* it is temporarily modified by the apicid reported as BSP
* through MP table. Concretely:
*
* - arch/x86/kernel/mpparse.c: MP_processor_info()
* - arch/x86/mm/amdtopology.c: amd_numa_init()
*
* This function is executed with the modified
* boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
* parameter doesn't work to disable APs on kdump 2nd kernel.
*
* Since fixing handling of boot_cpu_physical_apicid requires
* another discussion and tests on each platform, we leave it
* for now and here we use read_apic_id() directly in this
* function, generic_processor_info().
*/
if (disabled_cpu_apicid != BAD_APICID &&
disabled_cpu_apicid != read_apic_id() &&
disabled_cpu_apicid == apicid) {
/* The boot CPU must be set before MADT/MPTABLE parsing happens */
if (cpuid_to_apicid[0] == BAD_APICID)
panic("Boot CPU APIC not registered yet\n");
if (apicid == boot_cpu_physical_apicid)
return 0;
if (disabled_cpu_apicid == apicid) {
int thiscpu = num_processors + disabled_cpus;
pr_warn("APIC: Disabling requested cpu."
" Processor %d/0x%x ignored.\n", thiscpu, apicid);
disabled_cpus++;
return -ENODEV;
}
/*
* If boot cpu has not been detected yet, then only allow upto
* nr_cpu_ids - 1 processors and keep one slot free for boot cpu
*/
if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
apicid != boot_cpu_physical_apicid) {
int thiscpu = max + disabled_cpus - 1;
pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost"
" reached. Keeping one slot for boot cpu."
" Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n",
thiscpu, apicid);
disabled_cpus++;
return -ENODEV;
@ -2509,66 +2434,16 @@ int generic_processor_info(int apicid, int version)
return -EINVAL;
}
if (apicid == boot_cpu_physical_apicid) {
/*
* x86_bios_cpu_apicid is required to have processors listed
* in same order as logical cpu numbers. Hence the first
* entry is BSP, and so on.
* boot_cpu_init() already hold bit 0 in cpu_present_mask
* for BSP.
*/
cpu = 0;
/* Logical cpuid 0 is reserved for BSP. */
cpuid_to_apicid[0] = apicid;
} else {
cpu = allocate_logical_cpuid(apicid);
if (cpu < 0) {
disabled_cpus++;
return -EINVAL;
}
cpu = allocate_logical_cpuid(apicid);
if (cpu < 0) {
disabled_cpus++;
return -EINVAL;
}
/*
* Validate version
*/
if (version == 0x0) {
pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
cpu, apicid);
version = 0x10;
}
if (version != boot_cpu_apic_version) {
pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
boot_cpu_apic_version, cpu, version);
}
if (apicid > max_physical_apicid)
max_physical_apicid = apicid;
#if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
#endif
#ifdef CONFIG_X86_32
early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
apic->x86_32_early_logical_apicid(cpu);
#endif
set_cpu_possible(cpu, true);
physid_set(apicid, phys_cpu_present_map);
set_cpu_present(cpu, true);
num_processors++;
if (system_state != SYSTEM_BOOTING)
cpu_mark_primary_thread(cpu, apicid);
cpu_update_apic(cpu, apicid);
return cpu;
}
int hard_smp_processor_id(void)
{
return read_apic_id();
}
void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
bool dmar)
@ -2610,47 +2485,10 @@ u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
}
EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
#ifdef CONFIG_X86_64
void __init acpi_wake_cpu_handler_update(wakeup_cpu_handler handler)
{
struct apic **drv;
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++)
(*drv)->wakeup_secondary_cpu_64 = handler;
}
#endif
/*
* Override the generic EOI implementation with an optimized version.
* Only called during early boot when only one CPU is active and with
* interrupts disabled, so we know this does not race with actual APIC driver
* use.
*/
void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v))
{
struct apic **drv;
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
/* Should happen once for each apic */
WARN_ON((*drv)->eoi_write == eoi_write);
(*drv)->native_eoi_write = (*drv)->eoi_write;
(*drv)->eoi_write = eoi_write;
}
}
static void __init apic_bsp_up_setup(void)
{
#ifdef CONFIG_X86_64
apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid));
#else
/*
* Hack: In case of kdump, after a crash, kernel might be booting
* on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
* might be zero if read from MP tables. Get it from LAPIC.
*/
# ifdef CONFIG_CRASH_DUMP
boot_cpu_physical_apicid = read_apic_id();
# endif
#endif
physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
}
@ -2919,7 +2757,7 @@ int apic_is_clustered_box(void)
*/
static int __init setup_disableapic(char *arg)
{
disable_apic = 1;
apic_is_disabled = true;
setup_clear_cpu_cap(X86_FEATURE_APIC);
return 0;
}
@ -2956,11 +2794,11 @@ early_param("nolapic_timer", parse_nolapic_timer);
static int __init apic_set_verbosity(char *arg)
{
if (!arg) {
#ifdef CONFIG_X86_64
skip_ioapic_setup = 0;
if (IS_ENABLED(CONFIG_X86_32))
return -EINVAL;
ioapic_is_disabled = false;
return 0;
#endif
return -EINVAL;
}
if (strcmp("debug", arg) == 0)
@ -2981,11 +2819,11 @@ early_param("apic", apic_set_verbosity);
static int __init lapic_insert_resource(void)
{
if (!apic_phys)
if (!apic_mmio_base)
return -1;
/* Put local APIC into the resource map. */
lapic_resource.start = apic_phys;
lapic_resource.start = apic_mmio_base;
lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
insert_resource(&iomem_resource, &lapic_resource);

21
arch/x86/kernel/apic/apic_common.c
View file

@ -6,6 +6,8 @@
#include <linux/irq.h>
#include <asm/apic.h>
#include "local.h"
u32 apic_default_calc_apicid(unsigned int cpu)
{
return per_cpu(x86_cpu_to_apicid, cpu);
@ -29,18 +31,27 @@ void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap)
int default_cpu_present_to_apicid(int mps_cpu)
{
if (mps_cpu < nr_cpu_ids && cpu_present(mps_cpu))
return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu);
return (int)per_cpu(x86_cpu_to_apicid, mps_cpu);
else
return BAD_APICID;
}
EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid);
int default_check_phys_apicid_present(int phys_apicid)
bool default_apic_id_registered(void)
{
return physid_isset(phys_apicid, phys_cpu_present_map);
return physid_isset(read_apic_id(), phys_cpu_present_map);
}
int default_apic_id_valid(u32 apicid)
/*
* Set up the logical destination ID when the APIC operates in logical
* destination mode.
*/
void default_init_apic_ldr(void)
{
return (apicid < 255);
unsigned long val;
apic_write(APIC_DFR, APIC_DFR_FLAT);
val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
apic_write(APIC_LDR, val);
}

80
arch/x86/kernel/apic/apic_flat_64.c
View file

@ -28,26 +28,6 @@ static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
return 1;
}
/*
* Set up the logical destination ID.
*
* Intel recommends to set DFR, LDR and TPR before enabling
* an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
* document number 292116). So here it goes...
*/
void flat_init_apic_ldr(void)
{
unsigned long val;
unsigned long num, id;
num = smp_processor_id();
id = 1UL << num;
apic_write(APIC_DFR, APIC_DFR_FLAT);
val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
val |= SET_APIC_LOGICAL_ID(id);
apic_write(APIC_LDR, val);
}
static void _flat_send_IPI_mask(unsigned long mask, int vector)
{
unsigned long flags;
@ -86,16 +66,6 @@ static u32 set_apic_id(unsigned int id)
return (id & 0xFF) << 24;
}
static unsigned int read_xapic_id(void)
{
return flat_get_apic_id(apic_read(APIC_ID));
}
static int flat_apic_id_registered(void)
{
return physid_isset(read_xapic_id(), phys_cpu_present_map);
}
static int flat_phys_pkg_id(int initial_apic_id, int index_msb)
{
return initial_apic_id >> index_msb;
@ -110,23 +80,18 @@ static struct apic apic_flat __ro_after_init = {
.name = "flat",
.probe = flat_probe,
.acpi_madt_oem_check = flat_acpi_madt_oem_check,
.apic_id_valid = default_apic_id_valid,
.apic_id_registered = flat_apic_id_registered,
.apic_id_registered = default_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = true,
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.init_apic_ldr = default_init_apic_ldr,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = flat_phys_pkg_id,
.max_apic_id = 0xFE,
.get_apic_id = flat_get_apic_id,
.set_apic_id = set_apic_id,
@ -139,15 +104,13 @@ static struct apic apic_flat __ro_after_init = {
.send_IPI_all = default_send_IPI_all,
.send_IPI_self = default_send_IPI_self,
.inquire_remote_apic = default_inquire_remote_apic,
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = native_apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
.wait_icr_idle = apic_mem_wait_icr_idle,
.safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout,
};
/*
@ -178,22 +141,9 @@ static int physflat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
return 0;
}
static void physflat_init_apic_ldr(void)
{
/*
* LDR and DFR are not involved in physflat mode, rather:
* "In physical destination mode, the destination processor is
* specified by its local APIC ID [...]." (Intel SDM, 10.6.2.1)
*/
}
static int physflat_probe(void)
{
if (apic == &apic_physflat || num_possible_cpus() > 8 ||
jailhouse_paravirt())
return 1;
return 0;
return apic == &apic_physflat || num_possible_cpus() > 8 || jailhouse_paravirt();
}
static struct apic apic_physflat __ro_after_init = {
@ -201,8 +151,7 @@ static struct apic apic_physflat __ro_after_init = {
.name = "physical flat",
.probe = physflat_probe,
.acpi_madt_oem_check = physflat_acpi_madt_oem_check,
.apic_id_valid = default_apic_id_valid,
.apic_id_registered = flat_apic_id_registered,
.apic_id_registered = default_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
@ -210,14 +159,11 @@ static struct apic apic_physflat __ro_after_init = {
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = physflat_init_apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = flat_phys_pkg_id,
.max_apic_id = 0xFE,
.get_apic_id = flat_get_apic_id,
.set_apic_id = set_apic_id,
@ -230,15 +176,13 @@ static struct apic apic_physflat __ro_after_init = {
.send_IPI_all = default_send_IPI_all,
.send_IPI_self = default_send_IPI_self,
.inquire_remote_apic = default_inquire_remote_apic,
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = native_apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
.wait_icr_idle = apic_mem_wait_icr_idle,
.safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout,
};
/*

91
arch/x86/kernel/apic/apic_noop.c
View file

@ -8,92 +8,42 @@
* Though in case if apic is disabled (for some reason) we try
* to not uglify the caller's code and allow to call (some) apic routines
* like self-ipi, etc...
*
* FIXME: Remove this gunk. The above argument which was intentionally left
* in place is silly to begin with because none of the callbacks except for
* APIC::read/write() have a WARN_ON_ONCE() in them. Sigh...
*/
#include <linux/cpumask.h>
#include <linux/thread_info.h>
#include <asm/apic.h>
static void noop_init_apic_ldr(void) { }
static void noop_send_IPI(int cpu, int vector) { }
static void noop_send_IPI_mask(const struct cpumask *cpumask, int vector) { }
static void noop_send_IPI_mask_allbutself(const struct cpumask *cpumask, int vector) { }
static void noop_send_IPI_allbutself(int vector) { }
static void noop_send_IPI_all(int vector) { }
static void noop_send_IPI_self(int vector) { }
static void noop_apic_wait_icr_idle(void) { }
static void noop_apic_icr_write(u32 low, u32 id) { }
static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip)
{
return -1;
}
static u32 noop_safe_apic_wait_icr_idle(void)
{
return 0;
}
static u64 noop_apic_icr_read(void)
{
return 0;
}
static int noop_phys_pkg_id(int cpuid_apic, int index_msb)
{
return 0;
}
static unsigned int noop_get_apic_id(unsigned long x)
{
return 0;
}
static int noop_probe(void)
{
/*
* NOOP apic should not ever be
* enabled via probe routine
*/
return 0;
}
static int noop_apic_id_registered(void)
{
/*
* if we would be really "pedantic"
* we should pass read_apic_id() here
* but since NOOP suppose APIC ID = 0
* lets save a few cycles
*/
return physid_isset(0, phys_cpu_present_map);
}
static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) { return -1; }
static u64 noop_apic_icr_read(void) { return 0; }
static int noop_phys_pkg_id(int cpuid_apic, int index_msb) { return 0; }
static unsigned int noop_get_apic_id(unsigned long x) { return 0; }
static void noop_apic_eoi(void) { }
static u32 noop_apic_read(u32 reg)
{
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic);
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled);
return 0;
}
static void noop_apic_write(u32 reg, u32 v)
static void noop_apic_write(u32 reg, u32 val)
{
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic);
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled);
}
#ifdef CONFIG_X86_32
static int noop_x86_32_early_logical_apicid(int cpu)
{
return BAD_APICID;
}
#endif
struct apic apic_noop __ro_after_init = {
.name = "noop",
.probe = noop_probe,
.acpi_madt_oem_check = NULL,
.apic_id_valid = default_apic_id_valid,
.apic_id_registered = noop_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = true,
@ -101,18 +51,13 @@ struct apic apic_noop __ro_after_init = {
.disable_esr = 0,
.check_apicid_used = default_check_apicid_used,
.init_apic_ldr = noop_init_apic_ldr,
.ioapic_phys_id_map = default_ioapic_phys_id_map,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = physid_set_mask_of_physid,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = noop_phys_pkg_id,
.max_apic_id = 0xFE,
.get_apic_id = noop_get_apic_id,
.set_apic_id = NULL,
.calc_dest_apicid = apic_flat_calc_apicid,
@ -125,17 +70,9 @@ struct apic apic_noop __ro_after_init = {
.wakeup_secondary_cpu = noop_wakeup_secondary_cpu,
.inquire_remote_apic = NULL,
.read = noop_apic_read,
.write = noop_apic_write,
.eoi_write = noop_apic_write,
.eoi = noop_apic_eoi,
.icr_read = noop_apic_icr_read,
.icr_write = noop_apic_icr_write,
.wait_icr_idle = noop_apic_wait_icr_idle,
.safe_wait_icr_idle = noop_safe_apic_wait_icr_idle,
#ifdef CONFIG_X86_32
.x86_32_early_logical_apicid = noop_x86_32_early_logical_apicid,
#endif
};

50
arch/x86/kernel/apic/apic_numachip.c
View file

@ -56,17 +56,6 @@ static u32 numachip2_set_apic_id(unsigned int id)
return id << 24;
}
static int numachip_apic_id_valid(u32 apicid)
{
/* Trust what bootloader passes in MADT */
return 1;
}
static int numachip_apic_id_registered(void)
{
return 1;
}
static int numachip_phys_pkg_id(int initial_apic_id, int index_msb)
{
return initial_apic_id >> index_msb;
@ -228,38 +217,20 @@ static int numachip2_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
return 1;
}
/* APIC IPIs are queued */
static void numachip_apic_wait_icr_idle(void)
{
}
/* APIC NMI IPIs are queued */
static u32 numachip_safe_apic_wait_icr_idle(void)
{
return 0;
}
static const struct apic apic_numachip1 __refconst = {
.name = "NumaConnect system",
.probe = numachip1_probe,
.acpi_madt_oem_check = numachip1_acpi_madt_oem_check,
.apic_id_valid = numachip_apic_id_valid,
.apic_id_registered = numachip_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = numachip_phys_pkg_id,
.max_apic_id = UINT_MAX,
.get_apic_id = numachip1_get_apic_id,
.set_apic_id = numachip1_set_apic_id,
@ -273,15 +244,12 @@ static const struct apic apic_numachip1 __refconst = {
.send_IPI_self = numachip_send_IPI_self,
.wakeup_secondary_cpu = numachip_wakeup_secondary,
.inquire_remote_apic = NULL, /* REMRD not supported */
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = numachip_apic_wait_icr_idle,
.safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle,
};
apic_driver(apic_numachip1);
@ -290,23 +258,16 @@ static const struct apic apic_numachip2 __refconst = {
.name = "NumaConnect2 system",
.probe = numachip2_probe,
.acpi_madt_oem_check = numachip2_acpi_madt_oem_check,
.apic_id_valid = numachip_apic_id_valid,
.apic_id_registered = numachip_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = numachip_phys_pkg_id,
.max_apic_id = UINT_MAX,
.get_apic_id = numachip2_get_apic_id,
.set_apic_id = numachip2_set_apic_id,
@ -320,15 +281,12 @@ static const struct apic apic_numachip2 __refconst = {
.send_IPI_self = numachip_send_IPI_self,
.wakeup_secondary_cpu = numachip_wakeup_secondary,
.inquire_remote_apic = NULL, /* REMRD not supported */
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = numachip_apic_wait_icr_idle,
.safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle,
};
apic_driver(apic_numachip2);

89
arch/x86/kernel/apic/bigsmp_32.c
View file

@ -18,56 +18,17 @@ static unsigned bigsmp_get_apic_id(unsigned long x)
return (x >> 24) & 0xFF;
}
static int bigsmp_apic_id_registered(void)
{
return 1;
}
static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid)
{
return false;
}
static int bigsmp_early_logical_apicid(int cpu)
{
/* on bigsmp, logical apicid is the same as physical */
return early_per_cpu(x86_cpu_to_apicid, cpu);
}
/*
* bigsmp enables physical destination mode
* and doesn't use LDR and DFR
*/
static void bigsmp_init_apic_ldr(void)
{
}
static void bigsmp_setup_apic_routing(void)
{
printk(KERN_INFO
"Enabling APIC mode: Physflat. Using %d I/O APICs\n",
nr_ioapics);
}
static int bigsmp_cpu_present_to_apicid(int mps_cpu)
{
if (mps_cpu < nr_cpu_ids)
return (int) per_cpu(x86_bios_cpu_apicid, mps_cpu);
return BAD_APICID;
}
static void bigsmp_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap)
{
/* For clustered we don't have a good way to do this yet - hack */
physids_promote(0xFFL, retmap);
}
static int bigsmp_check_phys_apicid_present(int phys_apicid)
{
return 1;
}
static int bigsmp_phys_pkg_id(int cpuid_apic, int index_msb)
{
return cpuid_apic >> index_msb;
@ -111,21 +72,13 @@ static const struct dmi_system_id bigsmp_dmi_table[] = {
static int probe_bigsmp(void)
{
if (def_to_bigsmp)
dmi_bigsmp = 1;
else
dmi_check_system(bigsmp_dmi_table);
return dmi_bigsmp;
return dmi_check_system(bigsmp_dmi_table);
}
static struct apic apic_bigsmp __ro_after_init = {
.name = "bigsmp",
.probe = probe_bigsmp,
.acpi_madt_oem_check = NULL,
.apic_id_valid = default_apic_id_valid,
.apic_id_registered = bigsmp_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
@ -133,14 +86,11 @@ static struct apic apic_bigsmp __ro_after_init = {
.disable_esr = 1,
.check_apicid_used = bigsmp_check_apicid_used,
.init_apic_ldr = bigsmp_init_apic_ldr,
.ioapic_phys_id_map = bigsmp_ioapic_phys_id_map,
.setup_apic_routing = bigsmp_setup_apic_routing,
.cpu_present_to_apicid = bigsmp_cpu_present_to_apicid,
.apicid_to_cpu_present = physid_set_mask_of_physid,
.check_phys_apicid_present = bigsmp_check_phys_apicid_present,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.phys_pkg_id = bigsmp_phys_pkg_id,
.max_apic_id = 0xFE,
.get_apic_id = bigsmp_get_apic_id,
.set_apic_id = NULL,
@ -153,37 +103,24 @@ static struct apic apic_bigsmp __ro_after_init = {
.send_IPI_all = bigsmp_send_IPI_all,
.send_IPI_self = default_send_IPI_self,
.inquire_remote_apic = default_inquire_remote_apic,
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = native_apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
.x86_32_early_logical_apicid = bigsmp_early_logical_apicid,
.wait_icr_idle = apic_mem_wait_icr_idle,
.safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout,
};
void __init generic_bigsmp_probe(void)
bool __init apic_bigsmp_possible(bool cmdline_override)
{
unsigned int cpu;
return apic == &apic_bigsmp || !cmdline_override;
}
if (!probe_bigsmp())
return;
apic = &apic_bigsmp;
for_each_possible_cpu(cpu) {
if (early_per_cpu(x86_cpu_to_logical_apicid,
cpu) == BAD_APICID)
continue;
early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
bigsmp_early_logical_apicid(cpu);
}
pr_info("Overriding APIC driver with %s\n", apic_bigsmp.name);
void __init apic_bigsmp_force(void)
{
if (apic != &apic_bigsmp)
apic_install_driver(&apic_bigsmp);
}
apic_driver(apic_bigsmp);

4
arch/x86/kernel/apic/hw_nmi.c
View file

@ -21,6 +21,8 @@
#include <linux/init.h>
#include <linux/delay.h>
#include "local.h"
#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF
u64 hw_nmi_get_sample_period(int watchdog_thresh)
{
@ -31,7 +33,7 @@ u64 hw_nmi_get_sample_period(int watchdog_thresh)
#ifdef arch_trigger_cpumask_backtrace
static void nmi_raise_cpu_backtrace(cpumask_t *mask)
{
apic->send_IPI_mask(mask, NMI_VECTOR);
__apic_send_IPI_mask(mask, NMI_VECTOR);
}
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)

110
arch/x86/kernel/apic/init.c Normal file
View file

@ -0,0 +1,110 @@
// SPDX-License-Identifier: GPL-2.0-only
#define pr_fmt(fmt) "APIC: " fmt
#include <asm/apic.h>
#include "local.h"
/*
* Use DEFINE_STATIC_CALL_NULL() to avoid having to provide stub functions
* for each callback. The callbacks are setup during boot and all except
* wait_icr_idle() must be initialized before usage. The IPI wrappers
* use static_call() and not static_call_cond() to catch any fails.
*/
#define DEFINE_APIC_CALL(__cb) \
DEFINE_STATIC_CALL_NULL(apic_call_##__cb, *apic->__cb)
DEFINE_APIC_CALL(eoi);
DEFINE_APIC_CALL(native_eoi);
DEFINE_APIC_CALL(icr_read);
DEFINE_APIC_CALL(icr_write);
DEFINE_APIC_CALL(read);
DEFINE_APIC_CALL(send_IPI);
DEFINE_APIC_CALL(send_IPI_mask);
DEFINE_APIC_CALL(send_IPI_mask_allbutself);
DEFINE_APIC_CALL(send_IPI_allbutself);
DEFINE_APIC_CALL(send_IPI_all);
DEFINE_APIC_CALL(send_IPI_self);
DEFINE_APIC_CALL(wait_icr_idle);
DEFINE_APIC_CALL(wakeup_secondary_cpu);
DEFINE_APIC_CALL(wakeup_secondary_cpu_64);
DEFINE_APIC_CALL(write);
EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_mask);
EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_self);
/* The container for function call overrides */
struct apic_override __x86_apic_override __initdata;
#define apply_override(__cb) \
if (__x86_apic_override.__cb) \
apic->__cb = __x86_apic_override.__cb
static __init void restore_override_callbacks(void)
{
apply_override(eoi);
apply_override(native_eoi);
apply_override(write);
apply_override(read);
apply_override(send_IPI);
apply_override(send_IPI_mask);
apply_override(send_IPI_mask_allbutself);
apply_override(send_IPI_allbutself);
apply_override(send_IPI_all);
apply_override(send_IPI_self);
apply_override(icr_read);
apply_override(icr_write);
apply_override(wakeup_secondary_cpu);
apply_override(wakeup_secondary_cpu_64);
}
#define update_call(__cb) \
static_call_update(apic_call_##__cb, *apic->__cb)
static __init void update_static_calls(void)
{
update_call(eoi);
update_call(native_eoi);
update_call(write);
update_call(read);
update_call(send_IPI);
update_call(send_IPI_mask);
update_call(send_IPI_mask_allbutself);
update_call(send_IPI_allbutself);
update_call(send_IPI_all);
update_call(send_IPI_self);
update_call(icr_read);
update_call(icr_write);
update_call(wait_icr_idle);
update_call(wakeup_secondary_cpu);
update_call(wakeup_secondary_cpu_64);
}
void __init apic_setup_apic_calls(void)
{
/* Ensure that the default APIC has native_eoi populated */
apic->native_eoi = apic->eoi;
update_static_calls();
pr_info("Static calls initialized\n");
}
void __init apic_install_driver(struct apic *driver)
{
if (apic == driver)
return;
apic = driver;
if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid)
apic->max_apic_id = x2apic_max_apicid;
/* Copy the original eoi() callback as KVM/HyperV might overwrite it */
if (!apic->native_eoi)
apic->native_eoi = apic->eoi;
/* Apply any already installed callback overrides */
restore_override_callbacks();
update_static_calls();
pr_info("Switched APIC routing to: %s\n", driver->name);
}

30
arch/x86/kernel/apic/io_apic.c
View file

@ -178,7 +178,7 @@ int mp_bus_id_to_type[MAX_MP_BUSSES];
DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
int skip_ioapic_setup;
bool ioapic_is_disabled __ro_after_init;
/**
* disable_ioapic_support() - disables ioapic support at runtime
@ -189,7 +189,7 @@ void disable_ioapic_support(void)
noioapicquirk = 1;
noioapicreroute = -1;
#endif
skip_ioapic_setup = 1;
ioapic_is_disabled = true;
}
static int __init parse_noapic(char *str)
@ -831,7 +831,7 @@ static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity)
{
int ioapic, pin, idx;
if (skip_ioapic_setup)
if (ioapic_is_disabled)
return -1;
ioapic = mp_find_ioapic(gsi);
@ -1366,7 +1366,7 @@ void __init enable_IO_APIC(void)
int i8259_apic, i8259_pin;
int apic, pin;
if (skip_ioapic_setup)
if (ioapic_is_disabled)
nr_ioapics = 0;
if (!nr_legacy_irqs() || !nr_ioapics)
@ -1511,13 +1511,9 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void)
physid_set(i, phys_id_present_map);
ioapics[ioapic_idx].mp_config.apicid = i;
} else {
physid_mask_t tmp;
apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
&tmp);
apic_printk(APIC_VERBOSE, "Setting %d in the "
"phys_id_present_map\n",
mpc_ioapic_id(ioapic_idx));
physids_or(phys_id_present_map, phys_id_present_map, tmp);
apic_printk(APIC_VERBOSE, "Setting %d in the phys_id_present_map\n",
mpc_ioapic_id(ioapic_idx));
physid_set(mpc_ioapic_id(ioapic_idx), phys_id_present_map);
}
/*
@ -1827,7 +1823,7 @@ static void ioapic_ack_level(struct irq_data *irq_data)
* We must acknowledge the irq before we move it or the acknowledge will
* not propagate properly.
*/
ack_APIC_irq();
apic_eoi();
/*
* Tail end of clearing remote IRR bit (either by delivering the EOI
@ -2050,7 +2046,7 @@ static void unmask_lapic_irq(struct irq_data *data)
static void ack_lapic_irq(struct irq_data *data)
{
ack_APIC_irq();
apic_eoi();
}
static struct irq_chip lapic_chip __read_mostly = {
@ -2095,7 +2091,7 @@ static inline void __init unlock_ExtINT_logic(void)
entry0 = ioapic_read_entry(apic, pin);
clear_IO_APIC_pin(apic, pin);
apic_id = hard_smp_processor_id();
apic_id = read_apic_id();
memset(&entry1, 0, sizeof(entry1));
entry1.dest_mode_logical = true;
@ -2399,7 +2395,7 @@ void __init setup_IO_APIC(void)
{
int ioapic;
if (skip_ioapic_setup || !nr_ioapics)
if (ioapic_is_disabled || !nr_ioapics)
return;
io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL;
@ -2546,7 +2542,7 @@ static int io_apic_get_unique_id(int ioapic, int apic_id)
apic_id = i;
}
apic->apicid_to_cpu_present(apic_id, &tmp);
physid_set_mask_of_physid(apic_id, &tmp);
physids_or(apic_id_map, apic_id_map, tmp);
if (reg_00.bits.ID != apic_id) {
@ -2715,7 +2711,7 @@ void __init io_apic_init_mappings(void)
"address found in MPTABLE, "
"disabling IO/APIC support!\n");
smp_found_config = 0;
skip_ioapic_setup = 1;
ioapic_is_disabled = true;
goto fake_ioapic_page;
}
#endif

176
arch/x86/kernel/apic/ipi.c
View file

@ -1,7 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpumask.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <asm/io_apic.h>
#include "local.h"
@ -52,9 +54,9 @@ void apic_send_IPI_allbutself(unsigned int vector)
return;
if (static_branch_likely(&apic_use_ipi_shorthand))
apic->send_IPI_allbutself(vector);
__apic_send_IPI_allbutself(vector);
else
apic->send_IPI_mask_allbutself(cpu_online_mask, vector);
__apic_send_IPI_mask_allbutself(cpu_online_mask, vector);
}
/*
@ -68,12 +70,12 @@ void native_smp_send_reschedule(int cpu)
WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
return;
}
apic->send_IPI(cpu, RESCHEDULE_VECTOR);
__apic_send_IPI(cpu, RESCHEDULE_VECTOR);
}
void native_send_call_func_single_ipi(int cpu)
{
apic->send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
__apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
}
void native_send_call_func_ipi(const struct cpumask *mask)
@ -85,14 +87,14 @@ void native_send_call_func_ipi(const struct cpumask *mask)
goto sendmask;
if (cpumask_test_cpu(cpu, mask))
apic->send_IPI_all(CALL_FUNCTION_VECTOR);
__apic_send_IPI_all(CALL_FUNCTION_VECTOR);
else if (num_online_cpus() > 1)
apic->send_IPI_allbutself(CALL_FUNCTION_VECTOR);
__apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR);
return;
}
sendmask:
apic->send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
__apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
}
#endif /* CONFIG_SMP */
@ -102,74 +104,77 @@ static inline int __prepare_ICR2(unsigned int mask)
return SET_XAPIC_DEST_FIELD(mask);
}
static inline void __xapic_wait_icr_idle(void)
u32 apic_mem_wait_icr_idle_timeout(void)
{
int cnt;
for (cnt = 0; cnt < 1000; cnt++) {
if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
return 0;
inc_irq_stat(icr_read_retry_count);
udelay(100);
}
return APIC_ICR_BUSY;
}
void apic_mem_wait_icr_idle(void)
{
while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
cpu_relax();
}
void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
/*
* This is safe against interruption because it only writes the lower 32
* bits of the APIC_ICR register. The destination field is ignored for
* short hand IPIs.
*
* wait_icr_idle()
* write(ICR2, dest)
* NMI
* wait_icr_idle()
* write(ICR)
* wait_icr_idle()
* write(ICR)
*
* This function does not need to disable interrupts as there is no ICR2
* interaction. The memory write is direct except when the machine is
* affected by the 11AP Pentium erratum, which turns the plain write into
* an XCHG operation.
*/
static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
{
/*
* Subtle. In the case of the 'never do double writes' workaround
* we have to lock out interrupts to be safe. As we don't care
* of the value read we use an atomic rmw access to avoid costly
* cli/sti. Otherwise we use an even cheaper single atomic write
* to the APIC.
*/
unsigned int cfg;
/*
* Wait for idle.
* Wait for the previous ICR command to complete. Use
* safe_apic_wait_icr_idle() for the NMI vector as there have been
* issues where otherwise the system hangs when the panic CPU tries
* to stop the others before launching the kdump kernel.
*/
if (unlikely(vector == NMI_VECTOR))
safe_apic_wait_icr_idle();
apic_mem_wait_icr_idle_timeout();
else
__xapic_wait_icr_idle();
apic_mem_wait_icr_idle();
/*
* No need to touch the target chip field. Also the destination
* mode is ignored when a shorthand is used.
*/
cfg = __prepare_ICR(shortcut, vector, 0);
/*
* Send the IPI. The write to APIC_ICR fires this off.
*/
native_apic_mem_write(APIC_ICR, cfg);
/* Destination field (ICR2) and the destination mode are ignored */
native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0));
}
/*
* This is used to send an IPI with no shorthand notation (the destination is
* specified in bits 56 to 63 of the ICR).
*/
void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest)
void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
unsigned int dest_mode)
{
unsigned long cfg;
/*
* Wait for idle.
*/
/* See comment in __default_send_IPI_shortcut() */
if (unlikely(vector == NMI_VECTOR))
safe_apic_wait_icr_idle();
apic_mem_wait_icr_idle_timeout();
else
__xapic_wait_icr_idle();
apic_mem_wait_icr_idle();
/*
* prepare target chip field
*/
cfg = __prepare_ICR2(mask);
native_apic_mem_write(APIC_ICR2, cfg);
/*
* program the ICR
*/
cfg = __prepare_ICR(0, vector, dest);
/*
* Send the IPI. The write to APIC_ICR fires this off.
*/
native_apic_mem_write(APIC_ICR, cfg);
/* Set the IPI destination field in the ICR */
native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask));
/* Send it with the proper destination mode */
native_apic_mem_write(APIC_ICR, __prepare_ICR(0, vector, dest_mode));
}
void default_send_IPI_single_phys(int cpu, int vector)
@ -184,18 +189,13 @@ void default_send_IPI_single_phys(int cpu, int vector)
void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
{
unsigned long query_cpu;
unsigned long flags;
unsigned long cpu;
/*
* Hack. The clustered APIC addressing mode doesn't allow us to send
* to an arbitrary mask, so I do a unicast to each CPU instead.
* - mbligh
*/
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
for_each_cpu(cpu, mask) {
__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
query_cpu), vector, APIC_DEST_PHYSICAL);
cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
}
@ -203,18 +203,15 @@ void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
int vector)
{
unsigned int this_cpu = smp_processor_id();
unsigned int query_cpu;
unsigned int cpu, this_cpu = smp_processor_id();
unsigned long flags;
/* See Hack comment above */
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
for_each_cpu(cpu, mask) {
if (cpu == this_cpu)
continue;
__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
query_cpu), vector, APIC_DEST_PHYSICAL);
cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
}
@ -224,7 +221,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
*/
void default_send_IPI_single(int cpu, int vector)
{
apic->send_IPI_mask(cpumask_of(cpu), vector);
__apic_send_IPI_mask(cpumask_of(cpu), vector);
}
void default_send_IPI_allbutself(int vector)
@ -243,50 +240,32 @@ void default_send_IPI_self(int vector)
}
#ifdef CONFIG_X86_32
void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
int vector)
void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
{
unsigned long flags;
unsigned int query_cpu;
/*
* Hack. The clustered APIC addressing mode doesn't allow us to send
* to an arbitrary mask, so I do a unicasts to each CPU instead. This
* should be modified to do 1 message per cluster ID - mbligh
*/
unsigned int cpu;
local_irq_save(flags);
for_each_cpu(query_cpu, mask)
__default_send_IPI_dest_field(
early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
vector, APIC_DEST_LOGICAL);
for_each_cpu(cpu, mask)
__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
local_irq_restore(flags);
}
void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
int vector)
{
unsigned int cpu, this_cpu = smp_processor_id();
unsigned long flags;
unsigned int query_cpu;
unsigned int this_cpu = smp_processor_id();
/* See Hack comment above */
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
for_each_cpu(cpu, mask) {
if (cpu == this_cpu)
continue;
__default_send_IPI_dest_field(
early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
vector, APIC_DEST_LOGICAL);
}
__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
}
local_irq_restore(flags);
}
/*
* This is only used on smaller machines.
*/
void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
{
unsigned long mask = cpumask_bits(cpumask)[0];
@ -302,7 +281,6 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
}
#ifdef CONFIG_SMP
/* must come after the send_IPI functions above for inlining */
static int convert_apicid_to_cpu(int apic_id)
{
int i;
@ -321,7 +299,7 @@ int safe_smp_processor_id(void)
if (!boot_cpu_has(X86_FEATURE_APIC))
return 0;
apicid = hard_smp_processor_id();
apicid = read_apic_id();
if (apicid == BAD_APICID)
return 0;

30
arch/x86/kernel/apic/local.h
View file

@ -13,18 +13,16 @@
#include <asm/irq_vectors.h>
#include <asm/apic.h>
/* APIC flat 64 */
void flat_init_apic_ldr(void);
/* X2APIC */
int x2apic_apic_id_valid(u32 apicid);
int x2apic_apic_id_registered(void);
void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest);
unsigned int x2apic_get_apic_id(unsigned long id);
u32 x2apic_set_apic_id(unsigned int id);
int x2apic_phys_pkg_id(int initial_apicid, int index_msb);
void x2apic_send_IPI_all(int vector);
void x2apic_send_IPI_allbutself(int vector);
void x2apic_send_IPI_self(int vector);
void __x2apic_send_IPI_shorthand(int vector, u32 which);
extern u32 x2apic_max_apicid;
/* IPI */
@ -46,7 +44,10 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector,
return icr;
}
void __default_send_IPI_shortcut(unsigned int shortcut, int vector);
void default_init_apic_ldr(void);
void apic_mem_wait_icr_idle(void);
u32 apic_mem_wait_icr_idle_timeout(void);
/*
* This is used to send an IPI with no shorthand notation (the destination is
@ -62,8 +63,23 @@ void default_send_IPI_allbutself(int vector);
void default_send_IPI_all(int vector);
void default_send_IPI_self(int vector);
bool default_apic_id_registered(void);
#ifdef CONFIG_X86_32
void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector);
void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector);
void default_send_IPI_mask_logical(const struct cpumask *mask, int vector);
void x86_32_probe_bigsmp_early(void);
void x86_32_install_bigsmp(void);
#else
static inline void x86_32_probe_bigsmp_early(void) { }
static inline void x86_32_install_bigsmp(void) { }
#endif
#ifdef CONFIG_X86_BIGSMP
bool apic_bigsmp_possible(bool cmdline_selected);
void apic_bigsmp_force(void);
#else
static inline bool apic_bigsmp_possible(bool cmdline_selected) { return false; };
static inline void apic_bigsmp_force(void) { }
#endif

2
arch/x86/kernel/apic/msi.c
View file

@ -269,7 +269,7 @@ static const struct msi_parent_ops x86_vector_msi_parent_ops = {
struct irq_domain * __init native_create_pci_msi_domain(void)
{
if (disable_apic)
if (apic_is_disabled)
return NULL;
x86_vector_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;

119
arch/x86/kernel/apic/probe_32.c
View file

@ -10,46 +10,14 @@
#include <linux/errno.h>
#include <linux/smp.h>
#include <xen/xen.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
#include <asm/acpi.h>
#include "local.h"
static int default_x86_32_early_logical_apicid(int cpu)
{
return 1 << cpu;
}
static void setup_apic_flat_routing(void)
{
#ifdef CONFIG_X86_IO_APIC
printk(KERN_INFO
"Enabling APIC mode: Flat. Using %d I/O APICs\n",
nr_ioapics);
#endif
}
static int default_apic_id_registered(void)
{
return physid_isset(read_apic_id(), phys_cpu_present_map);
}
/*
* Set up the logical destination ID. Intel recommends to set DFR, LDR and
* TPR before enabling an APIC. See e.g. "AP-388 82489DX User's Manual"
* (Intel document number 292116).
*/
static void default_init_apic_ldr(void)
{
unsigned long val;
apic_write(APIC_DFR, APIC_DFR_VALUE);
val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
apic_write(APIC_LDR, val);
}
static int default_phys_pkg_id(int cpuid_apic, int index_msb)
{
return cpuid_apic >> index_msb;
@ -65,8 +33,6 @@ static struct apic apic_default __ro_after_init = {
.name = "default",
.probe = probe_default,
.acpi_madt_oem_check = NULL,
.apic_id_valid = default_apic_id_valid,
.apic_id_registered = default_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
@ -77,14 +43,11 @@ static struct apic apic_default __ro_after_init = {
.check_apicid_used = default_check_apicid_used,
.init_apic_ldr = default_init_apic_ldr,
.ioapic_phys_id_map = default_ioapic_phys_id_map,
.setup_apic_routing = setup_apic_flat_routing,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = physid_set_mask_of_physid,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = default_phys_pkg_id,
.max_apic_id = 0xFE,
.get_apic_id = default_get_apic_id,
.set_apic_id = NULL,
.calc_dest_apicid = apic_flat_calc_apicid,
@ -95,17 +58,13 @@ static struct apic apic_default __ro_after_init = {
.send_IPI_all = default_send_IPI_all,
.send_IPI_self = default_send_IPI_self,
.inquire_remote_apic = default_inquire_remote_apic,
.read = native_apic_mem_read,
.write = native_apic_mem_write,
.eoi_write = native_apic_mem_write,
.eoi = native_apic_mem_eoi,
.icr_read = native_apic_icr_read,
.icr_write = native_apic_icr_write,
.wait_icr_idle = native_apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
.x86_32_early_logical_apicid = default_x86_32_early_logical_apicid,
.wait_icr_idle = apic_mem_wait_icr_idle,
.safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout,
};
apic_driver(apic_default);
@ -123,7 +82,7 @@ static int __init parse_apic(char *arg)
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
if (!strcmp((*drv)->name, arg)) {
apic = *drv;
apic_install_driver(*drv);
cmdline_apic = 1;
return 0;
}
@ -134,49 +93,43 @@ static int __init parse_apic(char *arg)
}
early_param("apic", parse_apic);
void __init default_setup_apic_routing(void)
void __init x86_32_probe_bigsmp_early(void)
{
int version = boot_cpu_apic_version;
if (nr_cpu_ids <= 8 || xen_pv_domain())
return;
if (num_possible_cpus() > 8) {
if (IS_ENABLED(CONFIG_X86_BIGSMP)) {
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_INTEL:
if (!APIC_XAPIC(version)) {
def_to_bigsmp = 0;
if (!APIC_XAPIC(boot_cpu_apic_version))
break;
}
/* P4 and above */
fallthrough;
case X86_VENDOR_HYGON:
case X86_VENDOR_AMD:
def_to_bigsmp = 1;
if (apic_bigsmp_possible(cmdline_apic))
return;
break;
}
}
#ifdef CONFIG_X86_BIGSMP
/*
* This is used to switch to bigsmp mode when
* - There is no apic= option specified by the user
* - generic_apic_probe() has chosen apic_default as the sub_arch
* - we find more than 8 CPUs in acpi LAPIC listing with xAPIC support
*/
if (!cmdline_apic && apic == &apic_default)
generic_bigsmp_probe();
#endif
if (apic->setup_apic_routing)
apic->setup_apic_routing();
pr_info("Limiting to 8 possible CPUs\n");
set_nr_cpu_ids(8);
}
void __init generic_apic_probe(void)
void __init x86_32_install_bigsmp(void)
{
if (nr_cpu_ids > 8 && !xen_pv_domain())
apic_bigsmp_force();
}
void __init x86_32_probe_apic(void)
{
if (!cmdline_apic) {
struct apic **drv;
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
if ((*drv)->probe()) {
apic = *drv;
apic_install_driver(*drv);
break;
}
}
@ -184,26 +137,4 @@ void __init generic_apic_probe(void)
if (drv == __apicdrivers_end)
panic("Didn't find an APIC driver");
}
printk(KERN_INFO "Using APIC driver %s\n", apic->name);
}
/* This function can switch the APIC even after the initial ->probe() */
int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
struct apic **drv;
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
if (!(*drv)->acpi_madt_oem_check)
continue;
if (!(*drv)->acpi_madt_oem_check(oem_id, oem_table_id))
continue;
if (!cmdline_apic) {
apic = *drv;
printk(KERN_INFO "Switched to APIC driver `%s'.\n",
apic->name);
}
return 1;
}
return 0;
}

18
arch/x86/kernel/apic/probe_64.c
View file

@ -13,10 +13,8 @@
#include "local.h"
/*
* Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
*/
void __init default_setup_apic_routing(void)
/* Select the appropriate APIC driver */
void __init x86_64_probe_apic(void)
{
struct apic **drv;
@ -24,11 +22,7 @@ void __init default_setup_apic_routing(void)
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
if ((*drv)->probe && (*drv)->probe()) {
if (apic != *drv) {
apic = *drv;
pr_info("Switched APIC routing to %s.\n",
apic->name);
}
apic_install_driver(*drv);
break;
}
}
@ -40,11 +34,7 @@ int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
if ((*drv)->acpi_madt_oem_check(oem_id, oem_table_id)) {
if (apic != *drv) {
apic = *drv;
pr_info("Setting APIC routing to %s.\n",
apic->name);
}
apic_install_driver(*drv);
return 1;
}
}

116
arch/x86/kernel/apic/vector.c
View file

@ -44,7 +44,18 @@ static cpumask_var_t vector_searchmask;
static struct irq_chip lapic_controller;
static struct irq_matrix *vector_matrix;
#ifdef CONFIG_SMP
static DEFINE_PER_CPU(struct hlist_head, cleanup_list);
static void vector_cleanup_callback(struct timer_list *tmr);
struct vector_cleanup {
struct hlist_head head;
struct timer_list timer;
};
static DEFINE_PER_CPU(struct vector_cleanup, vector_cleanup) = {
.head = HLIST_HEAD_INIT,
.timer = __TIMER_INITIALIZER(vector_cleanup_callback, TIMER_PINNED),
};
#endif
void lock_vector_lock(void)
@ -536,7 +547,7 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
struct irq_data *irqd;
int i, err, node;
if (disable_apic)
if (apic_is_disabled)
return -ENXIO;
/*
@ -680,7 +691,7 @@ static int x86_vector_select(struct irq_domain *d, struct irq_fwspec *fwspec,
* if IRQ remapping is enabled. APIC IDs above 15 bits are
* only permitted if IRQ remapping is enabled, so check that.
*/
if (apic->apic_id_valid(32768))
if (apic_id_valid(32768))
return 0;
return x86_fwspec_is_ioapic(fwspec) || x86_fwspec_is_hpet(fwspec);
@ -841,10 +852,21 @@ void lapic_online(void)
this_cpu_write(vector_irq[vector], __setup_vector_irq(vector));
}
static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr);
void lapic_offline(void)
{
struct vector_cleanup *cl = this_cpu_ptr(&vector_cleanup);
lock_vector_lock();
/* In case the vector cleanup timer has not expired */
__vector_cleanup(cl, false);
irq_matrix_offline(vector_matrix);
WARN_ON_ONCE(try_to_del_timer_sync(&cl->timer) < 0);
WARN_ON_ONCE(!hlist_empty(&cl->head));
unlock_vector_lock();
}
@ -876,7 +898,7 @@ static int apic_retrigger_irq(struct irq_data *irqd)
unsigned long flags;
raw_spin_lock_irqsave(&vector_lock, flags);
apic->send_IPI(apicd->cpu, apicd->vector);
__apic_send_IPI(apicd->cpu, apicd->vector);
raw_spin_unlock_irqrestore(&vector_lock, flags);
return 1;
@ -885,7 +907,7 @@ static int apic_retrigger_irq(struct irq_data *irqd)
void apic_ack_irq(struct irq_data *irqd)
{
irq_move_irq(irqd);
ack_APIC_irq();
apic_eoi();
}
void apic_ack_edge(struct irq_data *irqd)
@ -934,62 +956,98 @@ static void free_moved_vector(struct apic_chip_data *apicd)
apicd->move_in_progress = 0;
}
DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup)
static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr)
{
struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
struct apic_chip_data *apicd;
struct hlist_node *tmp;
bool rearm = false;
ack_APIC_irq();
/* Prevent vectors vanishing under us */
raw_spin_lock(&vector_lock);
lockdep_assert_held(&vector_lock);
hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
hlist_for_each_entry_safe(apicd, tmp, &cl->head, clist) {
unsigned int irr, vector = apicd->prev_vector;
/*
* Paranoia: Check if the vector that needs to be cleaned
* up is registered at the APICs IRR. If so, then this is
* not the best time to clean it up. Clean it up in the
* next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
* to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest
* priority external vector, so on return from this
* interrupt the device interrupt will happen first.
* up is registered at the APICs IRR. That's clearly a
* hardware issue if the vector arrived on the old target
* _after_ interrupts were disabled above. Keep @apicd
* on the list and schedule the timer again to give the CPU
* a chance to handle the pending interrupt.
*
* Do not check IRR when called from lapic_offline(), because
* fixup_irqs() was just called to scan IRR for set bits and
* forward them to new destination CPUs via IPIs.
*/
irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
irr = check_irr ? apic_read(APIC_IRR + (vector / 32 * 0x10)) : 0;
if (irr & (1U << (vector % 32))) {
apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
pr_warn_once("Moved interrupt pending in old target APIC %u\n", apicd->irq);
rearm = true;
continue;
}
free_moved_vector(apicd);
}
raw_spin_unlock(&vector_lock);
/*
* Must happen under vector_lock to make the timer_pending() check
* in __vector_schedule_cleanup() race free against the rearm here.
*/
if (rearm)
mod_timer(&cl->timer, jiffies + 1);
}
static void __send_cleanup_vector(struct apic_chip_data *apicd)
static void vector_cleanup_callback(struct timer_list *tmr)
{
unsigned int cpu;
struct vector_cleanup *cl = container_of(tmr, typeof(*cl), timer);
/* Prevent vectors vanishing under us */
raw_spin_lock_irq(&vector_lock);
__vector_cleanup(cl, true);
raw_spin_unlock_irq(&vector_lock);
}
static void __vector_schedule_cleanup(struct apic_chip_data *apicd)
{
unsigned int cpu = apicd->prev_cpu;
raw_spin_lock(&vector_lock);
apicd->move_in_progress = 0;
cpu = apicd->prev_cpu;
if (cpu_online(cpu)) {
hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu));
apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR);
struct vector_cleanup *cl = per_cpu_ptr(&vector_cleanup, cpu);
hlist_add_head(&apicd->clist, &cl->head);
/*
* The lockless timer_pending() check is safe here. If it
* returns true, then the callback will observe this new
* apic data in the hlist as everything is serialized by
* vector lock.
*
* If it returns false then the timer is either not armed
* or the other CPU executes the callback, which again
* would be blocked on vector lock. Rearming it in the
* latter case makes it fire for nothing.
*
* This is also safe against the callback rearming the timer
* because that's serialized via vector lock too.
*/
if (!timer_pending(&cl->timer)) {
cl->timer.expires = jiffies + 1;
add_timer_on(&cl->timer, cpu);
}
} else {
apicd->prev_vector = 0;
}
raw_spin_unlock(&vector_lock);
}
void send_cleanup_vector(struct irq_cfg *cfg)
void vector_schedule_cleanup(struct irq_cfg *cfg)
{
struct apic_chip_data *apicd;
apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg);
if (apicd->move_in_progress)
__send_cleanup_vector(apicd);
__vector_schedule_cleanup(apicd);
}
void irq_complete_move(struct irq_cfg *cfg)
@ -1007,7 +1065,7 @@ void irq_complete_move(struct irq_cfg *cfg)
* on the same CPU.
*/
if (apicd->cpu == smp_processor_id())
__send_cleanup_vector(apicd);
__vector_schedule_cleanup(apicd);
}
/*
@ -1150,7 +1208,7 @@ static void __init print_local_APIC(void *dummy)
u64 icr;
pr_debug("printing local APIC contents on CPU#%d/%d:\n",
smp_processor_id(), hard_smp_processor_id());
smp_processor_id(), read_apic_id());
v = apic_read(APIC_ID);
pr_info("... APIC ID: %08x (%01x)\n", v, read_apic_id());
v = apic_read(APIC_LVR);

23
arch/x86/kernel/apic/x2apic_cluster.c
View file

@ -83,16 +83,6 @@ x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
__x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_allbutself(int vector)
{
__x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_all(int vector)
{
__x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC);
}
static u32 x2apic_calc_apicid(unsigned int cpu)
{
return x86_cpu_to_logical_apicid[cpu];
@ -236,8 +226,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
.name = "cluster x2apic",
.probe = x2apic_cluster_probe,
.acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
.apic_id_valid = x2apic_apic_id_valid,
.apic_id_registered = x2apic_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = true,
@ -247,12 +235,11 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
.check_apicid_used = NULL,
.init_apic_ldr = init_x2apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = x2apic_phys_pkg_id,
.max_apic_id = UINT_MAX,
.x2apic_set_max_apicid = true,
.get_apic_id = x2apic_get_apic_id,
.set_apic_id = x2apic_set_apic_id,
@ -265,15 +252,11 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
.send_IPI_all = x2apic_send_IPI_all,
.send_IPI_self = x2apic_send_IPI_self,
.inquire_remote_apic = NULL,
.read = native_apic_msr_read,
.write = native_apic_msr_write,
.eoi_write = native_apic_msr_eoi_write,
.eoi = native_apic_msr_eoi,
.icr_read = native_x2apic_icr_read,
.icr_write = native_x2apic_icr_write,
.wait_icr_idle = native_x2apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
};
apic_driver(apic_x2apic_cluster);

76
arch/x86/kernel/apic/x2apic_phys.c
View file

@ -8,11 +8,13 @@
int x2apic_phys;
static struct apic apic_x2apic_phys;
static u32 x2apic_max_apicid __ro_after_init;
u32 x2apic_max_apicid __ro_after_init = UINT_MAX;
void __init x2apic_set_max_apicid(u32 apicid)
{
x2apic_max_apicid = apicid;
if (apic->x2apic_set_max_apicid)
apic->max_apic_id = apicid;
}
static int __init set_x2apic_phys_mode(char *arg)
@ -81,18 +83,34 @@ static void
__x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_allbutself(int vector)
static void __x2apic_send_IPI_shorthand(int vector, u32 which)
{
unsigned long cfg = __prepare_ICR(which, vector, 0);
/* x2apic MSRs are special and need a special fence: */
weak_wrmsr_fence();
native_x2apic_icr_write(cfg, 0);
}
void x2apic_send_IPI_allbutself(int vector)
{
__x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_all(int vector)
void x2apic_send_IPI_all(int vector)
{
__x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC);
}
static void init_x2apic_ldr(void)
void x2apic_send_IPI_self(int vector)
{
apic_write(APIC_SELF_IPI, vector);
}
void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
{
unsigned long cfg = __prepare_ICR(0, vector, dest);
native_x2apic_icr_write(cfg, apicid);
}
static int x2apic_phys_probe(void)
@ -106,35 +124,6 @@ static int x2apic_phys_probe(void)
return apic == &apic_x2apic_phys;
}
/* Common x2apic functions, also used by x2apic_cluster */
int x2apic_apic_id_valid(u32 apicid)
{
if (x2apic_max_apicid && apicid > x2apic_max_apicid)
return 0;
return 1;
}
int x2apic_apic_id_registered(void)
{
return 1;
}
void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
{
unsigned long cfg = __prepare_ICR(0, vector, dest);
native_x2apic_icr_write(cfg, apicid);
}
void __x2apic_send_IPI_shorthand(int vector, u32 which)
{
unsigned long cfg = __prepare_ICR(which, vector, 0);
/* x2apic MSRs are special and need a special fence: */
weak_wrmsr_fence();
native_x2apic_icr_write(cfg, 0);
}
unsigned int x2apic_get_apic_id(unsigned long id)
{
return id;
@ -150,33 +139,22 @@ int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
return initial_apicid >> index_msb;
}
void x2apic_send_IPI_self(int vector)
{
apic_write(APIC_SELF_IPI, vector);
}
static struct apic apic_x2apic_phys __ro_after_init = {
.name = "physical x2apic",
.probe = x2apic_phys_probe,
.acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
.apic_id_valid = x2apic_apic_id_valid,
.apic_id_registered = x2apic_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = init_x2apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = x2apic_phys_pkg_id,
.max_apic_id = UINT_MAX,
.x2apic_set_max_apicid = true,
.get_apic_id = x2apic_get_apic_id,
.set_apic_id = x2apic_set_apic_id,
@ -189,15 +167,11 @@ static struct apic apic_x2apic_phys __ro_after_init = {
.send_IPI_all = x2apic_send_IPI_all,
.send_IPI_self = x2apic_send_IPI_self,
.inquire_remote_apic = NULL,
.read = native_apic_msr_read,
.write = native_apic_msr_write,
.eoi_write = native_apic_msr_eoi_write,
.eoi = native_apic_msr_eoi,
.icr_read = native_x2apic_icr_read,
.icr_write = native_x2apic_icr_write,
.wait_icr_idle = native_x2apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
};
apic_driver(apic_x2apic_phys);

51
arch/x86/kernel/apic/x2apic_uv_x.c
View file

@ -25,6 +25,8 @@
#include <asm/uv/uv.h>
#include <asm/apic.h>
#include "local.h"
static enum uv_system_type uv_system_type;
static int uv_hubbed_system;
static int uv_hubless_system;
@ -777,30 +779,6 @@ static void uv_send_IPI_all(int vector)
uv_send_IPI_mask(cpu_online_mask, vector);
}
static int uv_apic_id_valid(u32 apicid)
{
return 1;
}
static int uv_apic_id_registered(void)
{
return 1;
}
static void uv_init_apic_ldr(void)
{
}
static u32 apic_uv_calc_apicid(unsigned int cpu)
{
return apic_default_calc_apicid(cpu);
}
static unsigned int x2apic_get_apic_id(unsigned long id)
{
return id;
}
static u32 set_apic_id(unsigned int id)
{
return id;
@ -816,11 +794,6 @@ static int uv_phys_pkg_id(int initial_apicid, int index_msb)
return uv_read_apic_id() >> index_msb;
}
static void uv_send_IPI_self(int vector)
{
apic_write(APIC_SELF_IPI, vector);
}
static int uv_probe(void)
{
return apic == &apic_x2apic_uv_x;
@ -831,45 +804,35 @@ static struct apic apic_x2apic_uv_x __ro_after_init = {
.name = "UV large system",
.probe = uv_probe,
.acpi_madt_oem_check = uv_acpi_madt_oem_check,
.apic_id_valid = uv_apic_id_valid,
.apic_id_registered = uv_apic_id_registered,
.delivery_mode = APIC_DELIVERY_MODE_FIXED,
.dest_mode_logical = false,
.disable_esr = 0,
.check_apicid_used = NULL,
.init_apic_ldr = uv_init_apic_ldr,
.ioapic_phys_id_map = NULL,
.setup_apic_routing = NULL,
.cpu_present_to_apicid = default_cpu_present_to_apicid,
.apicid_to_cpu_present = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.phys_pkg_id = uv_phys_pkg_id,
.max_apic_id = UINT_MAX,
.get_apic_id = x2apic_get_apic_id,
.set_apic_id = set_apic_id,
.calc_dest_apicid = apic_uv_calc_apicid,
.calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = uv_send_IPI_one,
.send_IPI_mask = uv_send_IPI_mask,
.send_IPI_mask_allbutself = uv_send_IPI_mask_allbutself,
.send_IPI_allbutself = uv_send_IPI_allbutself,
.send_IPI_all = uv_send_IPI_all,
.send_IPI_self = uv_send_IPI_self,
.send_IPI_self = x2apic_send_IPI_self,
.wakeup_secondary_cpu = uv_wakeup_secondary,
.inquire_remote_apic = NULL,
.read = native_apic_msr_read,
.write = native_apic_msr_write,
.eoi_write = native_apic_msr_eoi_write,
.eoi = native_apic_msr_eoi,
.icr_read = native_x2apic_icr_read,
.icr_write = native_x2apic_icr_write,
.wait_icr_idle = native_x2apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
};
#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_LENGTH 3
@ -1844,7 +1807,7 @@ static void __init uv_system_init_hub(void)
/* Initialize per CPU info: */
for_each_possible_cpu(cpu) {
int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
int apicid = per_cpu(x86_cpu_to_apicid, cpu);
unsigned short bid;
unsigned short pnode;

2
arch/x86/kernel/cpu/acrn.c
View file

@ -51,7 +51,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback)
* will block the interrupt whose vector is lower than
* HYPERVISOR_CALLBACK_VECTOR.
*/
ack_APIC_irq();
apic_eoi();
inc_irq_stat(irq_hv_callback_count);
if (acrn_intr_handler)

2
arch/x86/kernel/cpu/amd.c
View file

@ -1047,7 +1047,7 @@ static void init_amd(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_FSRS);
/* get apicid instead of initial apic id from cpuid */
c->apicid = hard_smp_processor_id();
c->apicid = read_apic_id();
/* K6s reports MCEs but don't actually have all the MSRs */
if (c->x86 < 6)

2
arch/x86/kernel/cpu/common.c
View file

@ -1958,7 +1958,7 @@ void enable_sep_cpu(void)
}
#endif
void __init identify_boot_cpu(void)
static __init void identify_boot_cpu(void)
{
identify_cpu(&boot_cpu_data);
if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))

3
arch/x86/kernel/cpu/hygon.c
View file

@ -8,6 +8,7 @@
*/
#include <linux/io.h>
#include <asm/apic.h>
#include <asm/cpu.h>
#include <asm/smp.h>
#include <asm/numa.h>
@ -300,7 +301,7 @@ static void init_hygon(struct cpuinfo_x86 *c)
set_cpu_cap(c, X86_FEATURE_REP_GOOD);
/* get apicid instead of initial apic id from cpuid */
c->apicid = hard_smp_processor_id();
c->apicid = read_apic_id();
/*
* XXX someone from Hygon needs to confirm this DTRT

2
arch/x86/kernel/cpu/mce/amd.c
View file

@ -759,7 +759,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error)
inc_irq_stat(irq_deferred_error_count);
deferred_error_int_vector();
trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR);
ack_APIC_irq();
apic_eoi();
}
/*

3
arch/x86/kernel/cpu/mce/inject.c
View file

@ -270,8 +270,7 @@ static void __maybe_unused raise_mce(struct mce *m)
mce_irq_ipi, NULL, 0);
preempt_enable();
} else if (m->inject_flags & MCJ_NMI_BROADCAST)
apic->send_IPI_mask(mce_inject_cpumask,
NMI_VECTOR);
__apic_send_IPI_mask(mce_inject_cpumask, NMI_VECTOR);
}
start = jiffies;
while (!cpumask_empty(mce_inject_cpumask)) {

2
arch/x86/kernel/cpu/mce/threshold.c
View file

@ -27,5 +27,5 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_threshold)
inc_irq_stat(irq_threshold_count);
mce_threshold_vector();
trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR);
ack_APIC_irq();
apic_eoi();
}

4
arch/x86/kernel/cpu/mshyperv.c
View file

@ -119,7 +119,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback)
vmbus_handler();
if (ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED)
ack_APIC_irq();
apic_eoi();
set_irq_regs(old_regs);
}
@ -147,7 +147,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0)
if (hv_stimer0_handler)
hv_stimer0_handler();
add_interrupt_randomness(HYPERV_STIMER0_VECTOR);
ack_APIC_irq();
apic_eoi();
set_irq_regs(old_regs);
}

21
arch/x86/kernel/devicetree.c
View file

@ -128,16 +128,15 @@ static void __init dtb_setup_hpet(void)
static void __init dtb_cpu_setup(void)
{
struct device_node *dn;
u32 apic_id, version;
u32 apic_id;
version = GET_APIC_VERSION(apic_read(APIC_LVR));
for_each_of_cpu_node(dn) {
apic_id = of_get_cpu_hwid(dn, 0);
if (apic_id == ~0U) {
pr_warn("%pOF: missing local APIC ID\n", dn);
continue;
}
generic_processor_info(apic_id, version);
generic_processor_info(apic_id);
}
}
@ -158,19 +157,15 @@ static void __init dtb_lapic_setup(void)
/* Did the boot loader setup the local APIC ? */
if (!boot_cpu_has(X86_FEATURE_APIC)) {
if (apic_force_enable(lapic_addr))
/* Try force enabling, which registers the APIC address */
if (!apic_force_enable(lapic_addr))
return;
} else {
register_lapic_address(lapic_addr);
}
smp_found_config = 1;
if (of_property_read_bool(dn, "intel,virtual-wire-mode")) {
pr_info("Virtual Wire compatibility mode.\n");
pic_mode = 0;
} else {
pr_info("IMCR and PIC compatibility mode.\n");
pic_mode = 1;
}
register_lapic_address(lapic_addr);
pic_mode = !of_property_read_bool(dn, "intel,virtual-wire-mode");
pr_info("%s compatibility mode.\n", pic_mode ? "IMCR and PIC" : "Virtual Wire");
}
#endif /* CONFIG_X86_LOCAL_APIC */

1
arch/x86/kernel/idt.c
View file

@ -131,7 +131,6 @@ static const __initconst struct idt_data apic_idts[] = {
INTG(RESCHEDULE_VECTOR, asm_sysvec_reschedule_ipi),
INTG(CALL_FUNCTION_VECTOR, asm_sysvec_call_function),
INTG(CALL_FUNCTION_SINGLE_VECTOR, asm_sysvec_call_function_single),
INTG(IRQ_MOVE_CLEANUP_VECTOR, asm_sysvec_irq_move_cleanup),
INTG(REBOOT_VECTOR, asm_sysvec_reboot),
#endif

14
arch/x86/kernel/irq.c
View file

@ -49,7 +49,7 @@ void ack_bad_irq(unsigned int irq)
* completely.
* But only ack when the APIC is enabled -AK
*/
ack_APIC_irq();
apic_eoi();
}
#define irq_stats(x) (&per_cpu(irq_stat, x))
@ -256,7 +256,7 @@ DEFINE_IDTENTRY_IRQ(common_interrupt)
if (likely(!IS_ERR_OR_NULL(desc))) {
handle_irq(desc, regs);
} else {
ack_APIC_irq();
apic_eoi();
if (desc == VECTOR_UNUSED) {
pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n",
@ -280,7 +280,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
{
struct pt_regs *old_regs = set_irq_regs(regs);
ack_APIC_irq();
apic_eoi();
trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
inc_irq_stat(x86_platform_ipis);
if (x86_platform_ipi_callback)
@ -310,7 +310,7 @@ EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
*/
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
{
ack_APIC_irq();
apic_eoi();
inc_irq_stat(kvm_posted_intr_ipis);
}
@ -319,7 +319,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
*/
DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
{
ack_APIC_irq();
apic_eoi();
inc_irq_stat(kvm_posted_intr_wakeup_ipis);
kvm_posted_intr_wakeup_handler();
}
@ -329,7 +329,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
*/
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
{
ack_APIC_irq();
apic_eoi();
inc_irq_stat(kvm_posted_intr_nested_ipis);
}
#endif
@ -401,6 +401,6 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_thermal)
inc_irq_stat(irq_thermal_count);
smp_thermal_vector();
trace_thermal_apic_exit(THERMAL_APIC_VECTOR);
ack_APIC_irq();
apic_eoi();
}
#endif

4
arch/x86/kernel/irq_work.c
View file

@ -16,7 +16,7 @@
#ifdef CONFIG_X86_LOCAL_APIC
DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work)
{
ack_APIC_irq();
apic_eoi();
trace_irq_work_entry(IRQ_WORK_VECTOR);
inc_irq_stat(apic_irq_work_irqs);
irq_work_run();
@ -28,7 +28,7 @@ void arch_irq_work_raise(void)
if (!arch_irq_work_has_interrupt())
return;
apic->send_IPI_self(IRQ_WORK_VECTOR);
__apic_send_IPI_self(IRQ_WORK_VECTOR);
apic_wait_icr_idle();
}
#endif

6
arch/x86/kernel/jailhouse.c
View file

@ -101,10 +101,8 @@ static void __init jailhouse_get_smp_config(unsigned int early)
register_lapic_address(0xfee00000);
for (cpu = 0; cpu < setup_data.v1.num_cpus; cpu++) {
generic_processor_info(setup_data.v1.cpu_ids[cpu],
boot_cpu_apic_version);
}
for (cpu = 0; cpu < setup_data.v1.num_cpus; cpu++)
generic_processor_info(setup_data.v1.cpu_ids[cpu]);
smp_found_config = 1;

14
arch/x86/kernel/kvm.c
View file

@ -291,7 +291,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt)
struct pt_regs *old_regs = set_irq_regs(regs);
u32 token;
ack_APIC_irq();
apic_eoi();
inc_irq_stat(irq_hv_callback_count);
@ -332,7 +332,7 @@ static void kvm_register_steal_time(void)
static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val)
static notrace __maybe_unused void kvm_guest_apic_eoi_write(void)
{
/**
* This relies on __test_and_clear_bit to modify the memory
@ -343,7 +343,7 @@ static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val)
*/
if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
return;
apic->native_eoi_write(APIC_EOI, APIC_EOI_ACK);
apic_native_eoi();
}
static void kvm_guest_cpu_init(void)
@ -622,10 +622,10 @@ late_initcall(setup_efi_kvm_sev_migration);
/*
* Set the IPI entry points
*/
static void kvm_setup_pv_ipi(void)
static __init void kvm_setup_pv_ipi(void)
{
apic->send_IPI_mask = kvm_send_ipi_mask;
apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself;
apic_update_callback(send_IPI_mask, kvm_send_ipi_mask);
apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself);
pr_info("setup PV IPIs\n");
}
@ -825,7 +825,7 @@ static void __init kvm_guest_init(void)
}
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
apic_set_eoi_write(kvm_guest_apic_eoi_write);
apic_update_callback(eoi, kvm_guest_apic_eoi_write);
if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) {
static_branch_enable(&kvm_async_pf_enabled);

20
arch/x86/kernel/mpparse.c
View file

@ -48,7 +48,6 @@ static int __init mpf_checksum(unsigned char *mp, int len)
static void __init MP_processor_info(struct mpc_cpu *m)
{
int apicid;
char *bootup_cpu = "";
if (!(m->cpuflag & CPU_ENABLED)) {
@ -56,15 +55,11 @@ static void __init MP_processor_info(struct mpc_cpu *m)
return;
}
apicid = m->apicid;
if (m->cpuflag & CPU_BOOTPROCESSOR) {
if (m->cpuflag & CPU_BOOTPROCESSOR)
bootup_cpu = " (Bootup-CPU)";
boot_cpu_physical_apicid = m->apicid;
}
pr_info("Processor #%d%s\n", m->apicid, bootup_cpu);
generic_processor_info(apicid, m->apicver);
generic_processor_info(m->apicid);
}
#ifdef CONFIG_X86_IO_APIC
@ -379,11 +374,6 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
int linttypes[2] = { mp_ExtINT, mp_NMI };
int i;
/*
* local APIC has default address
*/
mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
/*
* 2 CPUs, numbered 0 & 1.
*/
@ -525,10 +515,8 @@ void __init default_get_smp_config(unsigned int early)
*/
if (mpf->feature1) {
if (early) {
/*
* local APIC has default address
*/
mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
/* Local APIC has default address */
register_lapic_address(APIC_DEFAULT_PHYS_BASE);
goto out;
}

2
arch/x86/kernel/nmi_selftest.c
View file

@ -75,7 +75,7 @@ static void __init test_nmi_ipi(struct cpumask *mask)
/* sync above data before sending NMI */
wmb();
apic->send_IPI_mask(mask, NMI_VECTOR);
__apic_send_IPI_mask(mask, NMI_VECTOR);
/* Don't wait longer than a second */
timeout = USEC_PER_SEC;

7
arch/x86/kernel/setup.c
View file

@ -114,7 +114,6 @@ static struct resource bss_resource = {
#ifdef CONFIG_X86_32
/* CPU data as detected by the assembly code in head_32.S */
struct cpuinfo_x86 new_cpu_data;
unsigned int def_to_bigsmp;
struct apm_info apm_info;
EXPORT_SYMBOL(apm_info);
@ -1018,9 +1017,11 @@ void __init setup_arch(char **cmdline_p)
x86_report_nx();
apic_setup_apic_calls();
if (acpi_mps_check()) {
#ifdef CONFIG_X86_LOCAL_APIC
disable_apic = 1;
apic_is_disabled = true;
#endif
setup_clear_cpu_cap(X86_FEATURE_APIC);
}
@ -1253,7 +1254,7 @@ void __init setup_arch(char **cmdline_p)
map_vsyscall();
generic_apic_probe();
x86_32_probe_apic();
early_quirks();

10
arch/x86/kernel/setup_percpu.c
View file

@ -181,15 +181,9 @@ void __init setup_per_cpu_areas(void)
#ifdef CONFIG_X86_LOCAL_APIC
per_cpu(x86_cpu_to_apicid, cpu) =
early_per_cpu_map(x86_cpu_to_apicid, cpu);
per_cpu(x86_bios_cpu_apicid, cpu) =
early_per_cpu_map(x86_bios_cpu_apicid, cpu);
per_cpu(x86_cpu_to_acpiid, cpu) =
early_per_cpu_map(x86_cpu_to_acpiid, cpu);
#endif
#ifdef CONFIG_X86_32
per_cpu(x86_cpu_to_logical_apicid, cpu) =
early_per_cpu_map(x86_cpu_to_logical_apicid, cpu);
#endif
#ifdef CONFIG_NUMA
per_cpu(x86_cpu_to_node_map, cpu) =
early_per_cpu_map(x86_cpu_to_node_map, cpu);
@ -214,12 +208,8 @@ void __init setup_per_cpu_areas(void)
/* indicate the early static arrays will soon be gone */
#ifdef CONFIG_X86_LOCAL_APIC
early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
early_per_cpu_ptr(x86_cpu_to_acpiid) = NULL;
#endif
#ifdef CONFIG_X86_32
early_per_cpu_ptr(x86_cpu_to_logical_apicid) = NULL;
#endif
#ifdef CONFIG_NUMA
early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif

4
arch/x86/kernel/sev.c
View file

@ -1089,7 +1089,7 @@ static int wakeup_cpu_via_vmgexit(int apic_id, unsigned long start_ip)
return ret;
}
void snp_set_wakeup_secondary_cpu(void)
void __init snp_set_wakeup_secondary_cpu(void)
{
if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
return;
@ -1099,7 +1099,7 @@ void snp_set_wakeup_secondary_cpu(void)
* required method to start APs under SNP. If the hypervisor does
* not support AP creation, then no APs will be started.
*/
apic->wakeup_secondary_cpu = wakeup_cpu_via_vmgexit;
apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit);
}
int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)

10
arch/x86/kernel/smp.c
View file

@ -135,7 +135,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
*/
DEFINE_IDTENTRY_SYSVEC(sysvec_reboot)
{
ack_APIC_irq();
apic_eoi();
cpu_emergency_disable_virtualization();
stop_this_cpu(NULL);
}
@ -237,7 +237,7 @@ static void native_stop_other_cpus(int wait)
pr_emerg("Shutting down cpus with NMI\n");
for_each_cpu(cpu, &cpus_stop_mask)
apic->send_IPI(cpu, NMI_VECTOR);
__apic_send_IPI(cpu, NMI_VECTOR);
}
/*
* Don't wait longer than 10 ms if the caller didn't
@ -268,7 +268,7 @@ static void native_stop_other_cpus(int wait)
*/
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
{
ack_APIC_irq();
apic_eoi();
trace_reschedule_entry(RESCHEDULE_VECTOR);
inc_irq_stat(irq_resched_count);
scheduler_ipi();
@ -277,7 +277,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
{
ack_APIC_irq();
apic_eoi();
trace_call_function_entry(CALL_FUNCTION_VECTOR);
inc_irq_stat(irq_call_count);
generic_smp_call_function_interrupt();
@ -286,7 +286,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single)
{
ack_APIC_irq();
apic_eoi();
trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR);
inc_irq_stat(irq_call_count);
generic_smp_call_function_single_interrupt();

115
arch/x86/kernel/smpboot.c
View file

@ -761,44 +761,6 @@ static void impress_friends(void)
pr_debug("Before bogocount - setting activated=1\n");
}
void __inquire_remote_apic(int apicid)
{
unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
const char * const names[] = { "ID", "VERSION", "SPIV" };
int timeout;
u32 status;
pr_info("Inquiring remote APIC 0x%x...\n", apicid);
for (i = 0; i < ARRAY_SIZE(regs); i++) {
pr_info("... APIC 0x%x %s: ", apicid, names[i]);
/*
* Wait for idle.
*/
status = safe_apic_wait_icr_idle();
if (status)
pr_cont("a previous APIC delivery may have failed\n");
apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
timeout = 0;
do {
udelay(100);
status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
switch (status) {
case APIC_ICR_RR_VALID:
status = apic_read(APIC_RRR);
pr_cont("%08x\n", status);
break;
default:
pr_cont("failed\n");
}
}
}
/*
* The Multiprocessor Specification 1.4 (1997) example code suggests
* that there should be a 10ms delay between the BSP asserting INIT
@ -1089,9 +1051,8 @@ int native_kick_ap(unsigned int cpu, struct task_struct *tidle)
pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu);
if (apicid == BAD_APICID ||
!physid_isset(apicid, phys_cpu_present_map) ||
!apic->apic_id_valid(apicid)) {
if (apicid == BAD_APICID || !physid_isset(apicid, phys_cpu_present_map) ||
!apic_id_valid(apicid)) {
pr_err("%s: bad cpu %d\n", __func__, cpu);
return -EINVAL;
}
@ -1174,58 +1135,6 @@ static __init void disable_smp(void)
cpumask_set_cpu(0, topology_die_cpumask(0));
}
/*
* Various sanity checks.
*/
static void __init smp_sanity_check(void)
{
preempt_disable();
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
if (def_to_bigsmp && nr_cpu_ids > 8) {
unsigned int cpu;
unsigned nr;
pr_warn("More than 8 CPUs detected - skipping them\n"
"Use CONFIG_X86_BIGSMP\n");
nr = 0;
for_each_present_cpu(cpu) {
if (nr >= 8)
set_cpu_present(cpu, false);
nr++;
}
nr = 0;
for_each_possible_cpu(cpu) {
if (nr >= 8)
set_cpu_possible(cpu, false);
nr++;
}
set_nr_cpu_ids(8);
}
#endif
if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n",
hard_smp_processor_id());
physid_set(hard_smp_processor_id(), phys_cpu_present_map);
}
/*
* Should not be necessary because the MP table should list the boot
* CPU too, but we do it for the sake of robustness anyway.
*/
if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n",
boot_cpu_physical_apicid);
physid_set(hard_smp_processor_id(), phys_cpu_present_map);
}
preempt_enable();
}
static void __init smp_cpu_index_default(void)
{
int i;
@ -1285,8 +1194,6 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
smp_prepare_cpus_common();
smp_sanity_check();
switch (apic_intr_mode) {
case APIC_PIC:
case APIC_VIRTUAL_WIRE_NO_CONFIG:
@ -1422,24 +1329,6 @@ __init void prefill_possible_map(void)
{
int i, possible;
/* No boot processor was found in mptable or ACPI MADT */
if (!num_processors) {
if (boot_cpu_has(X86_FEATURE_APIC)) {
int apicid = boot_cpu_physical_apicid;
int cpu = hard_smp_processor_id();
pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu);
/* Make sure boot cpu is enumerated */
if (apic->cpu_present_to_apicid(0) == BAD_APICID &&
apic->apic_id_valid(apicid))
generic_processor_info(apicid, boot_cpu_apic_version);
}
if (!num_processors)
num_processors = 1;
}
i = setup_max_cpus ?: 1;
if (setup_possible_cpus == -1) {
possible = num_processors;

2
arch/x86/kernel/vsmp_64.c
View file

@ -129,7 +129,7 @@ static void __init vsmp_cap_cpus(void)
static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
{
return hard_smp_processor_id() >> index_msb;
return read_apic_id() >> index_msb;
}
static void vsmp_apic_post_init(void)

2
arch/x86/kvm/vmx/posted_intr.c
View file

@ -175,7 +175,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu)
* scheduled out).
*/
if (pi_test_on(&new))
apic->send_IPI_self(POSTED_INTR_WAKEUP_VECTOR);
__apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR);
local_irq_restore(flags);
}

2
arch/x86/kvm/vmx/vmx.c
View file

@ -4179,7 +4179,7 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
*/
if (vcpu != kvm_get_running_vcpu())
apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
__apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
return;
}
#endif

5
arch/x86/mm/srat.c
View file

@ -40,9 +40,8 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
return;
pxm = pa->proximity_domain;
apic_id = pa->apic_id;
if (!apic->apic_id_valid(apic_id)) {
printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n",
pxm, apic_id);
if (!apic_id_valid(apic_id)) {
pr_info("SRAT: PXM %u -> X2APIC 0x%04x ignored\n", pxm, apic_id);
return;
}
node = acpi_map_pxm_to_node(pxm);

2
arch/x86/pci/xen.c
View file

@ -517,7 +517,7 @@ int __init pci_xen_init(void)
#ifdef CONFIG_PCI_MSI
static void __init xen_hvm_msi_init(void)
{
if (!disable_apic) {
if (!apic_is_disabled) {
/*
* If hardware supports (x2)APIC virtualization (as indicated
* by hypervisor's leaf 4) then we don't need to use pirqs/

2
arch/x86/platform/uv/uv_irq.c
View file

@ -58,7 +58,7 @@ uv_set_irq_affinity(struct irq_data *data, const struct cpumask *mask,
ret = parent->chip->irq_set_affinity(parent, mask, force);
if (ret >= 0) {
uv_program_mmr(cfg, data->chip_data);
send_cleanup_vector(cfg);
vector_schedule_cleanup(cfg);
}
return ret;

2
arch/x86/platform/uv/uv_nmi.c
View file

@ -597,7 +597,7 @@ static void uv_nmi_nr_cpus_ping(void)
for_each_cpu(cpu, uv_nmi_cpu_mask)
uv_cpu_nmi_per(cpu).pinging = 1;
apic->send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI);
__apic_send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI);
}
/* Clean up flags for CPU's that ignored both NMI and ping */

85
arch/x86/xen/apic.c
View file

@ -81,6 +81,11 @@ static void xen_apic_write(u32 reg, u32 val)
WARN(1,"register: %x, value: %x\n", reg, val);
}
static void xen_apic_eoi(void)
{
WARN_ON_ONCE(1);
}
static u64 xen_apic_icr_read(void)
{
return 0;
@ -92,11 +97,6 @@ static void xen_apic_icr_write(u32 low, u32 id)
WARN_ON(1);
}
static u32 xen_safe_apic_wait_icr_idle(void)
{
return 0;
}
static int xen_apic_probe_pv(void)
{
if (xen_pv_domain())
@ -110,29 +110,11 @@ static int xen_madt_oem_check(char *oem_id, char *oem_table_id)
return xen_pv_domain();
}
static int xen_id_always_valid(u32 apicid)
{
return 1;
}
static int xen_id_always_registered(void)
{
return 1;
}
static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
{
return initial_apic_id >> index_msb;
}
static void xen_noop(void)
{
}
static void xen_silent_inquire(int apicid)
{
}
static int xen_cpu_present_to_apicid(int cpu)
{
if (cpu_present(cpu))
@ -141,68 +123,41 @@ static int xen_cpu_present_to_apicid(int cpu)
return BAD_APICID;
}
static struct apic xen_pv_apic = {
.name = "Xen PV",
.probe = xen_apic_probe_pv,
static struct apic xen_pv_apic __ro_after_init = {
.name = "Xen PV",
.probe = xen_apic_probe_pv,
.acpi_madt_oem_check = xen_madt_oem_check,
.apic_id_valid = xen_id_always_valid,
.apic_id_registered = xen_id_always_registered,
/* .delivery_mode and .dest_mode_logical not used by XENPV */
.disable_esr = 0,
.check_apicid_used = default_check_apicid_used, /* Used on 32-bit */
.init_apic_ldr = xen_noop, /* setup_local_APIC calls it */
.ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */
.setup_apic_routing = NULL,
.cpu_present_to_apicid = xen_cpu_present_to_apicid,
.apicid_to_cpu_present = physid_set_mask_of_physid, /* Used on 32-bit */
.check_phys_apicid_present = default_check_phys_apicid_present, /* smp_sanity_check needs it */
.phys_pkg_id = xen_phys_pkg_id, /* detect_ht */
.get_apic_id = xen_get_apic_id,
.set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */
.max_apic_id = UINT_MAX,
.get_apic_id = xen_get_apic_id,
.set_apic_id = xen_set_apic_id,
.calc_dest_apicid = apic_flat_calc_apicid,
#ifdef CONFIG_SMP
.send_IPI_mask = xen_send_IPI_mask,
.send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself,
.send_IPI_allbutself = xen_send_IPI_allbutself,
.send_IPI_all = xen_send_IPI_all,
.send_IPI_self = xen_send_IPI_self,
.send_IPI_mask = xen_send_IPI_mask,
.send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself,
.send_IPI_allbutself = xen_send_IPI_allbutself,
.send_IPI_all = xen_send_IPI_all,
.send_IPI_self = xen_send_IPI_self,
#endif
/* .wait_for_init_deassert- used by AP bootup - smp_callin which we don't use */
.inquire_remote_apic = xen_silent_inquire,
.read = xen_apic_read,
.write = xen_apic_write,
.eoi_write = xen_apic_write,
.eoi = xen_apic_eoi,
.icr_read = xen_apic_icr_read,
.icr_write = xen_apic_icr_write,
.wait_icr_idle = xen_noop,
.safe_wait_icr_idle = xen_safe_apic_wait_icr_idle,
.icr_read = xen_apic_icr_read,
.icr_write = xen_apic_icr_write,
};
apic_driver(xen_pv_apic);
static void __init xen_apic_check(void)
{
if (apic == &xen_pv_apic)
return;
pr_info("Switched APIC routing from %s to %s.\n", apic->name,
xen_pv_apic.name);
apic = &xen_pv_apic;
}
void __init xen_init_apic(void)
{
x86_apic_ops.io_apic_read = xen_io_apic_read;
/* On PV guests the APIC CPUID bit is disabled so none of the
* routines end up executing. */
if (!xen_initial_domain())
apic = &xen_pv_apic;
x86_platform.apic_post_init = xen_apic_check;
}
apic_driver(xen_pv_apic);

2
arch/x86/xen/enlighten_hvm.c
View file

@ -132,7 +132,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback)
struct pt_regs *old_regs = set_irq_regs(regs);
if (xen_percpu_upcall)
ack_APIC_irq();
apic_eoi();
inc_irq_stat(irq_hv_callback_count);

4
arch/x86/xen/enlighten_pv.c
View file

@ -1326,7 +1326,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
x86_init.resources.memory_setup = xen_memory_setup;
x86_init.irqs.intr_mode_select = x86_init_noop;
x86_init.irqs.intr_mode_init = x86_init_noop;
x86_init.irqs.intr_mode_init = x86_64_probe_apic;
x86_init.oem.arch_setup = xen_arch_setup;
x86_init.oem.banner = xen_banner;
x86_init.hyper.init_platform = xen_pv_init_platform;
@ -1366,12 +1366,10 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
xen_init_capabilities();
#ifdef CONFIG_X86_LOCAL_APIC
/*
* set up the basic apic ops.
*/
xen_init_apic();
#endif
machine_ops = xen_machine_ops;

5
arch/x86/xen/smp_pv.c
View file

@ -182,7 +182,8 @@ static void __init _get_smp_config(unsigned int early)
if (subtract)
set_nr_cpu_ids(nr_cpu_ids - subtract);
#endif
/* Pretend to be a proper enumerated system */
smp_found_config = 1;
}
static void __init xen_pv_smp_prepare_boot_cpu(void)
@ -210,7 +211,7 @@ static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus)
{
unsigned cpu;
if (skip_ioapic_setup) {
if (ioapic_is_disabled) {
char *m = (max_cpus == 0) ?
"The nosmp parameter is incompatible with Xen; " \
"use Xen dom0_max_vcpus=1 parameter" :

2
drivers/iommu/amd/iommu.c
View file

@ -3681,7 +3681,7 @@ static int amd_ir_set_affinity(struct irq_data *data,
* at the new destination. So, time to cleanup the previous
* vector allocation.
*/
send_cleanup_vector(cfg);
vector_schedule_cleanup(cfg);
return IRQ_SET_MASK_OK_DONE;
}

4
drivers/iommu/hyperv-iommu.c
View file

@ -51,7 +51,7 @@ static int hyperv_ir_set_affinity(struct irq_data *data,
if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
return ret;
send_cleanup_vector(cfg);
vector_schedule_cleanup(cfg);
return 0;
}
@ -257,7 +257,7 @@ static int hyperv_root_ir_set_affinity(struct irq_data *data,
if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
return ret;
send_cleanup_vector(cfg);
vector_schedule_cleanup(cfg);
return 0;
}

2
drivers/iommu/intel/irq_remapping.c
View file

@ -1176,7 +1176,7 @@ intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask,
* at the new destination. So, time to cleanup the previous
* vector allocation.
*/
send_cleanup_vector(cfg);
vector_schedule_cleanup(cfg);
return IRQ_SET_MASK_OK_DONE;
}

7
tools/arch/x86/include/asm/irq_vectors.h
View file

@ -35,13 +35,6 @@
*/
#define FIRST_EXTERNAL_VECTOR 0x20
/*
* Reserve the lowest usable vector (and hence lowest priority) 0x20 for
* triggering cleanup after irq migration. 0x21-0x2f will still be used
* for device interrupts.
*/
#define IRQ_MOVE_CLEANUP_VECTOR FIRST_EXTERNAL_VECTOR
#define IA32_SYSCALL_VECTOR 0x80
/*

2
tools/perf/trace/beauty/tracepoints/x86_irq_vectors.sh
View file

@ -12,7 +12,7 @@ x86_irq_vectors=${arch_x86_header_dir}/irq_vectors.h
# FIRST_EXTERNAL_VECTOR is not that useful, find what is its number
# and then replace whatever is using it and that is useful, which at
# the time of writing of this script was: IRQ_MOVE_CLEANUP_VECTOR.
# the time of writing of this script was: 0x20.
first_external_regex='^#define[[:space:]]+FIRST_EXTERNAL_VECTOR[[:space:]]+(0x[[:xdigit:]]+)$'
first_external_vector=$(grep -E ${first_external_regex} ${x86_irq_vectors} | sed -r "s/${first_external_regex}/\1/g")