From a539cc86a1cb688df24d9cff17d946a8c0b94b38 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Jun 2023 10:12:46 -0700 Subject: x86/vector: Rename send_cleanup_vector() to vector_schedule_cleanup() Rename send_cleanup_vector() to vector_schedule_cleanup() to prepare for replacing the vector cleanup IPI with a timer callback. Signed-off-by: Thomas Gleixner Signed-off-by: Xin Li Signed-off-by: Thomas Gleixner Reviewed-by: Steve Wahl Link: https://lore.kernel.org/r/20230621171248.6805-2-xin3.li@intel.com --- arch/x86/include/asm/hw_irq.h | 4 ++-- arch/x86/kernel/apic/vector.c | 8 ++++---- arch/x86/platform/uv/uv_irq.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index d465ece58151..551829884734 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -97,10 +97,10 @@ extern struct irq_cfg *irqd_cfg(struct irq_data *irq_data); extern void lock_vector_lock(void); extern void unlock_vector_lock(void); #ifdef CONFIG_SMP -extern void send_cleanup_vector(struct irq_cfg *); +extern void vector_schedule_cleanup(struct irq_cfg *); extern void irq_complete_move(struct irq_cfg *cfg); #else -static inline void send_cleanup_vector(struct irq_cfg *c) { } +static inline void vector_schedule_cleanup(struct irq_cfg *c) { } static inline void irq_complete_move(struct irq_cfg *c) { } #endif diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index c1efebd27e6c..aa370bd0d933 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -967,7 +967,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup) raw_spin_unlock(&vector_lock); } -static void __send_cleanup_vector(struct apic_chip_data *apicd) +static void __vector_schedule_cleanup(struct apic_chip_data *apicd) { unsigned int cpu; @@ -983,13 +983,13 @@ static void __send_cleanup_vector(struct apic_chip_data *apicd) raw_spin_unlock(&vector_lock); } -void send_cleanup_vector(struct irq_cfg *cfg) +void vector_schedule_cleanup(struct irq_cfg *cfg) { struct apic_chip_data *apicd; apicd = container_of(cfg, struct apic_chip_data, hw_irq_cfg); if (apicd->move_in_progress) - __send_cleanup_vector(apicd); + __vector_schedule_cleanup(apicd); } void irq_complete_move(struct irq_cfg *cfg) @@ -1007,7 +1007,7 @@ void irq_complete_move(struct irq_cfg *cfg) * on the same CPU. */ if (apicd->cpu == smp_processor_id()) - __send_cleanup_vector(apicd); + __vector_schedule_cleanup(apicd); } /* diff --git a/arch/x86/platform/uv/uv_irq.c b/arch/x86/platform/uv/uv_irq.c index ee21d6a36a80..4221259a5870 100644 --- a/arch/x86/platform/uv/uv_irq.c +++ b/arch/x86/platform/uv/uv_irq.c @@ -58,7 +58,7 @@ uv_set_irq_affinity(struct irq_data *data, const struct cpumask *mask, ret = parent->chip->irq_set_affinity(parent, mask, force); if (ret >= 0) { uv_program_mmr(cfg, data->chip_data); - send_cleanup_vector(cfg); + vector_schedule_cleanup(cfg); } return ret; -- cgit v1.2.3 From bdc1dad299bb1d38ce329de0dad0beadc76badf4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 21 Jun 2023 10:12:47 -0700 Subject: x86/vector: Replace IRQ_MOVE_CLEANUP_VECTOR with a timer callback The left overs of a moved interrupt are cleaned up once the interrupt is raised on the new target CPU. Keeping the vector valid on the original target CPU guarantees that there can't be an interrupt lost if the affinity change races with an concurrent interrupt from the device. This cleanup utilizes the lowest priority interrupt vector for this cleanup, which makes sure that in the unlikely case when the to be cleaned up interrupt is pending in the local APICs IRR the cleanup vector does not live lock. But there is no real reason to use an interrupt vector for cleaning up the leftovers of a moved interrupt. It's not a high performance operation. The only requirement is that it happens on the original target CPU. Convert it to use a timer instead and adjust the code accordingly. Signed-off-by: Thomas Gleixner Signed-off-by: Xin Li Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230621171248.6805-3-xin3.li@intel.com --- arch/x86/include/asm/idtentry.h | 1 - arch/x86/include/asm/irq_vectors.h | 7 --- arch/x86/kernel/apic/vector.c | 98 ++++++++++++++++++++++++++++++-------- arch/x86/kernel/idt.c | 1 - 4 files changed, 78 insertions(+), 29 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index b241af4ce9b4..cd5c10a74071 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -648,7 +648,6 @@ DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi); #ifdef CONFIG_SMP DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi); -DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR, sysvec_irq_move_cleanup); DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot); DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single); DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function); diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 43dcb9284208..3a19904c2db6 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -35,13 +35,6 @@ */ #define FIRST_EXTERNAL_VECTOR 0x20 -/* - * Reserve the lowest usable vector (and hence lowest priority) 0x20 for - * triggering cleanup after irq migration. 0x21-0x2f will still be used - * for device interrupts. - */ -#define IRQ_MOVE_CLEANUP_VECTOR FIRST_EXTERNAL_VECTOR - #define IA32_SYSCALL_VECTOR 0x80 /* diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index aa370bd0d933..01c359a66b04 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -44,7 +44,18 @@ static cpumask_var_t vector_searchmask; static struct irq_chip lapic_controller; static struct irq_matrix *vector_matrix; #ifdef CONFIG_SMP -static DEFINE_PER_CPU(struct hlist_head, cleanup_list); + +static void vector_cleanup_callback(struct timer_list *tmr); + +struct vector_cleanup { + struct hlist_head head; + struct timer_list timer; +}; + +static DEFINE_PER_CPU(struct vector_cleanup, vector_cleanup) = { + .head = HLIST_HEAD_INIT, + .timer = __TIMER_INITIALIZER(vector_cleanup_callback, TIMER_PINNED), +}; #endif void lock_vector_lock(void) @@ -841,10 +852,21 @@ void lapic_online(void) this_cpu_write(vector_irq[vector], __setup_vector_irq(vector)); } +static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr); + void lapic_offline(void) { + struct vector_cleanup *cl = this_cpu_ptr(&vector_cleanup); + lock_vector_lock(); + + /* In case the vector cleanup timer has not expired */ + __vector_cleanup(cl, false); + irq_matrix_offline(vector_matrix); + WARN_ON_ONCE(try_to_del_timer_sync(&cl->timer) < 0); + WARN_ON_ONCE(!hlist_empty(&cl->head)); + unlock_vector_lock(); } @@ -934,49 +956,85 @@ static void free_moved_vector(struct apic_chip_data *apicd) apicd->move_in_progress = 0; } -DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup) +static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr) { - struct hlist_head *clhead = this_cpu_ptr(&cleanup_list); struct apic_chip_data *apicd; struct hlist_node *tmp; + bool rearm = false; - ack_APIC_irq(); - /* Prevent vectors vanishing under us */ - raw_spin_lock(&vector_lock); + lockdep_assert_held(&vector_lock); - hlist_for_each_entry_safe(apicd, tmp, clhead, clist) { + hlist_for_each_entry_safe(apicd, tmp, &cl->head, clist) { unsigned int irr, vector = apicd->prev_vector; /* * Paranoia: Check if the vector that needs to be cleaned - * up is registered at the APICs IRR. If so, then this is - * not the best time to clean it up. Clean it up in the - * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR - * to this CPU. IRQ_MOVE_CLEANUP_VECTOR is the lowest - * priority external vector, so on return from this - * interrupt the device interrupt will happen first. + * up is registered at the APICs IRR. That's clearly a + * hardware issue if the vector arrived on the old target + * _after_ interrupts were disabled above. Keep @apicd + * on the list and schedule the timer again to give the CPU + * a chance to handle the pending interrupt. + * + * Do not check IRR when called from lapic_offline(), because + * fixup_irqs() was just called to scan IRR for set bits and + * forward them to new destination CPUs via IPIs. */ - irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); + irr = check_irr ? apic_read(APIC_IRR + (vector / 32 * 0x10)) : 0; if (irr & (1U << (vector % 32))) { - apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR); + pr_warn_once("Moved interrupt pending in old target APIC %u\n", apicd->irq); + rearm = true; continue; } free_moved_vector(apicd); } - raw_spin_unlock(&vector_lock); + /* + * Must happen under vector_lock to make the timer_pending() check + * in __vector_schedule_cleanup() race free against the rearm here. + */ + if (rearm) + mod_timer(&cl->timer, jiffies + 1); +} + +static void vector_cleanup_callback(struct timer_list *tmr) +{ + struct vector_cleanup *cl = container_of(tmr, typeof(*cl), timer); + + /* Prevent vectors vanishing under us */ + raw_spin_lock_irq(&vector_lock); + __vector_cleanup(cl, true); + raw_spin_unlock_irq(&vector_lock); } static void __vector_schedule_cleanup(struct apic_chip_data *apicd) { - unsigned int cpu; + unsigned int cpu = apicd->prev_cpu; raw_spin_lock(&vector_lock); apicd->move_in_progress = 0; - cpu = apicd->prev_cpu; if (cpu_online(cpu)) { - hlist_add_head(&apicd->clist, per_cpu_ptr(&cleanup_list, cpu)); - apic->send_IPI(cpu, IRQ_MOVE_CLEANUP_VECTOR); + struct vector_cleanup *cl = per_cpu_ptr(&vector_cleanup, cpu); + + hlist_add_head(&apicd->clist, &cl->head); + + /* + * The lockless timer_pending() check is safe here. If it + * returns true, then the callback will observe this new + * apic data in the hlist as everything is serialized by + * vector lock. + * + * If it returns false then the timer is either not armed + * or the other CPU executes the callback, which again + * would be blocked on vector lock. Rearming it in the + * latter case makes it fire for nothing. + * + * This is also safe against the callback rearming the timer + * because that's serialized via vector lock too. + */ + if (!timer_pending(&cl->timer)) { + cl->timer.expires = jiffies + 1; + add_timer_on(&cl->timer, cpu); + } } else { apicd->prev_vector = 0; } diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index a58c6bc1cd68..f3958262c725 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -131,7 +131,6 @@ static const __initconst struct idt_data apic_idts[] = { INTG(RESCHEDULE_VECTOR, asm_sysvec_reschedule_ipi), INTG(CALL_FUNCTION_VECTOR, asm_sysvec_call_function), INTG(CALL_FUNCTION_SINGLE_VECTOR, asm_sysvec_call_function_single), - INTG(IRQ_MOVE_CLEANUP_VECTOR, asm_sysvec_irq_move_cleanup), INTG(REBOOT_VECTOR, asm_sysvec_reboot), #endif -- cgit v1.2.3 From 3ba3fdfe2c3f28b9976f0c07eb7736080cb7d4a9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:39 -0700 Subject: x86/cpu: Make identify_boot_cpu() static It's not longer used outside the source file. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/processor.h | 1 - arch/x86/kernel/cpu/common.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index d46300e94f85..923ca0a4b550 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -190,7 +190,6 @@ static inline unsigned long long l1tf_pfn_limit(void) } extern void early_cpu_init(void); -extern void identify_boot_cpu(void); extern void identify_secondary_cpu(struct cpuinfo_x86 *); extern void print_cpu_info(struct cpuinfo_x86 *); void print_cpu_msr(struct cpuinfo_x86 *); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 0ba1067f4e5f..b156187afedb 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1937,7 +1937,7 @@ void enable_sep_cpu(void) } #endif -void __init identify_boot_cpu(void) +static __init void identify_boot_cpu(void) { identify_cpu(&boot_cpu_data); if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) -- cgit v1.2.3 From 13d88dcb1a1cdeaaea8362e1a9d9540e2b37ef76 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:39 -0700 Subject: x86/cpu: Remove unused physid_*() nonsense Tons of silly unused bitmap wrappers... Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/mpspec.h | 26 -------------------------- 1 file changed, 26 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index e90ac7e9ae2c..fa5e24a58595 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -87,13 +87,7 @@ struct physid_mask { typedef struct physid_mask physid_mask_t; #define physid_set(physid, map) set_bit(physid, (map).mask) -#define physid_clear(physid, map) clear_bit(physid, (map).mask) #define physid_isset(physid, map) test_bit(physid, (map).mask) -#define physid_test_and_set(physid, map) \ - test_and_set_bit(physid, (map).mask) - -#define physids_and(dst, src1, src2) \ - bitmap_and((dst).mask, (src1).mask, (src2).mask, MAX_LOCAL_APIC) #define physids_or(dst, src1, src2) \ bitmap_or((dst).mask, (src1).mask, (src2).mask, MAX_LOCAL_APIC) @@ -101,29 +95,9 @@ typedef struct physid_mask physid_mask_t; #define physids_clear(map) \ bitmap_zero((map).mask, MAX_LOCAL_APIC) -#define physids_complement(dst, src) \ - bitmap_complement((dst).mask, (src).mask, MAX_LOCAL_APIC) - #define physids_empty(map) \ bitmap_empty((map).mask, MAX_LOCAL_APIC) -#define physids_equal(map1, map2) \ - bitmap_equal((map1).mask, (map2).mask, MAX_LOCAL_APIC) - -#define physids_weight(map) \ - bitmap_weight((map).mask, MAX_LOCAL_APIC) - -#define physids_shift_right(d, s, n) \ - bitmap_shift_right((d).mask, (s).mask, n, MAX_LOCAL_APIC) - -#define physids_shift_left(d, s, n) \ - bitmap_shift_left((d).mask, (s).mask, n, MAX_LOCAL_APIC) - -static inline unsigned long physids_coerce(physid_mask_t *map) -{ - return map->mask[0]; -} - static inline void physids_promote(unsigned long physids, physid_mask_t *map) { physids_clear(*map); -- cgit v1.2.3 From 49062454a3eb4a27add7a3df57e7a34266f6a760 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:40 -0700 Subject: x86/apic: Rename disable_apic It reflects a state and not a command. Make it bool while at it. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 4 ++-- arch/x86/kernel/apic/apic.c | 22 +++++++++++----------- arch/x86/kernel/apic/apic_noop.c | 6 +++--- arch/x86/kernel/apic/msi.c | 2 +- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/setup.c | 2 +- arch/x86/pci/xen.c | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 98c32aa5963a..4b1110a28a18 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -52,7 +52,7 @@ static inline void generic_apic_probe(void) extern int apic_verbosity; extern int local_apic_timer_c2_ok; -extern int disable_apic; +extern bool apic_is_disabled; extern unsigned int lapic_timer_period; extern int cpuid_to_apicid[]; @@ -90,7 +90,7 @@ static inline void default_inquire_remote_apic(int apicid) */ static inline bool apic_from_smp_config(void) { - return smp_found_config && !disable_apic; + return smp_found_config && !apic_is_disabled; } /* diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index af49e24b46a4..a9dd9038f59c 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -180,7 +180,7 @@ __setup("apicpmtimer", setup_apicpmtimer); #endif unsigned long mp_lapic_addr __ro_after_init; -int disable_apic __ro_after_init; +bool apic_is_disabled __ro_after_init; /* Disable local APIC timer from the kernel commandline or via dmi quirk */ static int disable_apic_timer __initdata; /* Local APIC timer works in C2 */ @@ -810,7 +810,7 @@ bool __init apic_needs_pit(void) return true; /* Is there an APIC at all or is it disabled? */ - if (!boot_cpu_has(X86_FEATURE_APIC) || disable_apic) + if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled) return true; /* @@ -1299,7 +1299,7 @@ enum apic_intr_mode_id apic_intr_mode __ro_after_init; static int __init __apic_intr_mode_select(void) { /* Check kernel option */ - if (disable_apic) { + if (apic_is_disabled) { pr_info("APIC disabled via kernel command line\n"); return APIC_PIC; } @@ -1308,7 +1308,7 @@ static int __init __apic_intr_mode_select(void) #ifdef CONFIG_X86_64 /* On 64-bit, the APIC must be integrated, Check local APIC only */ if (!boot_cpu_has(X86_FEATURE_APIC)) { - disable_apic = 1; + apic_is_disabled = true; pr_info("APIC disabled by BIOS\n"); return APIC_PIC; } @@ -1317,14 +1317,14 @@ static int __init __apic_intr_mode_select(void) /* Neither 82489DX nor integrated APIC ? */ if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) { - disable_apic = 1; + apic_is_disabled = true; return APIC_PIC; } /* If the BIOS pretends there is an integrated APIC ? */ if (!boot_cpu_has(X86_FEATURE_APIC) && APIC_INTEGRATED(boot_cpu_apic_version)) { - disable_apic = 1; + apic_is_disabled = true; pr_err(FW_BUG "Local APIC %d not detected, force emulation\n", boot_cpu_physical_apicid); return APIC_PIC; @@ -1567,7 +1567,7 @@ static void setup_local_APIC(void) int cpu = smp_processor_id(); unsigned int value; - if (disable_apic) { + if (apic_is_disabled) { disable_ioapic_support(); return; } @@ -1943,7 +1943,7 @@ void __init check_x2apic(void) pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n"); pr_err("Disabling APIC, expect reduced performance and functionality.\n"); - disable_apic = 1; + apic_is_disabled = true; setup_clear_cpu_cap(X86_FEATURE_APIC); } @@ -2037,7 +2037,7 @@ int __init apic_force_enable(unsigned long addr) { u32 h, l; - if (disable_apic) + if (apic_is_disabled) return -1; /* @@ -2064,7 +2064,7 @@ int __init apic_force_enable(unsigned long addr) static int __init detect_init_APIC(void) { /* Disabled by kernel option? */ - if (disable_apic) + if (apic_is_disabled) return -1; switch (boot_cpu_data.x86_vendor) { @@ -2919,7 +2919,7 @@ int apic_is_clustered_box(void) */ static int __init setup_disableapic(char *arg) { - disable_apic = 1; + apic_is_disabled = true; setup_clear_cpu_cap(X86_FEATURE_APIC); return 0; } diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index fe78319e0f7a..f310dc228479 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -71,13 +71,13 @@ static int noop_apic_id_registered(void) static u32 noop_apic_read(u32 reg) { - WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); + WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled); return 0; } -static void noop_apic_write(u32 reg, u32 v) +static void noop_apic_write(u32 reg, u32 val) { - WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); + WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled); } #ifdef CONFIG_X86_32 diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index 35d5b8fb18ef..6b6b711678fe 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -269,7 +269,7 @@ static const struct msi_parent_ops x86_vector_msi_parent_ops = { struct irq_domain * __init native_create_pci_msi_domain(void) { - if (disable_apic) + if (apic_is_disabled) return NULL; x86_vector_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 01c359a66b04..04655b7e3107 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -547,7 +547,7 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, struct irq_data *irqd; int i, err, node; - if (disable_apic) + if (apic_is_disabled) return -ENXIO; /* diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index fd975a4a5200..a5ef400928da 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1020,7 +1020,7 @@ void __init setup_arch(char **cmdline_p) if (acpi_mps_check()) { #ifdef CONFIG_X86_LOCAL_APIC - disable_apic = 1; + apic_is_disabled = true; #endif setup_clear_cpu_cap(X86_FEATURE_APIC); } diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 014c508e914d..652cd53e77f6 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -517,7 +517,7 @@ int __init pci_xen_init(void) #ifdef CONFIG_PCI_MSI static void __init xen_hvm_msi_init(void) { - if (!disable_apic) { + if (!apic_is_disabled) { /* * If hardware supports (x2)APIC virtualization (as indicated * by hypervisor's leaf 4) then we don't need to use pirqs/ -- cgit v1.2.3 From ecf600f8942e04c9bf5f7046e096577eebaf6406 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:41 -0700 Subject: x86/apic/ioapic: Rename skip_ioapic_setup Another variable name which is confusing at best. Convert to bool. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/io_apic.h | 7 ++++--- arch/x86/kernel/acpi/boot.c | 2 +- arch/x86/kernel/apic/apic.c | 12 ++++++------ arch/x86/kernel/apic/io_apic.c | 12 ++++++------ arch/x86/xen/smp_pv.c | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h index 437aa8d00e53..51c782600e02 100644 --- a/arch/x86/include/asm/io_apic.h +++ b/arch/x86/include/asm/io_apic.h @@ -109,8 +109,8 @@ extern int mp_irq_entries; /* MP IRQ source entries */ extern struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES]; -/* 1 if "noapic" boot option passed */ -extern int skip_ioapic_setup; +/* True if "noapic" boot option passed */ +extern bool ioapic_is_disabled; /* 1 if "noapic" boot option passed */ extern int noioapicquirk; @@ -129,7 +129,7 @@ extern unsigned long io_apic_irqs; * assignment of PCI IRQ's. */ #define io_apic_assign_pci_irqs \ - (mp_irq_entries && !skip_ioapic_setup && io_apic_irqs) + (mp_irq_entries && !ioapic_is_disabled && io_apic_irqs) struct irq_cfg; extern void ioapic_insert_resources(void); @@ -179,6 +179,7 @@ extern void print_IO_APICs(void); #define IO_APIC_IRQ(x) 0 #define io_apic_assign_pci_irqs 0 #define setup_ioapic_ids_from_mpc x86_init_noop +#define nr_ioapics (0) static inline void ioapic_insert_resources(void) { } static inline int arch_early_ioapic_init(void) { return 0; } static inline void print_IO_APICs(void) {} diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 21b542a6866c..38a529830829 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -1275,7 +1275,7 @@ static int __init acpi_parse_madt_ioapic_entries(void) /* * if "noapic" boot option, don't look for IO-APICs */ - if (skip_ioapic_setup) { + if (ioapic_is_disabled) { pr_info("Skipping IOAPIC probe due to 'noapic' option.\n"); return -ENODEV; } diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index a9dd9038f59c..cffec1fe8486 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1691,7 +1691,7 @@ static void setup_local_APIC(void) * TODO: set up through-local-APIC from through-I/O-APIC? --macro */ value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; - if (!cpu && (pic_mode || !value || skip_ioapic_setup)) { + if (!cpu && (pic_mode || !value || ioapic_is_disabled)) { value = APIC_DM_EXTINT; apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu); } else { @@ -1956,7 +1956,7 @@ void __init enable_IR_x2apic(void) unsigned long flags; int ret, ir_stat; - if (skip_ioapic_setup) { + if (ioapic_is_disabled) { pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n"); return; } @@ -2956,11 +2956,11 @@ early_param("nolapic_timer", parse_nolapic_timer); static int __init apic_set_verbosity(char *arg) { if (!arg) { -#ifdef CONFIG_X86_64 - skip_ioapic_setup = 0; + if (IS_ENABLED(CONFIG_X86_32)) + return -EINVAL; + + ioapic_is_disabled = false; return 0; -#endif - return -EINVAL; } if (strcmp("debug", arg) == 0) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 4241dc243aa8..48975d53e845 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -178,7 +178,7 @@ int mp_bus_id_to_type[MAX_MP_BUSSES]; DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); -int skip_ioapic_setup; +bool ioapic_is_disabled __ro_after_init; /** * disable_ioapic_support() - disables ioapic support at runtime @@ -189,7 +189,7 @@ void disable_ioapic_support(void) noioapicquirk = 1; noioapicreroute = -1; #endif - skip_ioapic_setup = 1; + ioapic_is_disabled = true; } static int __init parse_noapic(char *str) @@ -831,7 +831,7 @@ static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity) { int ioapic, pin, idx; - if (skip_ioapic_setup) + if (ioapic_is_disabled) return -1; ioapic = mp_find_ioapic(gsi); @@ -1366,7 +1366,7 @@ void __init enable_IO_APIC(void) int i8259_apic, i8259_pin; int apic, pin; - if (skip_ioapic_setup) + if (ioapic_is_disabled) nr_ioapics = 0; if (!nr_legacy_irqs() || !nr_ioapics) @@ -2399,7 +2399,7 @@ void __init setup_IO_APIC(void) { int ioapic; - if (skip_ioapic_setup || !nr_ioapics) + if (ioapic_is_disabled || !nr_ioapics) return; io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL; @@ -2715,7 +2715,7 @@ void __init io_apic_init_mappings(void) "address found in MPTABLE, " "disabling IO/APIC support!\n"); smp_found_config = 0; - skip_ioapic_setup = 1; + ioapic_is_disabled = true; goto fake_ioapic_page; } #endif diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index cef78b8c89f4..c6b42c66c60c 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -210,7 +210,7 @@ static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus) { unsigned cpu; - if (skip_ioapic_setup) { + if (ioapic_is_disabled) { char *m = (max_cpus == 0) ? "The nosmp parameter is incompatible with Xen; " \ "use Xen dom0_max_vcpus=1 parameter" : -- cgit v1.2.3 From d23c977fb0785bd32dd54f0a3d24e003942b00a6 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:42 -0700 Subject: x86/apic: Remove pointless x86_bios_cpu_apicid It's a useless copy of x86_cpu_to_apicid. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 -- arch/x86/include/asm/smp.h | 1 - arch/x86/kernel/apic/apic.c | 5 +---- arch/x86/kernel/apic/apic_common.c | 2 +- arch/x86/kernel/apic/bigsmp_32.c | 2 +- arch/x86/kernel/apic/probe_64.c | 4 +--- arch/x86/kernel/setup_percpu.c | 3 --- 7 files changed, 4 insertions(+), 15 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 4b1110a28a18..36f0be7c005c 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -477,8 +477,6 @@ extern void generic_bigsmp_probe(void); #define APIC_DFR_VALUE (APIC_DFR_FLAT) -DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); - extern struct apic apic_noop; static inline unsigned int read_apic_id(void) diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 600cf25dbfc6..1db6faa59ad1 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -22,7 +22,6 @@ DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid); -DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid); #endif diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index cffec1fe8486..8af0ec8a393b 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -108,10 +108,8 @@ unsigned long apic_mmio_base __ro_after_init; * Map cpu index to physical APIC ID */ DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); -DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX); EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); -EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); #ifdef CONFIG_X86_32 @@ -2511,7 +2509,7 @@ int generic_processor_info(int apicid, int version) if (apicid == boot_cpu_physical_apicid) { /* - * x86_bios_cpu_apicid is required to have processors listed + * x86_cpu_to_apicid is required to have processors listed * in same order as logical cpu numbers. Hence the first * entry is BSP, and so on. * boot_cpu_init() already hold bit 0 in cpu_present_mask @@ -2548,7 +2546,6 @@ int generic_processor_info(int apicid, int version) #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; - early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid; #endif #ifdef CONFIG_X86_32 early_per_cpu(x86_cpu_to_logical_apicid, cpu) = diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index 02b4839478b1..e892506552c4 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -29,7 +29,7 @@ void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap) int default_cpu_present_to_apicid(int mps_cpu) { if (mps_cpu < nr_cpu_ids && cpu_present(mps_cpu)) - return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu); + return (int)per_cpu(x86_cpu_to_apicid, mps_cpu); else return BAD_APICID; } diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 77555f66c14d..628b008dbf06 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -52,7 +52,7 @@ static void bigsmp_setup_apic_routing(void) static int bigsmp_cpu_present_to_apicid(int mps_cpu) { if (mps_cpu < nr_cpu_ids) - return (int) per_cpu(x86_bios_cpu_apicid, mps_cpu); + return (int) per_cpu(x86_cpu_to_apicid, mps_cpu); return BAD_APICID; } diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c index c46720f185c0..3c0c8d32be10 100644 --- a/arch/x86/kernel/apic/probe_64.c +++ b/arch/x86/kernel/apic/probe_64.c @@ -13,9 +13,7 @@ #include "local.h" -/* - * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode. - */ +/* Select the appropriate APIC driver */ void __init default_setup_apic_routing(void) { struct apic **drv; diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index c242dc47e9cb..f6767d255c32 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -181,8 +181,6 @@ void __init setup_per_cpu_areas(void) #ifdef CONFIG_X86_LOCAL_APIC per_cpu(x86_cpu_to_apicid, cpu) = early_per_cpu_map(x86_cpu_to_apicid, cpu); - per_cpu(x86_bios_cpu_apicid, cpu) = - early_per_cpu_map(x86_bios_cpu_apicid, cpu); per_cpu(x86_cpu_to_acpiid, cpu) = early_per_cpu_map(x86_cpu_to_acpiid, cpu); #endif @@ -214,7 +212,6 @@ void __init setup_per_cpu_areas(void) /* indicate the early static arrays will soon be gone */ #ifdef CONFIG_X86_LOCAL_APIC early_per_cpu_ptr(x86_cpu_to_apicid) = NULL; - early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL; early_per_cpu_ptr(x86_cpu_to_acpiid) = NULL; #endif #ifdef CONFIG_X86_32 -- cgit v1.2.3 From a6625b473b943b8c624fde3f2b444781114942d3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:43 -0700 Subject: x86/apic: Get rid of hard_smp_processor_id() No point in having a wrapper around read_apic_id(). Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 6 +++++- arch/x86/include/asm/smp.h | 7 ------- arch/x86/kernel/apic/apic.c | 5 ----- arch/x86/kernel/apic/io_apic.c | 2 +- arch/x86/kernel/apic/ipi.c | 2 +- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/hygon.c | 3 ++- arch/x86/kernel/smpboot.c | 10 +++++----- arch/x86/kernel/vsmp_64.c | 2 +- 10 files changed, 17 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 36f0be7c005c..b59104085b50 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -503,7 +503,11 @@ extern void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *r extern int default_cpu_present_to_apicid(int mps_cpu); extern int default_check_phys_apicid_present(int phys_apicid); -#endif /* CONFIG_X86_LOCAL_APIC */ +#else /* CONFIG_X86_LOCAL_APIC */ + +static inline unsigned int read_apic_id(void) { return 0; } + +#endif /* !CONFIG_X86_LOCAL_APIC */ #ifdef CONFIG_SMP void apic_smt_update(void); diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 1db6faa59ad1..8d96d9a6962d 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -185,13 +185,6 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu) extern unsigned disabled_cpus; -#ifdef CONFIG_X86_LOCAL_APIC -extern int hard_smp_processor_id(void); - -#else /* CONFIG_X86_LOCAL_APIC */ -#define hard_smp_processor_id() 0 -#endif /* CONFIG_X86_LOCAL_APIC */ - #ifdef CONFIG_DEBUG_NMI_SELFTEST extern void nmi_selftest(void); #else diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 8af0ec8a393b..ee64f8f4e0e9 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2562,11 +2562,6 @@ int generic_processor_info(int apicid, int version) return cpu; } -int hard_smp_processor_id(void) -{ - return read_apic_id(); -} - void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, bool dmar) { diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 48975d53e845..a8b329aab672 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2095,7 +2095,7 @@ static inline void __init unlock_ExtINT_logic(void) entry0 = ioapic_read_entry(apic, pin); clear_IO_APIC_pin(apic, pin); - apic_id = hard_smp_processor_id(); + apic_id = read_apic_id(); memset(&entry1, 0, sizeof(entry1)); entry1.dest_mode_logical = true; diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 2a6509e8c840..29273e0151fc 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -320,7 +320,7 @@ int safe_smp_processor_id(void) if (!boot_cpu_has(X86_FEATURE_APIC)) return 0; - apicid = hard_smp_processor_id(); + apicid = read_apic_id(); if (apicid == BAD_APICID) return 0; diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 04655b7e3107..71feae718a87 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -1208,7 +1208,7 @@ static void __init print_local_APIC(void *dummy) u64 icr; pr_debug("printing local APIC contents on CPU#%d/%d:\n", - smp_processor_id(), hard_smp_processor_id()); + smp_processor_id(), read_apic_id()); v = apic_read(APIC_ID); pr_info("... APIC ID: %08x (%01x)\n", v, read_apic_id()); v = apic_read(APIC_LVR); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 26ad7ca423e7..5861eeab8c30 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1042,7 +1042,7 @@ static void init_amd(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_FSRS); /* get apicid instead of initial apic id from cpuid */ - c->apicid = hard_smp_processor_id(); + c->apicid = read_apic_id(); /* K6s reports MCEs but don't actually have all the MSRs */ if (c->x86 < 6) diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 5a2962c492d3..defdc594be14 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -300,7 +301,7 @@ static void init_hygon(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_REP_GOOD); /* get apicid instead of initial apic id from cpuid */ - c->apicid = hard_smp_processor_id(); + c->apicid = read_apic_id(); /* * XXX someone from Hygon needs to confirm this DTRT diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index e1aa2cd7734b..8229f41d1ec1 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1220,11 +1220,11 @@ static void __init smp_sanity_check(void) } #endif - if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) { + if (!physid_isset(read_apic_id(), phys_cpu_present_map)) { pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n", - hard_smp_processor_id()); + read_apic_id()); - physid_set(hard_smp_processor_id(), phys_cpu_present_map); + physid_set(read_apic_id(), phys_cpu_present_map); } /* @@ -1234,7 +1234,7 @@ static void __init smp_sanity_check(void) if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) { pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n", boot_cpu_physical_apicid); - physid_set(hard_smp_processor_id(), phys_cpu_present_map); + physid_set(read_apic_id(), phys_cpu_present_map); } preempt_enable(); } @@ -1439,7 +1439,7 @@ __init void prefill_possible_map(void) if (!num_processors) { if (boot_cpu_has(X86_FEATURE_APIC)) { int apicid = boot_cpu_physical_apicid; - int cpu = hard_smp_processor_id(); + int cpu = read_apic_id(); pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu); diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c index 796cfaa46bfa..65e96b76c423 100644 --- a/arch/x86/kernel/vsmp_64.c +++ b/arch/x86/kernel/vsmp_64.c @@ -129,7 +129,7 @@ static void __init vsmp_cap_cpus(void) static int apicid_phys_pkg_id(int initial_apic_id, int index_msb) { - return hard_smp_processor_id() >> index_msb; + return read_apic_id() >> index_msb; } static void vsmp_apic_post_init(void) -- cgit v1.2.3 From b3bc5dd99462218602145581a1907c4143e9d015 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:43 -0700 Subject: x86/apic: Remove unused max_physical_apicid max_physical_apicid is assigned but never read. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index ee64f8f4e0e9..d6b505942c7f 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -73,11 +73,6 @@ EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); u8 boot_cpu_apic_version __ro_after_init; -/* - * The highest APIC ID seen during enumeration. - */ -static unsigned int max_physical_apicid; - /* * Bitmask of physically existing CPUs: */ @@ -2541,9 +2536,6 @@ int generic_processor_info(int apicid, int version) boot_cpu_apic_version, cpu, version); } - if (apicid > max_physical_apicid) - max_physical_apicid = apicid; - #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; #endif -- cgit v1.2.3 From 1d90c9f7313011b67643228810435038d20b0744 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:44 -0700 Subject: x86/apic: Nuke unused apic::inquire_remote_apic() Put it to the other historical leftovers. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 16 --------------- arch/x86/kernel/apic/apic_flat_64.c | 4 ---- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 2 -- arch/x86/kernel/apic/bigsmp_32.c | 2 -- arch/x86/kernel/apic/probe_32.c | 2 -- arch/x86/kernel/apic/x2apic_cluster.c | 2 -- arch/x86/kernel/apic/x2apic_phys.c | 2 -- arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/kernel/smpboot.c | 38 ----------------------------------- arch/x86/xen/apic.c | 7 ------- 11 files changed, 78 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index b59104085b50..47bedb88c938 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -66,20 +66,6 @@ enum apic_intr_mode_id { APIC_SYMMETRIC_IO_NO_ROUTING }; -#ifdef CONFIG_SMP -extern void __inquire_remote_apic(int apicid); -#else /* CONFIG_SMP */ -static inline void __inquire_remote_apic(int apicid) -{ -} -#endif /* CONFIG_SMP */ - -static inline void default_inquire_remote_apic(int apicid) -{ - if (apic_verbosity >= APIC_DEBUG) - __inquire_remote_apic(apicid); -} - /* * With 82489DX we can't rely on apic feature bit * retrieved via cpuid but still have to deal with @@ -330,8 +316,6 @@ struct apic { /* wakeup secondary CPU using 64-bit wakeup point */ int (*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip); - void (*inquire_remote_apic)(int apicid); - #ifdef CONFIG_X86_32 /* * Called very early during boot from get_smp_config(). It should diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 8f72b4351c9f..35cd1cb39fae 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -139,8 +139,6 @@ static struct apic apic_flat __ro_after_init = { .send_IPI_all = default_send_IPI_all, .send_IPI_self = default_send_IPI_self, - .inquire_remote_apic = default_inquire_remote_apic, - .read = native_apic_mem_read, .write = native_apic_mem_write, .eoi_write = native_apic_mem_write, @@ -230,8 +228,6 @@ static struct apic apic_physflat __ro_after_init = { .send_IPI_all = default_send_IPI_all, .send_IPI_self = default_send_IPI_self, - .inquire_remote_apic = default_inquire_remote_apic, - .read = native_apic_mem_read, .write = native_apic_mem_write, .eoi_write = native_apic_mem_write, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index f310dc228479..90bf11e6827c 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -125,8 +125,6 @@ struct apic apic_noop __ro_after_init = { .wakeup_secondary_cpu = noop_wakeup_secondary_cpu, - .inquire_remote_apic = NULL, - .read = noop_apic_read, .write = noop_apic_write, .eoi_write = noop_apic_write, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index a54d817eb4b6..b6bbd7d2c4bc 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -273,7 +273,6 @@ static const struct apic apic_numachip1 __refconst = { .send_IPI_self = numachip_send_IPI_self, .wakeup_secondary_cpu = numachip_wakeup_secondary, - .inquire_remote_apic = NULL, /* REMRD not supported */ .read = native_apic_mem_read, .write = native_apic_mem_write, @@ -320,7 +319,6 @@ static const struct apic apic_numachip2 __refconst = { .send_IPI_self = numachip_send_IPI_self, .wakeup_secondary_cpu = numachip_wakeup_secondary, - .inquire_remote_apic = NULL, /* REMRD not supported */ .read = native_apic_mem_read, .write = native_apic_mem_write, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 628b008dbf06..9e254dc94e55 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -153,8 +153,6 @@ static struct apic apic_bigsmp __ro_after_init = { .send_IPI_all = bigsmp_send_IPI_all, .send_IPI_self = default_send_IPI_self, - .inquire_remote_apic = default_inquire_remote_apic, - .read = native_apic_mem_read, .write = native_apic_mem_write, .eoi_write = native_apic_mem_write, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index a61f642b1b90..8d1e456a77b8 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -95,8 +95,6 @@ static struct apic apic_default __ro_after_init = { .send_IPI_all = default_send_IPI_all, .send_IPI_self = default_send_IPI_self, - .inquire_remote_apic = default_inquire_remote_apic, - .read = native_apic_mem_read, .write = native_apic_mem_write, .eoi_write = native_apic_mem_write, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index b2b2b7f3e03f..ce1021c0191d 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -265,8 +265,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .send_IPI_all = x2apic_send_IPI_all, .send_IPI_self = x2apic_send_IPI_self, - .inquire_remote_apic = NULL, - .read = native_apic_msr_read, .write = native_apic_msr_write, .eoi_write = native_apic_msr_eoi_write, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 896bc41cb2ba..07bddaa08478 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -189,8 +189,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .send_IPI_all = x2apic_send_IPI_all, .send_IPI_self = x2apic_send_IPI_self, - .inquire_remote_apic = NULL, - .read = native_apic_msr_read, .write = native_apic_msr_write, .eoi_write = native_apic_msr_eoi_write, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index d9384d5b4b8e..0e23e416459e 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -862,7 +862,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .send_IPI_self = uv_send_IPI_self, .wakeup_secondary_cpu = uv_wakeup_secondary, - .inquire_remote_apic = NULL, .read = native_apic_msr_read, .write = native_apic_msr_write, diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 8229f41d1ec1..a33fc505ca7b 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -774,44 +774,6 @@ static void impress_friends(void) pr_debug("Before bogocount - setting activated=1\n"); } -void __inquire_remote_apic(int apicid) -{ - unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 }; - const char * const names[] = { "ID", "VERSION", "SPIV" }; - int timeout; - u32 status; - - pr_info("Inquiring remote APIC 0x%x...\n", apicid); - - for (i = 0; i < ARRAY_SIZE(regs); i++) { - pr_info("... APIC 0x%x %s: ", apicid, names[i]); - - /* - * Wait for idle. - */ - status = safe_apic_wait_icr_idle(); - if (status) - pr_cont("a previous APIC delivery may have failed\n"); - - apic_icr_write(APIC_DM_REMRD | regs[i], apicid); - - timeout = 0; - do { - udelay(100); - status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK; - } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000); - - switch (status) { - case APIC_ICR_RR_VALID: - status = apic_read(APIC_RRR); - pr_cont("%08x\n", status); - break; - default: - pr_cont("failed\n"); - } - } -} - /* * The Multiprocessor Specification 1.4 (1997) example code suggests * that there should be a 10ms delay between the BSP asserting INIT diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 62d34b6611c5..a5428d971da2 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -129,10 +129,6 @@ static void xen_noop(void) { } -static void xen_silent_inquire(int apicid) -{ -} - static int xen_cpu_present_to_apicid(int cpu) { if (cpu_present(cpu)) @@ -173,9 +169,6 @@ static struct apic xen_pv_apic = { .send_IPI_all = xen_send_IPI_all, .send_IPI_self = xen_send_IPI_self, #endif - /* .wait_for_init_deassert- used by AP bootup - smp_callin which we don't use */ - .inquire_remote_apic = xen_silent_inquire, - .read = xen_apic_read, .write = xen_apic_write, .eoi_write = xen_apic_write, -- cgit v1.2.3 From d10a904435fa63fccc740e750e32b80e5f63c6c8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:45 -0700 Subject: x86/apic: Consolidate boot_cpu_physical_apicid initialization sites boot_cpu_physical_apicid is written in random places and in the last consequence filled with the APIC ID read from the local APIC. That causes it to have inconsistent state when the MPTABLE is broken. As a consequence tons of moronic checks are sprinkled all over the place. Consolidate the code and read it exactly once when either X2APIC mode is detected early or when the APIC mapping is established. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 +- arch/x86/kernel/apic/apic.c | 102 ++++++++++++++------------------------------ arch/x86/kernel/mpparse.c | 4 +- 3 files changed, 34 insertions(+), 74 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 47bedb88c938..108fdc25f90b 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -247,7 +247,7 @@ static inline int x2apic_enabled(void) #else /* !CONFIG_X86_X2APIC */ static inline void x2apic_setup(void) { } static inline int x2apic_enabled(void) { return 0; } - +static inline u32 native_apic_msr_read(u32 reg) { BUG(); } #define x2apic_mode (0) #define x2apic_supported() (0) #endif /* !CONFIG_X86_X2APIC */ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index d6b505942c7f..e26447aa469e 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1318,8 +1318,7 @@ static int __init __apic_intr_mode_select(void) if (!boot_cpu_has(X86_FEATURE_APIC) && APIC_INTEGRATED(boot_cpu_apic_version)) { apic_is_disabled = true; - pr_err(FW_BUG "Local APIC %d not detected, force emulation\n", - boot_cpu_physical_apicid); + pr_err(FW_BUG "Local APIC not detected, force emulation\n"); return APIC_PIC; } #endif @@ -1340,12 +1339,6 @@ static int __init __apic_intr_mode_select(void) pr_info("APIC: SMP mode deactivated\n"); return APIC_SYMMETRIC_IO_NO_ROUTING; } - - if (read_apic_id() != boot_cpu_physical_apicid) { - panic("Boot APIC ID in local APIC unexpected (%d vs %d)", - read_apic_id(), boot_cpu_physical_apicid); - /* Or can we switch back to PIC here? */ - } #endif return APIC_SYMMETRIC_IO; @@ -1741,6 +1734,23 @@ void apic_ap_setup(void) end_local_APIC_setup(); } +static __init void apic_read_boot_cpu_id(bool x2apic) +{ + /* + * This can be invoked from check_x2apic() before the APIC has been + * selected. But that code knows for sure that the BIOS enabled + * X2APIC. + */ + if (x2apic) { + boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID); + boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR)); + } else { + boot_cpu_physical_apicid = read_apic_id(); + boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); + } +} + + #ifdef CONFIG_X86_X2APIC int x2apic_mode; EXPORT_SYMBOL_GPL(x2apic_mode); @@ -1921,6 +1931,7 @@ void __init check_x2apic(void) x2apic_state = X2APIC_ON_LOCKED; else x2apic_state = X2APIC_ON; + apic_read_boot_cpu_id(true); } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) { x2apic_state = X2APIC_DISABLED; } @@ -2109,15 +2120,11 @@ no_apic: */ void __init init_apic_mappings(void) { - unsigned int new_apicid; - if (apic_validate_deadline_timer()) pr_info("TSC deadline timer available\n"); - if (x2apic_mode) { - boot_cpu_physical_apicid = read_apic_id(); + if (x2apic_mode) return; - } /* If no local APIC can be found return early */ if (!smp_found_config && detect_init_APIC()) { @@ -2134,39 +2141,19 @@ void __init init_apic_mappings(void) if (!acpi_lapic && !smp_found_config) register_lapic_address(apic_phys); } - - /* - * Fetch the APIC ID of the BSP in case we have a - * default configuration (or the MP table is broken). - */ - new_apicid = read_apic_id(); - if (boot_cpu_physical_apicid != new_apicid) { - boot_cpu_physical_apicid = new_apicid; - /* - * yeah -- we lie about apic_version - * in case if apic was disabled via boot option - * but it's not a problem for SMP compiled kernel - * since apic_intr_mode_select is prepared for such - * a case and disable smp mode - */ - boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); - } } void __init register_lapic_address(unsigned long address) { mp_lapic_addr = address; - if (!x2apic_mode) { - set_fixmap_nocache(FIX_APIC_BASE, address); - apic_mmio_base = APIC_BASE; - apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", - APIC_BASE, address); - } - if (boot_cpu_physical_apicid == -1U) { - boot_cpu_physical_apicid = read_apic_id(); - boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); - } + if (x2apic_mode) + return; + + set_fixmap_nocache(FIX_APIC_BASE, address); + apic_mmio_base = APIC_BASE; + apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", APIC_BASE, address); + apic_read_boot_cpu_id(false); } /* @@ -2446,31 +2433,15 @@ int generic_processor_info(int apicid, int version) phys_cpu_present_map); /* - * boot_cpu_physical_apicid is designed to have the apicid - * returned by read_apic_id(), i.e, the apicid of the - * currently booting-up processor. However, on some platforms, - * it is temporarily modified by the apicid reported as BSP - * through MP table. Concretely: - * - * - arch/x86/kernel/mpparse.c: MP_processor_info() - * - arch/x86/mm/amdtopology.c: amd_numa_init() - * - * This function is executed with the modified - * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel - * parameter doesn't work to disable APs on kdump 2nd kernel. - * - * Since fixing handling of boot_cpu_physical_apicid requires - * another discussion and tests on each platform, we leave it - * for now and here we use read_apic_id() directly in this - * function, generic_processor_info(). + * boot_cpu_physical_apicid is guaranteed to contain the boot CPU + * APIC ID read from the local APIC when this function is invoked. */ - if (disabled_cpu_apicid != BAD_APICID && - disabled_cpu_apicid != read_apic_id() && + if (disabled_cpu_apicid != boot_cpu_physical_apicid && disabled_cpu_apicid == apicid) { int thiscpu = num_processors + disabled_cpus; - pr_warn("APIC: Disabling requested cpu." - " Processor %d/0x%x ignored.\n", thiscpu, apicid); + pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n", + thiscpu, apicid); disabled_cpus++; return -ENODEV; @@ -2626,15 +2597,6 @@ static void __init apic_bsp_up_setup(void) { #ifdef CONFIG_X86_64 apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid)); -#else - /* - * Hack: In case of kdump, after a crash, kernel might be booting - * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid - * might be zero if read from MP tables. Get it from LAPIC. - */ -# ifdef CONFIG_CRASH_DUMP - boot_cpu_physical_apicid = read_apic_id(); -# endif #endif physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); } diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c index fed721f90116..fe9a7f63d2d9 100644 --- a/arch/x86/kernel/mpparse.c +++ b/arch/x86/kernel/mpparse.c @@ -58,10 +58,8 @@ static void __init MP_processor_info(struct mpc_cpu *m) apicid = m->apicid; - if (m->cpuflag & CPU_BOOTPROCESSOR) { + if (m->cpuflag & CPU_BOOTPROCESSOR) bootup_cpu = " (Bootup-CPU)"; - boot_cpu_physical_apicid = m->apicid; - } pr_info("Processor #%d%s\n", m->apicid, bootup_cpu); generic_processor_info(apicid, m->apicver); -- cgit v1.2.3 From d63107fa882eadb59a040313510ef8511746efea Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:46 -0700 Subject: x86/apic: Register boot CPU APIC early Register the boot CPU APIC right when the boot CPUs APIC is read from the hardware. No point is doing this on random places and having wild heuristics to save the boot CPU APIC ID slot and CPU number 0 reserved. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic.c | 120 ++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 70 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index e26447aa469e..b7ffae46101d 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1734,6 +1734,8 @@ void apic_ap_setup(void) end_local_APIC_setup(); } +static __init void cpu_set_boot_apic(void); + static __init void apic_read_boot_cpu_id(bool x2apic) { /* @@ -1748,9 +1750,9 @@ static __init void apic_read_boot_cpu_id(bool x2apic) boot_cpu_physical_apicid = read_apic_id(); boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); } + cpu_set_boot_apic(); } - #ifdef CONFIG_X86_X2APIC int x2apic_mode; EXPORT_SYMBOL_GPL(x2apic_mode); @@ -2426,76 +2428,8 @@ static int allocate_logical_cpuid(int apicid) return nr_logical_cpuids++; } -int generic_processor_info(int apicid, int version) +static void cpu_update_apic(int cpu, int apicid, int version) { - int cpu, max = nr_cpu_ids; - bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, - phys_cpu_present_map); - - /* - * boot_cpu_physical_apicid is guaranteed to contain the boot CPU - * APIC ID read from the local APIC when this function is invoked. - */ - if (disabled_cpu_apicid != boot_cpu_physical_apicid && - disabled_cpu_apicid == apicid) { - int thiscpu = num_processors + disabled_cpus; - - pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n", - thiscpu, apicid); - - disabled_cpus++; - return -ENODEV; - } - - /* - * If boot cpu has not been detected yet, then only allow upto - * nr_cpu_ids - 1 processors and keep one slot free for boot cpu - */ - if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 && - apicid != boot_cpu_physical_apicid) { - int thiscpu = max + disabled_cpus - 1; - - pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost" - " reached. Keeping one slot for boot cpu." - " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); - - disabled_cpus++; - return -ENODEV; - } - - if (num_processors >= nr_cpu_ids) { - int thiscpu = max + disabled_cpus; - - pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. " - "Processor %d/0x%x ignored.\n", max, thiscpu, apicid); - - disabled_cpus++; - return -EINVAL; - } - - if (apicid == boot_cpu_physical_apicid) { - /* - * x86_cpu_to_apicid is required to have processors listed - * in same order as logical cpu numbers. Hence the first - * entry is BSP, and so on. - * boot_cpu_init() already hold bit 0 in cpu_present_mask - * for BSP. - */ - cpu = 0; - - /* Logical cpuid 0 is reserved for BSP. */ - cpuid_to_apicid[0] = apicid; - } else { - cpu = allocate_logical_cpuid(apicid); - if (cpu < 0) { - disabled_cpus++; - return -EINVAL; - } - } - - /* - * Validate version - */ if (version == 0x0) { pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n", cpu, apicid); @@ -2521,10 +2455,56 @@ int generic_processor_info(int apicid, int version) if (system_state != SYSTEM_BOOTING) cpu_mark_primary_thread(cpu, apicid); +} + +static __init void cpu_set_boot_apic(void) +{ + cpuid_to_apicid[0] = boot_cpu_physical_apicid; + cpu_update_apic(0, boot_cpu_physical_apicid, boot_cpu_apic_version); +} + +int generic_processor_info(int apicid, int version) +{ + int cpu, max = nr_cpu_ids; + + /* The boot CPU must be set before MADT/MPTABLE parsing happens */ + if (cpuid_to_apicid[0] == BAD_APICID) + panic("Boot CPU APIC not registered yet\n"); + + if (apicid == boot_cpu_physical_apicid) + return 0; + + if (disabled_cpu_apicid == apicid) { + int thiscpu = num_processors + disabled_cpus; + + pr_warn("APIC: Disabling requested cpu. Processor %d/0x%x ignored.\n", + thiscpu, apicid); + disabled_cpus++; + return -ENODEV; + } + + if (num_processors >= nr_cpu_ids) { + int thiscpu = max + disabled_cpus; + + pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. " + "Processor %d/0x%x ignored.\n", max, thiscpu, apicid); + + disabled_cpus++; + return -EINVAL; + } + + cpu = allocate_logical_cpuid(apicid); + if (cpu < 0) { + disabled_cpus++; + return -EINVAL; + } + + cpu_update_apic(cpu, apicid, version); return cpu; } + void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg, bool dmar) { -- cgit v1.2.3 From 249ada2c821ff6819b5214d2f08d73aebde814eb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:46 -0700 Subject: x86/apic: Remove the pointless APIC version check This historical leftover is really uninteresting today. Whatever MPTABLE or MADT delivers we only trust the hardware anyway. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/mpspec.h | 2 +- arch/x86/kernel/acpi/boot.c | 6 +----- arch/x86/kernel/apic/apic.c | 19 ++++--------------- arch/x86/kernel/devicetree.c | 5 ++--- arch/x86/kernel/jailhouse.c | 6 ++---- arch/x86/kernel/mpparse.c | 2 +- arch/x86/kernel/smpboot.c | 4 ++-- 7 files changed, 13 insertions(+), 31 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index fa5e24a58595..190d30685fac 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -76,7 +76,7 @@ static inline void e820__memblock_alloc_reserved_mpc_new(void) { } #define default_get_smp_config x86_init_uint_noop #endif -int generic_processor_info(int apicid, int version); +int generic_processor_info(int apicid); #define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_LOCAL_APIC) diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 38a529830829..cca1a0709605 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -169,7 +169,6 @@ static int __init acpi_parse_madt(struct acpi_table_header *table) */ static int acpi_register_lapic(int id, u32 acpiid, u8 enabled) { - unsigned int ver = 0; int cpu; if (id >= MAX_LOCAL_APIC) { @@ -182,10 +181,7 @@ static int acpi_register_lapic(int id, u32 acpiid, u8 enabled) return -EINVAL; } - if (boot_cpu_physical_apicid != -1U) - ver = boot_cpu_apic_version; - - cpu = generic_processor_info(id, ver); + cpu = generic_processor_info(id); if (cpu >= 0) early_per_cpu(x86_cpu_to_acpiid, cpu) = acpiid; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b7ffae46101d..32275ca6129f 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2428,19 +2428,8 @@ static int allocate_logical_cpuid(int apicid) return nr_logical_cpuids++; } -static void cpu_update_apic(int cpu, int apicid, int version) +static void cpu_update_apic(int cpu, int apicid) { - if (version == 0x0) { - pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n", - cpu, apicid); - version = 0x10; - } - - if (version != boot_cpu_apic_version) { - pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n", - boot_cpu_apic_version, cpu, version); - } - #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; #endif @@ -2460,10 +2449,10 @@ static void cpu_update_apic(int cpu, int apicid, int version) static __init void cpu_set_boot_apic(void) { cpuid_to_apicid[0] = boot_cpu_physical_apicid; - cpu_update_apic(0, boot_cpu_physical_apicid, boot_cpu_apic_version); + cpu_update_apic(0, boot_cpu_physical_apicid); } -int generic_processor_info(int apicid, int version) +int generic_processor_info(int apicid) { int cpu, max = nr_cpu_ids; @@ -2500,7 +2489,7 @@ int generic_processor_info(int apicid, int version) return -EINVAL; } - cpu_update_apic(cpu, apicid, version); + cpu_update_apic(cpu, apicid); return cpu; } diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 28da5dd83fc0..56f6d591d0df 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -128,16 +128,15 @@ static void __init dtb_setup_hpet(void) static void __init dtb_cpu_setup(void) { struct device_node *dn; - u32 apic_id, version; + u32 apic_id; - version = GET_APIC_VERSION(apic_read(APIC_LVR)); for_each_of_cpu_node(dn) { apic_id = of_get_cpu_hwid(dn, 0); if (apic_id == ~0U) { pr_warn("%pOF: missing local APIC ID\n", dn); continue; } - generic_processor_info(apic_id, version); + generic_processor_info(apic_id); } } diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c index 4eb8f2d19a87..578d16fc040f 100644 --- a/arch/x86/kernel/jailhouse.c +++ b/arch/x86/kernel/jailhouse.c @@ -101,10 +101,8 @@ static void __init jailhouse_get_smp_config(unsigned int early) register_lapic_address(0xfee00000); - for (cpu = 0; cpu < setup_data.v1.num_cpus; cpu++) { - generic_processor_info(setup_data.v1.cpu_ids[cpu], - boot_cpu_apic_version); - } + for (cpu = 0; cpu < setup_data.v1.num_cpus; cpu++) + generic_processor_info(setup_data.v1.cpu_ids[cpu]); smp_found_config = 1; diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c index fe9a7f63d2d9..82518c1aff8d 100644 --- a/arch/x86/kernel/mpparse.c +++ b/arch/x86/kernel/mpparse.c @@ -62,7 +62,7 @@ static void __init MP_processor_info(struct mpc_cpu *m) bootup_cpu = " (Bootup-CPU)"; pr_info("Processor #%d%s\n", m->apicid, bootup_cpu); - generic_processor_info(apicid, m->apicver); + generic_processor_info(apicid); } #ifdef CONFIG_X86_IO_APIC diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index a33fc505ca7b..5467c58827f5 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1397,7 +1397,7 @@ __init void prefill_possible_map(void) { int i, possible; - /* No boot processor was found in mptable or ACPI MADT */ + /* No processor was found in mptable or ACPI MADT */ if (!num_processors) { if (boot_cpu_has(X86_FEATURE_APIC)) { int apicid = boot_cpu_physical_apicid; @@ -1408,7 +1408,7 @@ __init void prefill_possible_map(void) /* Make sure boot cpu is enumerated */ if (apic->cpu_present_to_apicid(0) == BAD_APICID && apic->apic_id_valid(apicid)) - generic_processor_info(apicid, boot_cpu_apic_version); + generic_processor_info(apicid); } if (!num_processors) -- cgit v1.2.3 From 004671e5c9335736ef808482bfc69cccd93cccc5 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Tue, 8 Aug 2023 15:03:47 -0700 Subject: x86/apic: Remove mpparse 'apicid' variable From: Dave Hansen Some truly ancient code had different ways of calculating the 'apicid' but it is long gone. Zap the unnecssary local variablee Signed-off-by: Dave Hansen --- arch/x86/kernel/mpparse.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c index 82518c1aff8d..333dee2cfbb8 100644 --- a/arch/x86/kernel/mpparse.c +++ b/arch/x86/kernel/mpparse.c @@ -48,7 +48,6 @@ static int __init mpf_checksum(unsigned char *mp, int len) static void __init MP_processor_info(struct mpc_cpu *m) { - int apicid; char *bootup_cpu = ""; if (!(m->cpuflag & CPU_ENABLED)) { @@ -56,13 +55,11 @@ static void __init MP_processor_info(struct mpc_cpu *m) return; } - apicid = m->apicid; - if (m->cpuflag & CPU_BOOTPROCESSOR) bootup_cpu = " (Bootup-CPU)"; pr_info("Processor #%d%s\n", m->apicid, bootup_cpu); - generic_processor_info(apicid); + generic_processor_info(m->apicid); } #ifdef CONFIG_X86_IO_APIC -- cgit v1.2.3 From 2906a67ac83b06b874b32bf6413a8d5e186fc33e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:47 -0700 Subject: x86/of: Fix the APIC address registration The device tree APIC parser tries to force-enable the local APIC when it is not set in CPUID. apic_force_enable() registers the boot CPU apic on success. If that succeeds then dtb_lapic_setup() registers the local APIC again eventually with a different address. Rewrite the code so that it only registers it once. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/devicetree.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 56f6d591d0df..8f94eb6d7677 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -157,19 +157,15 @@ static void __init dtb_lapic_setup(void) /* Did the boot loader setup the local APIC ? */ if (!boot_cpu_has(X86_FEATURE_APIC)) { + /* Try force enabling, which registers the APIC address */ if (apic_force_enable(lapic_addr)) return; - } - smp_found_config = 1; - if (of_property_read_bool(dn, "intel,virtual-wire-mode")) { - pr_info("Virtual Wire compatibility mode.\n"); - pic_mode = 0; } else { - pr_info("IMCR and PIC compatibility mode.\n"); - pic_mode = 1; + register_lapic_address(lapic_addr); } - - register_lapic_address(lapic_addr); + smp_found_config = 1; + pic_mode = !of_property_read_bool(dn, "intel,virtual-wire-mode"); + pr_info("%s compatibility mode.\n", pic_mode ? "IMCR and PIC" : "Virtual Wire"); } #endif /* CONFIG_X86_LOCAL_APIC */ -- cgit v1.2.3 From 1751adedbd601d6a9da35de0c1e2592ac980fdfe Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:48 -0700 Subject: x86/apic: Make some APIC init functions bool Quite some APIC init functions are pure boolean, but use the success = 0, fail < 0 model. That's confusing as hell when reading through the code. Convert them to boolean. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 6 +++--- arch/x86/kernel/apic/apic.c | 36 ++++++++++++++++++------------------ arch/x86/kernel/devicetree.c | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 108fdc25f90b..35d236b46ad7 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -135,12 +135,12 @@ extern void setup_secondary_APIC_clock(void); extern void lapic_update_tsc_freq(void); #ifdef CONFIG_X86_64 -static inline int apic_force_enable(unsigned long addr) +static inline bool apic_force_enable(unsigned long addr) { - return -1; + return false; } #else -extern int apic_force_enable(unsigned long addr); +extern bool apic_force_enable(unsigned long addr); #endif extern void apic_ap_setup(void); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 32275ca6129f..d0f4cca30378 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2000,19 +2000,19 @@ void __init enable_IR_x2apic(void) * On AMD64 we trust the BIOS - if it says no APIC it is likely * not correctly set up (usually the APIC timer won't work etc.) */ -static int __init detect_init_APIC(void) +static bool __init detect_init_APIC(void) { if (!boot_cpu_has(X86_FEATURE_APIC)) { pr_info("No local APIC present\n"); - return -1; + return false; } mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; - return 0; + return true; } #else -static int __init apic_verify(void) +static bool __init apic_verify(void) { u32 features, h, l; @@ -2023,7 +2023,7 @@ static int __init apic_verify(void) features = cpuid_edx(1); if (!(features & (1 << X86_FEATURE_APIC))) { pr_warn("Could not enable APIC!\n"); - return -1; + return false; } set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; @@ -2036,15 +2036,15 @@ static int __init apic_verify(void) } pr_info("Found and enabled local APIC!\n"); - return 0; + return true; } -int __init apic_force_enable(unsigned long addr) +bool __init apic_force_enable(unsigned long addr) { u32 h, l; if (apic_is_disabled) - return -1; + return false; /* * Some BIOSes disable the local APIC in the APIC_BASE @@ -2067,11 +2067,11 @@ int __init apic_force_enable(unsigned long addr) /* * Detect and initialize APIC */ -static int __init detect_init_APIC(void) +static bool __init detect_init_APIC(void) { /* Disabled by kernel option? */ if (apic_is_disabled) - return -1; + return false; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_AMD: @@ -2098,22 +2098,22 @@ static int __init detect_init_APIC(void) if (!force_enable_local_apic) { pr_info("Local APIC disabled by BIOS -- " "you can enable it with \"lapic\"\n"); - return -1; + return false; } - if (apic_force_enable(APIC_DEFAULT_PHYS_BASE)) - return -1; + if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE)) + return false; } else { - if (apic_verify()) - return -1; + if (!apic_verify()) + return false; } apic_pm_activate(); - return 0; + return true; no_apic: pr_info("No local APIC present or hardware disabled\n"); - return -1; + return false; } #endif @@ -2129,7 +2129,7 @@ void __init init_apic_mappings(void) return; /* If no local APIC can be found return early */ - if (!smp_found_config && detect_init_APIC()) { + if (!smp_found_config && !detect_init_APIC()) { /* lets NOP'ify apic operations */ pr_info("APIC: disable apic facility\n"); apic_disable(); diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 8f94eb6d7677..87d38f17ff5c 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -158,7 +158,7 @@ static void __init dtb_lapic_setup(void) /* Did the boot loader setup the local APIC ? */ if (!boot_cpu_has(X86_FEATURE_APIC)) { /* Try force enabling, which registers the APIC address */ - if (apic_force_enable(lapic_addr)) + if (!apic_force_enable(lapic_addr)) return; } else { register_lapic_address(lapic_addr); -- cgit v1.2.3 From 5a88f354dcd8f19436bbda52f075276a5c70424f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:48 -0700 Subject: x86/apic: Split register_apic_address() Split the fixmap setup out of register_lapic_address() and reuse it when the X2APIC is disabled during setup. This avoids registering the APIC ID (setting 'mp_lapic_addr') twice. [ dhansen: changelog wording tweak ] Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index d0f4cca30378..45ae178b9e50 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1852,6 +1852,8 @@ void x2apic_setup(void) __x2apic_enable(); } +static __init void apic_set_fixmap(void); + static __init void x2apic_disable(void) { u32 x2apic_id, state = x2apic_state; @@ -1872,7 +1874,7 @@ static __init void x2apic_disable(void) } __x2apic_disable(); - register_lapic_address(mp_lapic_addr); + apic_set_fixmap(); } static __init void x2apic_enable(void) @@ -2145,17 +2147,21 @@ void __init init_apic_mappings(void) } } +static __init void apic_set_fixmap(void) +{ + set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr); + apic_mmio_base = APIC_BASE; + apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", + apic_mmio_base, mp_lapic_addr); + apic_read_boot_cpu_id(false); +} + void __init register_lapic_address(unsigned long address) { mp_lapic_addr = address; - if (x2apic_mode) - return; - - set_fixmap_nocache(FIX_APIC_BASE, address); - apic_mmio_base = APIC_BASE; - apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", APIC_BASE, address); - apic_read_boot_cpu_id(false); + if (!x2apic_mode) + apic_set_fixmap(); } /* -- cgit v1.2.3 From 81287ad65da5b82155f2eca03d81db5512802d16 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:49 -0700 Subject: x86/apic: Sanitize APIC address setup Convert places which just write mp_lapic_addr and let them register the local APIC address directly instead of relying on magic other code to do so. Add a WARN_ON() into register_lapic_address() which is raised when register_lapic_address() is invoked more than once during boot. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/mpspec.h | 1 - arch/x86/kernel/apic/apic.c | 29 ++++++++++------------------- arch/x86/kernel/mpparse.c | 11 ++--------- 3 files changed, 12 insertions(+), 29 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index 190d30685fac..5d01bdee0186 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -41,7 +41,6 @@ extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); extern unsigned int boot_cpu_physical_apicid; extern u8 boot_cpu_apic_version; -extern unsigned long mp_lapic_addr; #ifdef CONFIG_X86_LOCAL_APIC extern int smp_found_config; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 45ae178b9e50..84ff5b27590b 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -172,7 +172,7 @@ static __init int setup_apicpmtimer(char *s) __setup("apicpmtimer", setup_apicpmtimer); #endif -unsigned long mp_lapic_addr __ro_after_init; +static unsigned long mp_lapic_addr __ro_after_init; bool apic_is_disabled __ro_after_init; /* Disable local APIC timer from the kernel commandline or via dmi quirk */ static int disable_apic_timer __initdata; @@ -2009,12 +2009,12 @@ static bool __init detect_init_APIC(void) return false; } - mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; + register_lapic_address(APIC_DEFAULT_PHYS_BASE); return true; } #else -static bool __init apic_verify(void) +static bool __init apic_verify(unsigned long addr) { u32 features, h, l; @@ -2028,15 +2028,15 @@ static bool __init apic_verify(void) return false; } set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC); - mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; /* The BIOS may have set up the APIC at some other address */ if (boot_cpu_data.x86 >= 6) { rdmsr(MSR_IA32_APICBASE, l, h); if (l & MSR_IA32_APICBASE_ENABLE) - mp_lapic_addr = l & MSR_IA32_APICBASE_BASE; + addr = l & MSR_IA32_APICBASE_BASE; } + register_lapic_address(addr); pr_info("Found and enabled local APIC!\n"); return true; } @@ -2063,7 +2063,7 @@ bool __init apic_force_enable(unsigned long addr) enabled_via_apicbase = 1; } } - return apic_verify(); + return apic_verify(addr); } /* @@ -2105,7 +2105,7 @@ static bool __init detect_init_APIC(void) if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE)) return false; } else { - if (!apic_verify()) + if (!apic_verify(APIC_DEFAULT_PHYS_BASE)) return false; } @@ -2130,27 +2130,16 @@ void __init init_apic_mappings(void) if (x2apic_mode) return; - /* If no local APIC can be found return early */ if (!smp_found_config && !detect_init_APIC()) { - /* lets NOP'ify apic operations */ pr_info("APIC: disable apic facility\n"); apic_disable(); - } else { - apic_phys = mp_lapic_addr; - - /* - * If the system has ACPI MADT tables or MP info, the LAPIC - * address is already registered. - */ - if (!acpi_lapic && !smp_found_config) - register_lapic_address(apic_phys); } } static __init void apic_set_fixmap(void) { set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr); - apic_mmio_base = APIC_BASE; + apic_phys = apic_mmio_base = APIC_BASE; apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", apic_mmio_base, mp_lapic_addr); apic_read_boot_cpu_id(false); @@ -2158,6 +2147,8 @@ static __init void apic_set_fixmap(void) void __init register_lapic_address(unsigned long address) { + /* This should only happen once */ + WARN_ON_ONCE(mp_lapic_addr); mp_lapic_addr = address; if (!x2apic_mode) diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c index 333dee2cfbb8..b223922248e9 100644 --- a/arch/x86/kernel/mpparse.c +++ b/arch/x86/kernel/mpparse.c @@ -374,11 +374,6 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type) int linttypes[2] = { mp_ExtINT, mp_NMI }; int i; - /* - * local APIC has default address - */ - mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; - /* * 2 CPUs, numbered 0 & 1. */ @@ -520,10 +515,8 @@ void __init default_get_smp_config(unsigned int early) */ if (mpf->feature1) { if (early) { - /* - * local APIC has default address - */ - mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; + /* Local APIC has default address */ + register_lapic_address(APIC_DEFAULT_PHYS_BASE); goto out; } -- cgit v1.2.3 From 24889a3a5314f71e7bca586b607882d87c71799c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:49 -0700 Subject: x86/xen/pv: Pretend that it found SMP configuration Unlike all other SMP configuration "parsers" XEN/PV does not set smp_found_config which is inconsistent and prevents doing proper decision logic based on this flag. Make XEN/PV pretend that it found SMP configuration. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/xen/smp_pv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index c6b42c66c60c..a0f07bbfcd6e 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -182,7 +182,8 @@ static void __init _get_smp_config(unsigned int early) if (subtract) set_nr_cpu_ids(nr_cpu_ids - subtract); #endif - + /* Pretend to be a proper enumerated system */ + smp_found_config = 1; } static void __init xen_pv_smp_prepare_boot_cpu(void) -- cgit v1.2.3 From e8122513ff9e92cdc514ad6cbb5c25d551ac3629 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:50 -0700 Subject: x86/apic: Sanitize num_processors handling num_processors is 0 by default and only gets incremented when local APICs are registered. Make init_apic_mappings(), which tries to enable the local APIC in the case that no SMP configuration was found set num_processors to 1. This allows to remove yet another check for the local APIC and yet another place which registers the boot CPUs local APIC ID. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic.c | 9 ++++++--- arch/x86/kernel/smpboot.c | 18 ------------------ 2 files changed, 6 insertions(+), 21 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 84ff5b27590b..564d584c8b99 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2130,9 +2130,12 @@ void __init init_apic_mappings(void) if (x2apic_mode) return; - if (!smp_found_config && !detect_init_APIC()) { - pr_info("APIC: disable apic facility\n"); - apic_disable(); + if (!smp_found_config) { + if (!detect_init_APIC()) { + pr_info("APIC: disable apic facility\n"); + apic_disable(); + } + num_processors = 1; } } diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 5467c58827f5..fc2173b7dc89 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1397,24 +1397,6 @@ __init void prefill_possible_map(void) { int i, possible; - /* No processor was found in mptable or ACPI MADT */ - if (!num_processors) { - if (boot_cpu_has(X86_FEATURE_APIC)) { - int apicid = boot_cpu_physical_apicid; - int cpu = read_apic_id(); - - pr_warn("Boot CPU (id %d) not listed by BIOS\n", cpu); - - /* Make sure boot cpu is enumerated */ - if (apic->cpu_present_to_apicid(0) == BAD_APICID && - apic->apic_id_valid(apicid)) - generic_processor_info(apicid); - } - - if (!num_processors) - num_processors = 1; - } - i = setup_max_cpus ?: 1; if (setup_possible_cpus == -1) { possible = num_processors; -- cgit v1.2.3 From 55cc40d3df3c5c49f3a72e451bb710ec9e398ddc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:50 -0700 Subject: x86/apic: Nuke another processor check The boot CPUs local APIC is now always registered, so there is no point to have another unreadable validatation for it. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/smpboot.c | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index fc2173b7dc89..da795f743542 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1181,23 +1181,6 @@ static void __init smp_sanity_check(void) set_nr_cpu_ids(8); } #endif - - if (!physid_isset(read_apic_id(), phys_cpu_present_map)) { - pr_warn("weird, boot CPU (#%d) not listed by the BIOS\n", - read_apic_id()); - - physid_set(read_apic_id(), phys_cpu_present_map); - } - - /* - * Should not be necessary because the MP table should list the boot - * CPU too, but we do it for the sake of robustness anyway. - */ - if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) { - pr_notice("weird, boot CPU (#%d) not listed by the BIOS\n", - boot_cpu_physical_apicid); - physid_set(read_apic_id(), phys_cpu_present_map); - } preempt_enable(); } -- cgit v1.2.3 From f52e2c3e96d949b229897dc234679553c179f6b2 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:52 -0700 Subject: x86/apic: Remove check_phys_apicid_present() The only silly usage site is gone. Remove the gunk which was even outright wrong in the bigsmp_32 case which returned true unconditionally. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 -- arch/x86/kernel/apic/apic_common.c | 5 ----- arch/x86/kernel/apic/apic_flat_64.c | 2 -- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 2 -- arch/x86/kernel/apic/bigsmp_32.c | 6 ------ arch/x86/kernel/apic/probe_32.c | 1 - arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 1 - arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/xen/apic.c | 1 - 11 files changed, 24 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 35d236b46ad7..098d193f93fb 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -305,7 +305,6 @@ struct apic { void (*setup_apic_routing)(void); int (*cpu_present_to_apicid)(int mps_cpu); void (*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap); - int (*check_phys_apicid_present)(int phys_apicid); int (*phys_pkg_id)(int cpuid_apic, int index_msb); u32 (*get_apic_id)(unsigned long x); @@ -485,7 +484,6 @@ extern u32 apic_flat_calc_apicid(unsigned int cpu); extern bool default_check_apicid_used(physid_mask_t *map, int apicid); extern void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap); extern int default_cpu_present_to_apicid(int mps_cpu); -extern int default_check_phys_apicid_present(int phys_apicid); #else /* CONFIG_X86_LOCAL_APIC */ diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index e892506552c4..41528bb3a608 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -35,11 +35,6 @@ int default_cpu_present_to_apicid(int mps_cpu) } EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid); -int default_check_phys_apicid_present(int phys_apicid) -{ - return physid_isset(phys_apicid, phys_cpu_present_map); -} - int default_apic_id_valid(u32 apicid) { return (apicid < 255); diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 35cd1cb39fae..00482cf8255a 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -124,7 +124,6 @@ static struct apic apic_flat __ro_after_init = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = flat_phys_pkg_id, .get_apic_id = flat_get_apic_id, @@ -213,7 +212,6 @@ static struct apic apic_physflat __ro_after_init = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = flat_phys_pkg_id, .get_apic_id = flat_get_apic_id, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 90bf11e6827c..c0c3b6bf79b5 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -107,8 +107,6 @@ struct apic apic_noop __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = physid_set_mask_of_physid, - .check_phys_apicid_present = default_check_phys_apicid_present, - .phys_pkg_id = noop_phys_pkg_id, .get_apic_id = noop_get_apic_id, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index b6bbd7d2c4bc..de16fde97568 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -257,7 +257,6 @@ static const struct apic apic_numachip1 __refconst = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = numachip_phys_pkg_id, .get_apic_id = numachip1_get_apic_id, @@ -303,7 +302,6 @@ static const struct apic apic_numachip2 __refconst = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = numachip_phys_pkg_id, .get_apic_id = numachip2_get_apic_id, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 9e254dc94e55..6f4bd71e97ec 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -63,11 +63,6 @@ static void bigsmp_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *re physids_promote(0xFFL, retmap); } -static int bigsmp_check_phys_apicid_present(int phys_apicid) -{ - return 1; -} - static int bigsmp_phys_pkg_id(int cpuid_apic, int index_msb) { return cpuid_apic >> index_msb; @@ -138,7 +133,6 @@ static struct apic apic_bigsmp __ro_after_init = { .setup_apic_routing = bigsmp_setup_apic_routing, .cpu_present_to_apicid = bigsmp_cpu_present_to_apicid, .apicid_to_cpu_present = physid_set_mask_of_physid, - .check_phys_apicid_present = bigsmp_check_phys_apicid_present, .phys_pkg_id = bigsmp_phys_pkg_id, .get_apic_id = bigsmp_get_apic_id, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 8d1e456a77b8..baa8b141eba1 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -80,7 +80,6 @@ static struct apic apic_default __ro_after_init = { .setup_apic_routing = setup_apic_flat_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = physid_set_mask_of_physid, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = default_phys_pkg_id, .get_apic_id = default_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index ce1021c0191d..931e0b9950f3 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -250,7 +250,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = x2apic_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 07bddaa08478..9cf32ce779be 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -174,7 +174,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = x2apic_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 0e23e416459e..0d812fdc9b25 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -846,7 +846,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = NULL, - .check_phys_apicid_present = default_check_phys_apicid_present, .phys_pkg_id = uv_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index a5428d971da2..c8240afca77c 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -154,7 +154,6 @@ static struct apic xen_pv_apic = { .setup_apic_routing = NULL, .cpu_present_to_apicid = xen_cpu_present_to_apicid, .apicid_to_cpu_present = physid_set_mask_of_physid, /* Used on 32-bit */ - .check_phys_apicid_present = default_check_phys_apicid_present, /* smp_sanity_check needs it */ .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ .get_apic_id = xen_get_apic_id, -- cgit v1.2.3 From 78c32000848c9a3af69e2431d17caed7b555e0ea Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:52 -0700 Subject: x86/apic: Get rid of apic_phys No need for an extra variable to find out whether the APIC has been mapped or is accessible (X2APIC mode). Provide an inline for this and check apic_mmio_base which is only set when the local APIC has been mapped. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 564d584c8b99..6eee777ca73c 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -99,6 +99,11 @@ static bool virt_ext_dest_id __ro_after_init; /* For parallel bootup. */ unsigned long apic_mmio_base __ro_after_init; +static inline bool apic_accessible(void) +{ + return x2apic_mode || apic_mmio_base; +} + /* * Map cpu index to physical APIC ID */ @@ -199,8 +204,6 @@ unsigned int lapic_timer_period = 0; static void apic_pm_activate(void); -static unsigned long apic_phys __ro_after_init; - /* * Get the LAPIC version */ @@ -1127,8 +1130,7 @@ void clear_local_APIC(void) int maxlvt; u32 v; - /* APIC hasn't been mapped yet */ - if (!x2apic_mode && !apic_phys) + if (!apic_accessible()) return; maxlvt = lapic_get_maxlvt(); @@ -1218,8 +1220,7 @@ void apic_soft_disable(void) */ void disable_local_APIC(void) { - /* APIC hasn't been mapped yet */ - if (!x2apic_mode && !apic_phys) + if (!apic_accessible()) return; apic_soft_disable(); @@ -2142,7 +2143,7 @@ void __init init_apic_mappings(void) static __init void apic_set_fixmap(void) { set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr); - apic_phys = apic_mmio_base = APIC_BASE; + apic_mmio_base = APIC_BASE; apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n", apic_mmio_base, mp_lapic_addr); apic_read_boot_cpu_id(false); @@ -2896,11 +2897,11 @@ early_param("apic", apic_set_verbosity); static int __init lapic_insert_resource(void) { - if (!apic_phys) + if (!apic_mmio_base) return -1; /* Put local APIC into the resource map. */ - lapic_resource.start = apic_phys; + lapic_resource.start = apic_mmio_base; lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; insert_resource(&iomem_resource, &lapic_resource); -- cgit v1.2.3 From e120e58ec2932d3dee05da71168c7ba841bf4cf4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:53 -0700 Subject: x86/apic/32: Sanitize logical APIC ID handling apic::x86_32_early_logical_apicid() is yet another historical joke. It is used to preset the x86_cpu_to_logical_apicid per CPU variable during APIC enumeration with: - 1 shifted left by the CPU number - the physical APIC ID in case of bigsmp The latter is hillarious because bigsmp uses physical destination mode which never can use the logical APIC ID. It gets even worse. As bigsmp can be enforced late in the boot process the probe function overwrites the per CPU variable which is never used for this APIC type once again. Remove that gunk and store 1 << cpunr unconditionally if and only if the CPU number is less than 8, because the default logical destination mode only allows up to 8 CPUs. This is just an intermediate step before removing the per CPU insanity completely. Stay tuned. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 13 ------------- arch/x86/kernel/apic/apic.c | 4 ++-- arch/x86/kernel/apic/apic_noop.c | 11 ----------- arch/x86/kernel/apic/bigsmp_32.c | 18 ------------------ arch/x86/kernel/apic/probe_32.c | 7 ------- 5 files changed, 2 insertions(+), 51 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 098d193f93fb..daef6812141f 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -315,19 +315,6 @@ struct apic { /* wakeup secondary CPU using 64-bit wakeup point */ int (*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip); -#ifdef CONFIG_X86_32 - /* - * Called very early during boot from get_smp_config(). It should - * return the logical apicid. x86_[bios]_cpu_to_apicid is - * initialized before this function is called. - * - * If logical apicid can't be determined that early, the function - * may return BAD_APICID. Logical apicid will be configured after - * init_apic_ldr() while bringing up CPUs. Note that NUMA affinity - * won't be applied properly during early boot in this case. - */ - int (*x86_32_early_logical_apicid)(int cpu); -#endif char *name; }; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 6eee777ca73c..89f2a7e5a9f5 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2435,8 +2435,8 @@ static void cpu_update_apic(int cpu, int apicid) early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; #endif #ifdef CONFIG_X86_32 - early_per_cpu(x86_cpu_to_logical_apicid, cpu) = - apic->x86_32_early_logical_apicid(cpu); + if (cpu < 8) + early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 1U << cpu; #endif set_cpu_possible(cpu, true); physid_set(apicid, phys_cpu_present_map); diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index c0c3b6bf79b5..ccd74e8016fc 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -80,13 +80,6 @@ static void noop_apic_write(u32 reg, u32 val) WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled); } -#ifdef CONFIG_X86_32 -static int noop_x86_32_early_logical_apicid(int cpu) -{ - return BAD_APICID; -} -#endif - struct apic apic_noop __ro_after_init = { .name = "noop", .probe = noop_probe, @@ -130,8 +123,4 @@ struct apic apic_noop __ro_after_init = { .icr_write = noop_apic_icr_write, .wait_icr_idle = noop_apic_wait_icr_idle, .safe_wait_icr_idle = noop_safe_apic_wait_icr_idle, - -#ifdef CONFIG_X86_32 - .x86_32_early_logical_apicid = noop_x86_32_early_logical_apicid, -#endif }; diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 6f4bd71e97ec..b649048c2019 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -28,12 +28,6 @@ static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid) return false; } -static int bigsmp_early_logical_apicid(int cpu) -{ - /* on bigsmp, logical apicid is the same as physical */ - return early_per_cpu(x86_cpu_to_apicid, cpu); -} - /* * bigsmp enables physical destination mode * and doesn't use LDR and DFR @@ -154,27 +148,15 @@ static struct apic apic_bigsmp __ro_after_init = { .icr_write = native_apic_icr_write, .wait_icr_idle = native_apic_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, - - .x86_32_early_logical_apicid = bigsmp_early_logical_apicid, }; void __init generic_bigsmp_probe(void) { - unsigned int cpu; - if (!probe_bigsmp()) return; apic = &apic_bigsmp; - for_each_possible_cpu(cpu) { - if (early_per_cpu(x86_cpu_to_logical_apicid, - cpu) == BAD_APICID) - continue; - early_per_cpu(x86_cpu_to_logical_apicid, cpu) = - bigsmp_early_logical_apicid(cpu); - } - pr_info("Overriding APIC driver with %s\n", apic_bigsmp.name); } diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index baa8b141eba1..3ee0211b0c29 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -16,11 +16,6 @@ #include "local.h" -static int default_x86_32_early_logical_apicid(int cpu) -{ - return 1 << cpu; -} - static void setup_apic_flat_routing(void) { #ifdef CONFIG_X86_IO_APIC @@ -101,8 +96,6 @@ static struct apic apic_default __ro_after_init = { .icr_write = native_apic_icr_write, .wait_icr_idle = native_apic_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, - - .x86_32_early_logical_apicid = default_x86_32_early_logical_apicid, }; apic_driver(apic_default); -- cgit v1.2.3 From f2bb0b4f150595e53aad09201be33d474ed6f4b7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:53 -0700 Subject: x86/apic/32: Remove x86_cpu_to_logical_apicid This per CPU variable is just yet another form of voodoo programming. The boot ordering is: per_cpu(x86_cpu_to_logical_apicid, cpu) = 1U << cpu; ..... setup_apic() apic->init_apic_ldr() default_init_apic_ldr() apic_write(SET_APIC_LOGICAL_ID(1UL << smp_processor_id(), APIC_LDR); id = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR); WARN_ON(id != per_cpu(x86_cpu_to_logical_apicid, cpu)); per_cpu(x86_cpu_to_logical_apicid, cpu) = id; So first write the default into LDR and then validate it against the same default which was set up during early boot APIC enumeration. Brilliant, isn't it? The comment above the per CPU variable declaration describes it well: 'Let's keep it ugly for now.' Remove the useless gunk and use '1U << cpu' consistently all over the place. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/smp.h | 3 --- arch/x86/kernel/apic/apic.c | 31 ------------------------------- arch/x86/kernel/apic/ipi.c | 36 +++++++++--------------------------- arch/x86/kernel/setup_percpu.c | 7 ------- 4 files changed, 9 insertions(+), 68 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 8d96d9a6962d..8ece65b63b64 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -22,9 +22,6 @@ DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid); -#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) -DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid); -#endif struct task_struct; diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 89f2a7e5a9f5..224dff4b9258 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -113,15 +113,6 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid); #ifdef CONFIG_X86_32 - -/* - * On x86_32, the mapping between cpu and logical apicid may vary - * depending on apic in use. The following early percpu variable is - * used for the mapping. This is where the behaviors of x86_64 and 32 - * actually diverge. Let's keep it ugly for now. - */ -DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID); - /* Local APIC was disabled by the BIOS and enabled by the kernel */ static int enabled_via_apicbase __ro_after_init; @@ -1589,24 +1580,6 @@ static void setup_local_APIC(void) */ apic->init_apic_ldr(); -#ifdef CONFIG_X86_32 - if (apic->dest_mode_logical) { - int logical_apicid, ldr_apicid; - - /* - * APIC LDR is initialized. If logical_apicid mapping was - * initialized during get_smp_config(), make sure it matches - * the actual value. - */ - logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu); - ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR)); - if (logical_apicid != BAD_APICID) - WARN_ON(logical_apicid != ldr_apicid); - /* Always use the value from LDR. */ - early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid; - } -#endif - /* * Set Task Priority to 'accept all except vectors 0-31'. An APIC * vector in the 16-31 range could be delivered if TPR == 0, but we @@ -2433,10 +2406,6 @@ static void cpu_update_apic(int cpu, int apicid) { #if defined(CONFIG_SMP) || defined(CONFIG_X86_64) early_per_cpu(x86_cpu_to_apicid, cpu) = apicid; -#endif -#ifdef CONFIG_X86_32 - if (cpu < 8) - early_per_cpu(x86_cpu_to_logical_apicid, cpu) = 1U << cpu; #endif set_cpu_possible(cpu, true); physid_set(apicid, phys_cpu_present_map); diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 29273e0151fc..aa51e16cb7d8 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -243,50 +243,32 @@ void default_send_IPI_self(int vector) } #ifdef CONFIG_X86_32 - -void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, - int vector) +void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector) { unsigned long flags; - unsigned int query_cpu; - - /* - * Hack. The clustered APIC addressing mode doesn't allow us to send - * to an arbitrary mask, so I do a unicasts to each CPU instead. This - * should be modified to do 1 message per cluster ID - mbligh - */ + unsigned int cpu; local_irq_save(flags); - for_each_cpu(query_cpu, mask) - __default_send_IPI_dest_field( - early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), - vector, APIC_DEST_LOGICAL); + for_each_cpu(cpu, mask) + __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL); local_irq_restore(flags); } void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector) { + unsigned int cpu, this_cpu = smp_processor_id(); unsigned long flags; - unsigned int query_cpu; - unsigned int this_cpu = smp_processor_id(); - - /* See Hack comment above */ local_irq_save(flags); - for_each_cpu(query_cpu, mask) { - if (query_cpu == this_cpu) + for_each_cpu(cpu, mask) { + if (cpu == this_cpu) continue; - __default_send_IPI_dest_field( - early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), - vector, APIC_DEST_LOGICAL); - } + __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL); + } local_irq_restore(flags); } -/* - * This is only used on smaller machines. - */ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector) { unsigned long mask = cpumask_bits(cpumask)[0]; diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index f6767d255c32..2c97bf7b56ae 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -184,10 +184,6 @@ void __init setup_per_cpu_areas(void) per_cpu(x86_cpu_to_acpiid, cpu) = early_per_cpu_map(x86_cpu_to_acpiid, cpu); #endif -#ifdef CONFIG_X86_32 - per_cpu(x86_cpu_to_logical_apicid, cpu) = - early_per_cpu_map(x86_cpu_to_logical_apicid, cpu); -#endif #ifdef CONFIG_NUMA per_cpu(x86_cpu_to_node_map, cpu) = early_per_cpu_map(x86_cpu_to_node_map, cpu); @@ -214,9 +210,6 @@ void __init setup_per_cpu_areas(void) early_per_cpu_ptr(x86_cpu_to_apicid) = NULL; early_per_cpu_ptr(x86_cpu_to_acpiid) = NULL; #endif -#ifdef CONFIG_X86_32 - early_per_cpu_ptr(x86_cpu_to_logical_apicid) = NULL; -#endif #ifdef CONFIG_NUMA early_per_cpu_ptr(x86_cpu_to_node_map) = NULL; #endif -- cgit v1.2.3 From ec9fb3c5f4a4fc96bd523061b43d1e2ec6cb24b1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:54 -0700 Subject: x86/apic/ipi: Code cleanup Remove completely useless and mindlessly copied comments and tidy up the code which causes eye bleed when looking at it. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/ipi.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index aa51e16cb7d8..2ff9cf9aa2ef 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -184,18 +184,13 @@ void default_send_IPI_single_phys(int cpu, int vector) void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector) { - unsigned long query_cpu; unsigned long flags; + unsigned long cpu; - /* - * Hack. The clustered APIC addressing mode doesn't allow us to send - * to an arbitrary mask, so I do a unicast to each CPU instead. - * - mbligh - */ local_irq_save(flags); - for_each_cpu(query_cpu, mask) { + for_each_cpu(cpu, mask) { __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, - query_cpu), vector, APIC_DEST_PHYSICAL); + cpu), vector, APIC_DEST_PHYSICAL); } local_irq_restore(flags); } @@ -203,18 +198,15 @@ void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector) void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask, int vector) { - unsigned int this_cpu = smp_processor_id(); - unsigned int query_cpu; + unsigned int cpu, this_cpu = smp_processor_id(); unsigned long flags; - /* See Hack comment above */ - local_irq_save(flags); - for_each_cpu(query_cpu, mask) { - if (query_cpu == this_cpu) + for_each_cpu(cpu, mask) { + if (cpu == this_cpu) continue; __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, - query_cpu), vector, APIC_DEST_PHYSICAL); + cpu), vector, APIC_DEST_PHYSICAL); } local_irq_restore(flags); } @@ -283,7 +275,6 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector) local_irq_restore(flags); } -/* must come after the send_IPI functions above for inlining */ static int convert_apicid_to_cpu(int apic_id) { int i; -- cgit v1.2.3 From e3243ed0142bb438bbd284c8c04f4c8e0c2ff498 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:54 -0700 Subject: x86/apic: Mop up early_per_cpu() abuse UV X2APIC uses the per CPU variable from: native_smp_prepare_cpus() uv_system_init() uv_system_init_hub() which is long after the per CPU areas have been set up. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/x2apic_uv_x.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 0d812fdc9b25..8b14451611d4 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -1843,7 +1843,7 @@ static void __init uv_system_init_hub(void) /* Initialize per CPU info: */ for_each_possible_cpu(cpu) { - int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); + int apicid = per_cpu(x86_cpu_to_apicid, cpu); unsigned short bid; unsigned short pnode; -- cgit v1.2.3 From d75baa260c850b3d184b1e3cceeff450d65e5603 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:55 -0700 Subject: x86/apic/32: Remove pointless default_acpi_madt_oem_check() On 32bit there is no APIC implementing the acpi_madt_oem_check() except XEN PV, but that does not matter at all. generic_apic_probe() runs before ACPI tables are parsed. This selects the XEN APIC if there is no command line override because the XEN APIC driver is the first to be probed. If there is a command line override then the XEN PV driver won't be selected in the MADT OEM check either. As there is no other MADT check implemented for 32bit APICs, this whole excercise is a NOOP and can be removed. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 4 +++- arch/x86/kernel/apic/bigsmp_32.c | 1 - arch/x86/kernel/apic/probe_32.c | 22 ---------------------- 3 files changed, 3 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index daef6812141f..f6541a22bc69 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -459,10 +459,12 @@ static inline unsigned int read_apic_id(void) #ifdef CONFIG_X86_64 typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip); extern void acpi_wake_cpu_handler_update(wakeup_cpu_handler handler); +extern int default_acpi_madt_oem_check(char *, char *); +#else +static inline int default_acpi_madt_oem_check(char *a, char *b) { return 0; } #endif extern int default_apic_id_valid(u32 apicid); -extern int default_acpi_madt_oem_check(char *, char *); extern void default_setup_apic_routing(void); extern u32 apic_default_calc_apicid(unsigned int cpu); diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index b649048c2019..d19dcf99bd05 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -112,7 +112,6 @@ static struct apic apic_bigsmp __ro_after_init = { .name = "bigsmp", .probe = probe_bigsmp, - .acpi_madt_oem_check = NULL, .apic_id_valid = default_apic_id_valid, .apic_id_registered = bigsmp_apic_id_registered, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 3ee0211b0c29..20d7f571b80f 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -60,7 +60,6 @@ static struct apic apic_default __ro_after_init = { .name = "default", .probe = probe_default, - .acpi_madt_oem_check = NULL, .apic_id_valid = default_apic_id_valid, .apic_id_registered = default_apic_id_registered, @@ -176,24 +175,3 @@ void __init generic_apic_probe(void) } printk(KERN_INFO "Using APIC driver %s\n", apic->name); } - -/* This function can switch the APIC even after the initial ->probe() */ -int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id) -{ - struct apic **drv; - - for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { - if (!(*drv)->acpi_madt_oem_check) - continue; - if (!(*drv)->acpi_madt_oem_check(oem_id, oem_table_id)) - continue; - - if (!cmdline_apic) { - apic = *drv; - printk(KERN_INFO "Switched to APIC driver `%s'.\n", - apic->name); - } - return 1; - } - return 0; -} -- cgit v1.2.3 From 79c9a17c16e95e3c9e8646cf869fabea53bf283f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:56 -0700 Subject: x86/apic/32: Decrapify the def_bigsmp mechanism If the system has more than 8 CPUs then XAPIC and the bigsmp APIC driver is required. This is ensured via: 1) Enumerating all possible CPUs up to NR_CPUS 2) Checking at boot CPU APIC setup time whether the system has more than 8 CPUs and has an XAPIC. If that's the case then it's attempted to install the bigsmp APIC driver and a magic variable 'def_to_bigsmp' is set to one. 3) If that magic variable is set and CONFIG_X86_BIGSMP=n and the system has more than 8 CPUs smp_sanity_check() removes all CPUs >= #8 from the present and possible mask in the most convoluted way. This logic is completely broken for the case where the bigsmp driver is enabled, but not selected due to a command line option specifying the default APIC. In that case the system boots with default APIC in logical destination mode and fails to reduce the number of CPUs. That aside the above which is sprinkled over 3 different places is yet another piece of art. It would have been too obvious to check the requirements upfront and limit nr_cpu_ids _before_ enumerating tons of CPUs and then removing them again. Implement exactly this. Check the bigsmp requirement when the boot APIC is registered which happens _before_ ACPI/MPTABLE parsing and limit the number of CPUs to 8 if it can't be used. Switch it over when the boot CPU apic is set up if necessary. [ dhansen: fix nr_cpu_ids off-by-one in default_setup_apic_routing() ] Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/mpspec.h | 2 -- arch/x86/kernel/apic/apic.c | 3 +++ arch/x86/kernel/apic/bigsmp_32.c | 22 ++++++++++------------ arch/x86/kernel/apic/local.h | 11 +++++++++++ arch/x86/kernel/apic/probe_32.c | 35 +++++++++++++++++------------------ arch/x86/kernel/setup.c | 1 - arch/x86/kernel/smpboot.c | 37 ------------------------------------- 7 files changed, 41 insertions(+), 70 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h index 5d01bdee0186..f46df8349e86 100644 --- a/arch/x86/include/asm/mpspec.h +++ b/arch/x86/include/asm/mpspec.h @@ -23,8 +23,6 @@ extern int pic_mode; #define MAX_IRQ_SOURCES 256 -extern unsigned int def_to_bigsmp; - #else /* CONFIG_X86_64: */ #define MAX_MP_BUSSES 256 diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 224dff4b9258..24912b2279de 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -63,6 +63,8 @@ #include #include +#include "local.h" + unsigned int num_processors; unsigned disabled_cpus; @@ -2420,6 +2422,7 @@ static __init void cpu_set_boot_apic(void) { cpuid_to_apicid[0] = boot_cpu_physical_apicid; cpu_update_apic(0, boot_cpu_physical_apicid); + x86_32_probe_bigsmp_early(); } int generic_processor_info(int apicid) diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index d19dcf99bd05..a26cb4684017 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -100,12 +100,7 @@ static const struct dmi_system_id bigsmp_dmi_table[] = { static int probe_bigsmp(void) { - if (def_to_bigsmp) - dmi_bigsmp = 1; - else - dmi_check_system(bigsmp_dmi_table); - - return dmi_bigsmp; + return dmi_check_system(bigsmp_dmi_table); } static struct apic apic_bigsmp __ro_after_init = { @@ -149,14 +144,17 @@ static struct apic apic_bigsmp __ro_after_init = { .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, }; -void __init generic_bigsmp_probe(void) +bool __init apic_bigsmp_possible(bool cmdline_override) { - if (!probe_bigsmp()) - return; - - apic = &apic_bigsmp; + return apic == &apic_bigsmp || !cmdline_override; +} - pr_info("Overriding APIC driver with %s\n", apic_bigsmp.name); +void __init apic_bigsmp_force(void) +{ + if (apic != &apic_bigsmp) { + apic = &apic_bigsmp; + pr_info("Overriding APIC driver with bigsmp\n"); + } } apic_driver(apic_bigsmp); diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index a997d849509a..ad96c1aece2f 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -66,4 +66,15 @@ void default_send_IPI_self(int vector); void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector); void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector); void default_send_IPI_mask_logical(const struct cpumask *mask, int vector); +void x86_32_probe_bigsmp_early(void); +#else +static inline void x86_32_probe_bigsmp_early(void) { } +#endif + +#ifdef CONFIG_X86_BIGSMP +bool apic_bigsmp_possible(bool cmdline_selected); +void apic_bigsmp_force(void); +#else +static inline bool apic_bigsmp_possible(bool cmdline_selected) { return false; }; +static inline void apic_bigsmp_force(void) { } #endif diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 20d7f571b80f..706a8447ac17 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -10,6 +10,8 @@ #include #include +#include + #include #include #include @@ -123,36 +125,33 @@ static int __init parse_apic(char *arg) } early_param("apic", parse_apic); -void __init default_setup_apic_routing(void) +void __init x86_32_probe_bigsmp_early(void) { - int version = boot_cpu_apic_version; + if (nr_cpu_ids <= 8 || xen_pv_domain()) + return; - if (num_possible_cpus() > 8) { + if (IS_ENABLED(CONFIG_X86_BIGSMP)) { switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_INTEL: - if (!APIC_XAPIC(version)) { - def_to_bigsmp = 0; + if (!APIC_XAPIC(boot_cpu_apic_version)) break; - } /* P4 and above */ fallthrough; case X86_VENDOR_HYGON: case X86_VENDOR_AMD: - def_to_bigsmp = 1; + if (apic_bigsmp_possible(cmdline_apic)) + return; + break; } } + pr_info("Limiting to 8 possible CPUs\n"); + set_nr_cpu_ids(8); +} -#ifdef CONFIG_X86_BIGSMP - /* - * This is used to switch to bigsmp mode when - * - There is no apic= option specified by the user - * - generic_apic_probe() has chosen apic_default as the sub_arch - * - we find more than 8 CPUs in acpi LAPIC listing with xAPIC support - */ - - if (!cmdline_apic && apic == &apic_default) - generic_bigsmp_probe(); -#endif +void __init default_setup_apic_routing(void) +{ + if (nr_cpu_ids > 8 && !xen_pv_domain()) + apic_bigsmp_force(); if (apic->setup_apic_routing) apic->setup_apic_routing(); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index a5ef400928da..779477d92caa 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -114,7 +114,6 @@ static struct resource bss_resource = { #ifdef CONFIG_X86_32 /* CPU data as detected by the assembly code in head_32.S */ struct cpuinfo_x86 new_cpu_data; -unsigned int def_to_bigsmp; struct apm_info apm_info; EXPORT_SYMBOL(apm_info); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index da795f743542..df1c5f01411b 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1149,41 +1149,6 @@ static __init void disable_smp(void) cpumask_set_cpu(0, topology_die_cpumask(0)); } -/* - * Various sanity checks. - */ -static void __init smp_sanity_check(void) -{ - preempt_disable(); - -#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32) - if (def_to_bigsmp && nr_cpu_ids > 8) { - unsigned int cpu; - unsigned nr; - - pr_warn("More than 8 CPUs detected - skipping them\n" - "Use CONFIG_X86_BIGSMP\n"); - - nr = 0; - for_each_present_cpu(cpu) { - if (nr >= 8) - set_cpu_present(cpu, false); - nr++; - } - - nr = 0; - for_each_possible_cpu(cpu) { - if (nr >= 8) - set_cpu_possible(cpu, false); - nr++; - } - - set_nr_cpu_ids(8); - } -#endif - preempt_enable(); -} - static void __init smp_cpu_index_default(void) { int i; @@ -1243,8 +1208,6 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) { smp_prepare_cpus_common(); - smp_sanity_check(); - switch (apic_intr_mode) { case APIC_PIC: case APIC_VIRTUAL_WIRE_NO_CONFIG: -- cgit v1.2.3 From 4114e1686fdc350e4191ca670710b3fa9bd10947 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:56 -0700 Subject: x86/apic/32: Remove bigsmp_cpu_present_to_apicid() It's a copy of default_cpu_present_to_apicid() with the omission of the actual check whether the CPU is present. This APIC callback should die completely, but the XEN APIC implementation does something different which needs to be addressed first. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/bigsmp_32.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index a26cb4684017..aba8ce1d3f24 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -43,14 +43,6 @@ static void bigsmp_setup_apic_routing(void) nr_ioapics); } -static int bigsmp_cpu_present_to_apicid(int mps_cpu) -{ - if (mps_cpu < nr_cpu_ids) - return (int) per_cpu(x86_cpu_to_apicid, mps_cpu); - - return BAD_APICID; -} - static void bigsmp_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap) { /* For clustered we don't have a good way to do this yet - hack */ @@ -119,7 +111,7 @@ static struct apic apic_bigsmp __ro_after_init = { .init_apic_ldr = bigsmp_init_apic_ldr, .ioapic_phys_id_map = bigsmp_ioapic_phys_id_map, .setup_apic_routing = bigsmp_setup_apic_routing, - .cpu_present_to_apicid = bigsmp_cpu_present_to_apicid, + .cpu_present_to_apicid = default_cpu_present_to_apicid, .apicid_to_cpu_present = physid_set_mask_of_physid, .phys_pkg_id = bigsmp_phys_pkg_id, -- cgit v1.2.3 From 2f6df03f80a13ab2e1eb4bf00bbd5e63255756db Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:58 -0700 Subject: x86/apic: Nuke empty init_apic_ldr() callbacks apic::init_apic_ldr() is only invoked when the APIC is initialized. So there is really no point in having: - Default empty callbacks all over the place - Two implementations of the actual LDR init function where one is just unreadable gunk but does exactly the same as the other. Make the apic::init_apic_ldr() invocation conditional, remove the empty callbacks and consolidate the two implementation into one. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 -- arch/x86/kernel/apic/apic.c | 7 +++++-- arch/x86/kernel/apic/apic_common.c | 16 ++++++++++++++++ arch/x86/kernel/apic/apic_flat_64.c | 32 +------------------------------- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 2 -- arch/x86/kernel/apic/bigsmp_32.c | 9 --------- arch/x86/kernel/apic/local.h | 5 ++--- arch/x86/kernel/apic/probe_32.c | 15 --------------- arch/x86/kernel/apic/x2apic_phys.c | 5 ----- arch/x86/kernel/apic/x2apic_uv_x.c | 5 ----- arch/x86/xen/apic.c | 1 - 12 files changed, 24 insertions(+), 77 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index f6541a22bc69..f8a36319626a 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -445,8 +445,6 @@ extern void generic_bigsmp_probe(void); #include -#define APIC_DFR_VALUE (APIC_DFR_FLAT) - extern struct apic apic_noop; static inline unsigned int read_apic_id(void) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 24912b2279de..622dd60229e5 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1578,9 +1578,12 @@ static void setup_local_APIC(void) /* * Intel recommends to set DFR, LDR and TPR before enabling * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel - * document number 292116). So here it goes... + * document number 292116). + * + * Except for APICs which operate in physical destination mode. */ - apic->init_apic_ldr(); + if (apic->init_apic_ldr) + apic->init_apic_ldr(); /* * Set Task Priority to 'accept all except vectors 0-31'. An APIC diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index 41528bb3a608..d55fdb33b8e5 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -6,6 +6,8 @@ #include #include +#include "local.h" + u32 apic_default_calc_apicid(unsigned int cpu) { return per_cpu(x86_cpu_to_apicid, cpu); @@ -39,3 +41,17 @@ int default_apic_id_valid(u32 apicid) { return (apicid < 255); } + +/* + * Set up the logical destination ID when the APIC operates in logical + * destination mode. + */ +void default_init_apic_ldr(void) +{ + unsigned long val; + + apic_write(APIC_DFR, APIC_DFR_FLAT); + val = apic_read(APIC_LDR) & ~APIC_LDR_MASK; + val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id()); + apic_write(APIC_LDR, val); +} diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 00482cf8255a..4ddced9005d3 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -28,26 +28,6 @@ static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id) return 1; } -/* - * Set up the logical destination ID. - * - * Intel recommends to set DFR, LDR and TPR before enabling - * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel - * document number 292116). So here it goes... - */ -void flat_init_apic_ldr(void) -{ - unsigned long val; - unsigned long num, id; - - num = smp_processor_id(); - id = 1UL << num; - apic_write(APIC_DFR, APIC_DFR_FLAT); - val = apic_read(APIC_LDR) & ~APIC_LDR_MASK; - val |= SET_APIC_LOGICAL_ID(id); - apic_write(APIC_LDR, val); -} - static void _flat_send_IPI_mask(unsigned long mask, int vector) { unsigned long flags; @@ -119,7 +99,7 @@ static struct apic apic_flat __ro_after_init = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = flat_init_apic_ldr, + .init_apic_ldr = default_init_apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, @@ -175,15 +155,6 @@ static int physflat_acpi_madt_oem_check(char *oem_id, char *oem_table_id) return 0; } -static void physflat_init_apic_ldr(void) -{ - /* - * LDR and DFR are not involved in physflat mode, rather: - * "In physical destination mode, the destination processor is - * specified by its local APIC ID [...]." (Intel SDM, 10.6.2.1) - */ -} - static int physflat_probe(void) { if (apic == &apic_physflat || num_possible_cpus() > 8 || @@ -207,7 +178,6 @@ static struct apic apic_physflat __ro_after_init = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = physflat_init_apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index ccd74e8016fc..dee9e9c73ad9 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -14,7 +14,6 @@ #include -static void noop_init_apic_ldr(void) { } static void noop_send_IPI(int cpu, int vector) { } static void noop_send_IPI_mask(const struct cpumask *cpumask, int vector) { } static void noop_send_IPI_mask_allbutself(const struct cpumask *cpumask, int vector) { } @@ -94,7 +93,6 @@ struct apic apic_noop __ro_after_init = { .disable_esr = 0, .check_apicid_used = default_check_apicid_used, - .init_apic_ldr = noop_init_apic_ldr, .ioapic_phys_id_map = default_ioapic_phys_id_map, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index de16fde97568..d96d74cc8e8e 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -252,7 +252,6 @@ static const struct apic apic_numachip1 __refconst = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = flat_init_apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, @@ -297,7 +296,6 @@ static const struct apic apic_numachip2 __refconst = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = flat_init_apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index aba8ce1d3f24..c688af49bf98 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -28,14 +28,6 @@ static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid) return false; } -/* - * bigsmp enables physical destination mode - * and doesn't use LDR and DFR - */ -static void bigsmp_init_apic_ldr(void) -{ -} - static void bigsmp_setup_apic_routing(void) { printk(KERN_INFO @@ -108,7 +100,6 @@ static struct apic apic_bigsmp __ro_after_init = { .disable_esr = 1, .check_apicid_used = bigsmp_check_apicid_used, - .init_apic_ldr = bigsmp_init_apic_ldr, .ioapic_phys_id_map = bigsmp_ioapic_phys_id_map, .setup_apic_routing = bigsmp_setup_apic_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index ad96c1aece2f..c14674146cfe 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -13,9 +13,6 @@ #include #include -/* APIC flat 64 */ -void flat_init_apic_ldr(void); - /* X2APIC */ int x2apic_apic_id_valid(u32 apicid); int x2apic_apic_id_registered(void); @@ -46,6 +43,8 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, return icr; } +void default_init_apic_ldr(void); + void __default_send_IPI_shortcut(unsigned int shortcut, int vector); /* diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 706a8447ac17..68fc220efb95 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -32,21 +32,6 @@ static int default_apic_id_registered(void) return physid_isset(read_apic_id(), phys_cpu_present_map); } -/* - * Set up the logical destination ID. Intel recommends to set DFR, LDR and - * TPR before enabling an APIC. See e.g. "AP-388 82489DX User's Manual" - * (Intel document number 292116). - */ -static void default_init_apic_ldr(void) -{ - unsigned long val; - - apic_write(APIC_DFR, APIC_DFR_VALUE); - val = apic_read(APIC_LDR) & ~APIC_LDR_MASK; - val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id()); - apic_write(APIC_LDR, val); -} - static int default_phys_pkg_id(int cpuid_apic, int index_msb) { return cpuid_apic >> index_msb; diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 9cf32ce779be..c3c2e9814ea1 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -91,10 +91,6 @@ static void x2apic_send_IPI_all(int vector) __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC); } -static void init_x2apic_ldr(void) -{ -} - static int x2apic_phys_probe(void) { if (!x2apic_mode) @@ -169,7 +165,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = init_x2apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 8b14451611d4..ab17e588fd97 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -788,10 +788,6 @@ static int uv_apic_id_registered(void) return 1; } -static void uv_init_apic_ldr(void) -{ -} - static u32 apic_uv_calc_apicid(unsigned int cpu) { return apic_default_calc_apicid(cpu); @@ -841,7 +837,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .disable_esr = 0, .check_apicid_used = NULL, - .init_apic_ldr = uv_init_apic_ldr, .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index c8240afca77c..1ff8f75e5c88 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -149,7 +149,6 @@ static struct apic xen_pv_apic = { .disable_esr = 0, .check_apicid_used = default_check_apicid_used, /* Used on 32-bit */ - .init_apic_ldr = xen_noop, /* setup_local_APIC calls it */ .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ .setup_apic_routing = NULL, .cpu_present_to_apicid = xen_cpu_present_to_apicid, -- cgit v1.2.3 From 9a2a637af06621e69893e5d5cde3c0eba5aeba0f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:59 -0700 Subject: x86/apic: Nuke apic::apicid_to_cpu_present() This is only used on 32bit and is a wrapper around physid_set_mask_of_physid() in all 32bit APIC drivers. Remove the callback and use physid_set_mask_of_physid() in the code directly, Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 1 - arch/x86/kernel/apic/apic_flat_64.c | 2 -- arch/x86/kernel/apic/apic_noop.c | 1 - arch/x86/kernel/apic/apic_numachip.c | 2 -- arch/x86/kernel/apic/bigsmp_32.c | 1 - arch/x86/kernel/apic/io_apic.c | 11 +++++------ arch/x86/kernel/apic/probe_32.c | 1 - arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 1 - arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/xen/apic.c | 1 - 11 files changed, 5 insertions(+), 18 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index f8a36319626a..3932089cb9af 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -304,7 +304,6 @@ struct apic { void (*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap); void (*setup_apic_routing)(void); int (*cpu_present_to_apicid)(int mps_cpu); - void (*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap); int (*phys_pkg_id)(int cpuid_apic, int index_msb); u32 (*get_apic_id)(unsigned long x); diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 4ddced9005d3..803cf0123f82 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -103,7 +103,6 @@ static struct apic apic_flat __ro_after_init = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = flat_phys_pkg_id, .get_apic_id = flat_get_apic_id, @@ -181,7 +180,6 @@ static struct apic apic_physflat __ro_after_init = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = flat_phys_pkg_id, .get_apic_id = flat_get_apic_id, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index dee9e9c73ad9..8cab632ed273 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -96,7 +96,6 @@ struct apic apic_noop __ro_after_init = { .ioapic_phys_id_map = default_ioapic_phys_id_map, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = physid_set_mask_of_physid, .phys_pkg_id = noop_phys_pkg_id, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index d96d74cc8e8e..8f57155d5f30 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -255,7 +255,6 @@ static const struct apic apic_numachip1 __refconst = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = numachip_phys_pkg_id, .get_apic_id = numachip1_get_apic_id, @@ -299,7 +298,6 @@ static const struct apic apic_numachip2 __refconst = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = numachip_phys_pkg_id, .get_apic_id = numachip2_get_apic_id, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index c688af49bf98..099611e4ea06 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -103,7 +103,6 @@ static struct apic apic_bigsmp __ro_after_init = { .ioapic_phys_id_map = bigsmp_ioapic_phys_id_map, .setup_apic_routing = bigsmp_setup_apic_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = physid_set_mask_of_physid, .phys_pkg_id = bigsmp_phys_pkg_id, .get_apic_id = bigsmp_get_apic_id, diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index a8b329aab672..8c470995d8b4 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1512,11 +1512,10 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void) ioapics[ioapic_idx].mp_config.apicid = i; } else { physid_mask_t tmp; - apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx), - &tmp); - apic_printk(APIC_VERBOSE, "Setting %d in the " - "phys_id_present_map\n", - mpc_ioapic_id(ioapic_idx)); + + physid_set_mask_of_physid(mpc_ioapic_id(ioapic_idx), &tmp); + apic_printk(APIC_VERBOSE, "Setting %d in the phys_id_present_map\n", + mpc_ioapic_id(ioapic_idx)); physids_or(phys_id_present_map, phys_id_present_map, tmp); } @@ -2546,7 +2545,7 @@ static int io_apic_get_unique_id(int ioapic, int apic_id) apic_id = i; } - apic->apicid_to_cpu_present(apic_id, &tmp); + physid_set_mask_of_physid(apic_id, &tmp); physids_or(apic_id_map, apic_id_map, tmp); if (reg_00.bits.ID != apic_id) { diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 68fc220efb95..c6c3a4b2fcfa 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -60,7 +60,6 @@ static struct apic apic_default __ro_after_init = { .ioapic_phys_id_map = default_ioapic_phys_id_map, .setup_apic_routing = setup_apic_flat_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = physid_set_mask_of_physid, .phys_pkg_id = default_phys_pkg_id, .get_apic_id = default_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 931e0b9950f3..a8e7a8f8f22a 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -249,7 +249,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = x2apic_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index c3c2e9814ea1..5ac212940901 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -168,7 +168,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = x2apic_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index ab17e588fd97..d6b98bc0745a 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -840,7 +840,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .ioapic_phys_id_map = NULL, .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, - .apicid_to_cpu_present = NULL, .phys_pkg_id = uv_phys_pkg_id, .get_apic_id = x2apic_get_apic_id, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 1ff8f75e5c88..426f4a0ac664 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -152,7 +152,6 @@ static struct apic xen_pv_apic = { .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ .setup_apic_routing = NULL, .cpu_present_to_apicid = xen_cpu_present_to_apicid, - .apicid_to_cpu_present = physid_set_mask_of_physid, /* Used on 32-bit */ .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ .get_apic_id = xen_get_apic_id, -- cgit v1.2.3 From 9faee3ecbfedebc6cdd0d96bfb2788ad8d62ef7e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:03:59 -0700 Subject: x86/ioapic/32: Decrapify phys_id_present_map operation The operation to set the IOAPIC ID in phys_id_present_map is as convoluted as it can be. 1) Allocate a bitmap of 32byte size on the stack 2) Zero the bitmap and set the IOAPIC ID bit 3) Or the temporary bitmap over phys_id_present_map The same functionality can be achieved by setting the IOAPIC ID bit directly in the phys_id_present_map. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/io_apic.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 8c470995d8b4..ff2d27e2089a 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1511,12 +1511,9 @@ void __init setup_ioapic_ids_from_mpc_nocheck(void) physid_set(i, phys_id_present_map); ioapics[ioapic_idx].mp_config.apicid = i; } else { - physid_mask_t tmp; - - physid_set_mask_of_physid(mpc_ioapic_id(ioapic_idx), &tmp); apic_printk(APIC_VERBOSE, "Setting %d in the phys_id_present_map\n", mpc_ioapic_id(ioapic_idx)); - physids_or(phys_id_present_map, phys_id_present_map, tmp); + physid_set(mpc_ioapic_id(ioapic_idx), phys_id_present_map); } /* -- cgit v1.2.3 From 9d87f5b67e100f253aed995a31fedff2983b7639 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:01 -0700 Subject: x86/apic: Mop up *setup_apic_routing() default_setup_apic_routing() is a complete misnomer. On 64bit it does the actual APIC probing and on 32bit it is used to force select the bigsmp APIC and to emit a redundant message in the apic::setup_apic_routing() callback. Rename the 64bit and 32bit function so they reflect what they are doing and remove the useless APIC callback. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 10 ++++------ arch/x86/kernel/apic/apic.c | 4 +++- arch/x86/kernel/apic/apic_flat_64.c | 2 -- arch/x86/kernel/apic/apic_noop.c | 1 - arch/x86/kernel/apic/apic_numachip.c | 2 -- arch/x86/kernel/apic/bigsmp_32.c | 8 -------- arch/x86/kernel/apic/local.h | 2 ++ arch/x86/kernel/apic/probe_32.c | 17 ++--------------- arch/x86/kernel/apic/probe_64.c | 2 +- arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 1 - arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/kernel/setup.c | 2 +- arch/x86/xen/apic.c | 1 - 14 files changed, 13 insertions(+), 41 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 3932089cb9af..83ecea3baaaf 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -40,11 +40,9 @@ #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) -extern void generic_apic_probe(void); +extern void x86_32_probe_apic(void); #else -static inline void generic_apic_probe(void) -{ -} +static inline void x86_32_probe_apic(void) { } #endif #ifdef CONFIG_X86_LOCAL_APIC @@ -302,7 +300,6 @@ struct apic { bool (*check_apicid_used)(physid_mask_t *map, int apicid); void (*init_apic_ldr)(void); void (*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap); - void (*setup_apic_routing)(void); int (*cpu_present_to_apicid)(int mps_cpu); int (*phys_pkg_id)(int cpuid_apic, int index_msb); @@ -457,12 +454,13 @@ static inline unsigned int read_apic_id(void) typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip); extern void acpi_wake_cpu_handler_update(wakeup_cpu_handler handler); extern int default_acpi_madt_oem_check(char *, char *); +extern void x86_64_probe_apic(void); #else static inline int default_acpi_madt_oem_check(char *a, char *b) { return 0; } +static inline void x86_64_probe_apic(void) { } #endif extern int default_apic_id_valid(u32 apicid); -extern void default_setup_apic_routing(void); extern u32 apic_default_calc_apicid(unsigned int cpu); extern u32 apic_flat_calc_apicid(unsigned int cpu); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 622dd60229e5..c2656a3ffbec 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1419,7 +1419,9 @@ void __init apic_intr_mode_init(void) break; } - default_setup_apic_routing(); + x86_64_probe_apic(); + + x86_32_install_bigsmp(); if (x86_platform.apic_post_init) x86_platform.apic_post_init(); diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 803cf0123f82..83e02b5796e6 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -101,7 +101,6 @@ static struct apic apic_flat __ro_after_init = { .check_apicid_used = NULL, .init_apic_ldr = default_init_apic_ldr, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = flat_phys_pkg_id, @@ -178,7 +177,6 @@ static struct apic apic_physflat __ro_after_init = { .check_apicid_used = NULL, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = flat_phys_pkg_id, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 8cab632ed273..e7309fd9cb55 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -94,7 +94,6 @@ struct apic apic_noop __ro_after_init = { .check_apicid_used = default_check_apicid_used, .ioapic_phys_id_map = default_ioapic_phys_id_map, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = noop_phys_pkg_id, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 8f57155d5f30..b0d5a6ebc379 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -253,7 +253,6 @@ static const struct apic apic_numachip1 __refconst = { .check_apicid_used = NULL, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, @@ -296,7 +295,6 @@ static const struct apic apic_numachip2 __refconst = { .check_apicid_used = NULL, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 099611e4ea06..9bbdd40107f7 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -28,13 +28,6 @@ static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid) return false; } -static void bigsmp_setup_apic_routing(void) -{ - printk(KERN_INFO - "Enabling APIC mode: Physflat. Using %d I/O APICs\n", - nr_ioapics); -} - static void bigsmp_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap) { /* For clustered we don't have a good way to do this yet - hack */ @@ -101,7 +94,6 @@ static struct apic apic_bigsmp __ro_after_init = { .check_apicid_used = bigsmp_check_apicid_used, .ioapic_phys_id_map = bigsmp_ioapic_phys_id_map, - .setup_apic_routing = bigsmp_setup_apic_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = bigsmp_phys_pkg_id, diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index c14674146cfe..345acdeab448 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -66,8 +66,10 @@ void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vect void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector); void default_send_IPI_mask_logical(const struct cpumask *mask, int vector); void x86_32_probe_bigsmp_early(void); +void x86_32_install_bigsmp(void); #else static inline void x86_32_probe_bigsmp_early(void) { } +static inline void x86_32_install_bigsmp(void) { } #endif #ifdef CONFIG_X86_BIGSMP diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index c6c3a4b2fcfa..3f717f4c72bc 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -18,15 +18,6 @@ #include "local.h" -static void setup_apic_flat_routing(void) -{ -#ifdef CONFIG_X86_IO_APIC - printk(KERN_INFO - "Enabling APIC mode: Flat. Using %d I/O APICs\n", - nr_ioapics); -#endif -} - static int default_apic_id_registered(void) { return physid_isset(read_apic_id(), phys_cpu_present_map); @@ -58,7 +49,6 @@ static struct apic apic_default __ro_after_init = { .check_apicid_used = default_check_apicid_used, .init_apic_ldr = default_init_apic_ldr, .ioapic_phys_id_map = default_ioapic_phys_id_map, - .setup_apic_routing = setup_apic_flat_routing, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = default_phys_pkg_id, @@ -132,16 +122,13 @@ void __init x86_32_probe_bigsmp_early(void) set_nr_cpu_ids(8); } -void __init default_setup_apic_routing(void) +void __init x86_32_install_bigsmp(void) { if (nr_cpu_ids > 8 && !xen_pv_domain()) apic_bigsmp_force(); - - if (apic->setup_apic_routing) - apic->setup_apic_routing(); } -void __init generic_apic_probe(void) +void __init x86_32_probe_apic(void) { if (!cmdline_apic) { struct apic **drv; diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c index 3c0c8d32be10..7c9f1a8a83c8 100644 --- a/arch/x86/kernel/apic/probe_64.c +++ b/arch/x86/kernel/apic/probe_64.c @@ -14,7 +14,7 @@ #include "local.h" /* Select the appropriate APIC driver */ -void __init default_setup_apic_routing(void) +void __init x86_64_probe_apic(void) { struct apic **drv; diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index a8e7a8f8f22a..52d4be15a7dc 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -247,7 +247,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .check_apicid_used = NULL, .init_apic_ldr = init_x2apic_ldr, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = x2apic_phys_pkg_id, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 5ac212940901..c57b5c82b14a 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -166,7 +166,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .check_apicid_used = NULL, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = x2apic_phys_pkg_id, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index d6b98bc0745a..a8caff181adf 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -838,7 +838,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .check_apicid_used = NULL, .ioapic_phys_id_map = NULL, - .setup_apic_routing = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = uv_phys_pkg_id, diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 779477d92caa..f83d02b4fb25 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1252,7 +1252,7 @@ void __init setup_arch(char **cmdline_p) map_vsyscall(); - generic_apic_probe(); + x86_32_probe_apic(); early_quirks(); diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 426f4a0ac664..4a27f92b51d1 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -150,7 +150,6 @@ static struct apic xen_pv_apic = { .check_apicid_used = default_check_apicid_used, /* Used on 32-bit */ .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ - .setup_apic_routing = NULL, .cpu_present_to_apicid = xen_cpu_present_to_apicid, .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ -- cgit v1.2.3 From 5a3a46bd161d76f117b030ece0eb0de0dfe3ba59 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:02 -0700 Subject: x86/apic: Mop up apic::apic_id_registered() Really not a hotpath and again no reason for having a gazillion of empty callbacks returning 1. Make it return bool and provide one shared implementation for the remaining users. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 +- arch/x86/kernel/apic/apic.c | 7 ++----- arch/x86/kernel/apic/apic_common.c | 5 +++++ arch/x86/kernel/apic/apic_flat_64.c | 14 ++------------ arch/x86/kernel/apic/apic_noop.c | 12 ------------ arch/x86/kernel/apic/apic_numachip.c | 7 ------- arch/x86/kernel/apic/bigsmp_32.c | 6 ------ arch/x86/kernel/apic/local.h | 3 ++- arch/x86/kernel/apic/probe_32.c | 5 ----- arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 6 ------ arch/x86/kernel/apic/x2apic_uv_x.c | 6 ------ arch/x86/xen/apic.c | 6 ------ 13 files changed, 12 insertions(+), 68 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 83ecea3baaaf..4fb71b7d597a 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -295,7 +295,7 @@ struct apic { int (*probe)(void); int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); int (*apic_id_valid)(u32 apicid); - int (*apic_id_registered)(void); + bool (*apic_id_registered)(void); bool (*check_apicid_used)(physid_mask_t *map, int apicid); void (*init_apic_ldr)(void); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index c2656a3ffbec..4ee95cb2178e 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1571,11 +1571,8 @@ static void setup_local_APIC(void) apic_write(APIC_ESR, 0); } #endif - /* - * Double-check whether this APIC is really registered. - * This is meaningless in clustered apic mode, so we skip it. - */ - BUG_ON(!apic->apic_id_registered()); + /* Validate that the APIC is registered if required */ + BUG_ON(apic->apic_id_registered && !apic->apic_id_registered()); /* * Intel recommends to set DFR, LDR and TPR before enabling diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index d55fdb33b8e5..bbb5d5a6c820 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -42,6 +42,11 @@ int default_apic_id_valid(u32 apicid) return (apicid < 255); } +bool default_apic_id_registered(void) +{ + return physid_isset(read_apic_id(), phys_cpu_present_map); +} + /* * Set up the logical destination ID when the APIC operates in logical * destination mode. diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 83e02b5796e6..a0c875d6ef3e 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -66,16 +66,6 @@ static u32 set_apic_id(unsigned int id) return (id & 0xFF) << 24; } -static unsigned int read_xapic_id(void) -{ - return flat_get_apic_id(apic_read(APIC_ID)); -} - -static int flat_apic_id_registered(void) -{ - return physid_isset(read_xapic_id(), phys_cpu_present_map); -} - static int flat_phys_pkg_id(int initial_apic_id, int index_msb) { return initial_apic_id >> index_msb; @@ -91,7 +81,7 @@ static struct apic apic_flat __ro_after_init = { .probe = flat_probe, .acpi_madt_oem_check = flat_acpi_madt_oem_check, .apic_id_valid = default_apic_id_valid, - .apic_id_registered = flat_apic_id_registered, + .apic_id_registered = default_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, @@ -168,7 +158,7 @@ static struct apic apic_physflat __ro_after_init = { .probe = physflat_probe, .acpi_madt_oem_check = physflat_acpi_madt_oem_check, .apic_id_valid = default_apic_id_valid, - .apic_id_registered = flat_apic_id_registered, + .apic_id_registered = default_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index e7309fd9cb55..c73a7a811bb9 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -57,17 +57,6 @@ static int noop_probe(void) return 0; } -static int noop_apic_id_registered(void) -{ - /* - * if we would be really "pedantic" - * we should pass read_apic_id() here - * but since NOOP suppose APIC ID = 0 - * lets save a few cycles - */ - return physid_isset(0, phys_cpu_present_map); -} - static u32 noop_apic_read(u32 reg) { WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !apic_is_disabled); @@ -85,7 +74,6 @@ struct apic apic_noop __ro_after_init = { .acpi_madt_oem_check = NULL, .apic_id_valid = default_apic_id_valid, - .apic_id_registered = noop_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index b0d5a6ebc379..e8e13d7c84d7 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -62,11 +62,6 @@ static int numachip_apic_id_valid(u32 apicid) return 1; } -static int numachip_apic_id_registered(void) -{ - return 1; -} - static int numachip_phys_pkg_id(int initial_apic_id, int index_msb) { return initial_apic_id >> index_msb; @@ -244,7 +239,6 @@ static const struct apic apic_numachip1 __refconst = { .probe = numachip1_probe, .acpi_madt_oem_check = numachip1_acpi_madt_oem_check, .apic_id_valid = numachip_apic_id_valid, - .apic_id_registered = numachip_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, @@ -286,7 +280,6 @@ static const struct apic apic_numachip2 __refconst = { .probe = numachip2_probe, .acpi_madt_oem_check = numachip2_acpi_madt_oem_check, .apic_id_valid = numachip_apic_id_valid, - .apic_id_registered = numachip_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 9bbdd40107f7..ffff294f4b44 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -18,11 +18,6 @@ static unsigned bigsmp_get_apic_id(unsigned long x) return (x >> 24) & 0xFF; } -static int bigsmp_apic_id_registered(void) -{ - return 1; -} - static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid) { return false; @@ -85,7 +80,6 @@ static struct apic apic_bigsmp __ro_after_init = { .name = "bigsmp", .probe = probe_bigsmp, .apic_id_valid = default_apic_id_valid, - .apic_id_registered = bigsmp_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 345acdeab448..0c477110baaf 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -15,7 +15,6 @@ /* X2APIC */ int x2apic_apic_id_valid(u32 apicid); -int x2apic_apic_id_registered(void); void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest); unsigned int x2apic_get_apic_id(unsigned long id); u32 x2apic_set_apic_id(unsigned int id); @@ -61,6 +60,8 @@ void default_send_IPI_allbutself(int vector); void default_send_IPI_all(int vector); void default_send_IPI_self(int vector); +bool default_apic_id_registered(void); + #ifdef CONFIG_X86_32 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector); void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, int vector); diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 3f717f4c72bc..81c69d766a30 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -18,11 +18,6 @@ #include "local.h" -static int default_apic_id_registered(void) -{ - return physid_isset(read_apic_id(), phys_cpu_present_map); -} - static int default_phys_pkg_id(int cpuid_apic, int index_msb) { return cpuid_apic >> index_msb; diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 52d4be15a7dc..705b33d7072a 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -237,7 +237,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .probe = x2apic_cluster_probe, .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, .apic_id_valid = x2apic_apic_id_valid, - .apic_id_registered = x2apic_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index c57b5c82b14a..e0fdbf5522cd 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -111,11 +111,6 @@ int x2apic_apic_id_valid(u32 apicid) return 1; } -int x2apic_apic_id_registered(void) -{ - return 1; -} - void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) { unsigned long cfg = __prepare_ICR(0, vector, dest); @@ -157,7 +152,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .probe = x2apic_phys_probe, .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, .apic_id_valid = x2apic_apic_id_valid, - .apic_id_registered = x2apic_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index a8caff181adf..9e25e3033d0d 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -783,11 +783,6 @@ static int uv_apic_id_valid(u32 apicid) return 1; } -static int uv_apic_id_registered(void) -{ - return 1; -} - static u32 apic_uv_calc_apicid(unsigned int cpu) { return apic_default_calc_apicid(cpu); @@ -829,7 +824,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .probe = uv_probe, .acpi_madt_oem_check = uv_acpi_madt_oem_check, .apic_id_valid = uv_apic_id_valid, - .apic_id_registered = uv_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 4a27f92b51d1..f1cd7f2b3ac0 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -115,11 +115,6 @@ static int xen_id_always_valid(u32 apicid) return 1; } -static int xen_id_always_registered(void) -{ - return 1; -} - static int xen_phys_pkg_id(int initial_apic_id, int index_msb) { return initial_apic_id >> index_msb; @@ -142,7 +137,6 @@ static struct apic xen_pv_apic = { .probe = xen_apic_probe_pv, .acpi_madt_oem_check = xen_madt_oem_check, .apic_id_valid = xen_id_always_valid, - .apic_id_registered = xen_id_always_registered, /* .delivery_mode and .dest_mode_logical not used by XENPV */ -- cgit v1.2.3 From 01363d4f762c3fb7285cc5bb0fdd23356dbacbef Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:03 -0700 Subject: x86/apic/ipi: Tidy up the code and fixup comments Replace the undecodable comment on top of the function, replace the space consuming zero content comments with useful ones and tidy up the implementation to prevent further eye bleed. Make __default_send_IPI_shortcut() static as it has no other users. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/ipi.c | 74 +++++++++++++++++++------------------------- arch/x86/kernel/apic/local.h | 2 -- 2 files changed, 32 insertions(+), 44 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 2ff9cf9aa2ef..614ac557f5e4 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -108,68 +108,58 @@ static inline void __xapic_wait_icr_idle(void) cpu_relax(); } -void __default_send_IPI_shortcut(unsigned int shortcut, int vector) +/* + * This is safe against interruption because it only writes the lower 32 + * bits of the APIC_ICR register. The destination field is ignored for + * short hand IPIs. + * + * wait_icr_idle() + * write(ICR2, dest) + * NMI + * wait_icr_idle() + * write(ICR) + * wait_icr_idle() + * write(ICR) + * + * This function does not need to disable interrupts as there is no ICR2 + * interaction. The memory write is direct except when the machine is + * affected by the 11AP Pentium erratum, which turns the plain write into + * an XCHG operation. + */ +static void __default_send_IPI_shortcut(unsigned int shortcut, int vector) { /* - * Subtle. In the case of the 'never do double writes' workaround - * we have to lock out interrupts to be safe. As we don't care - * of the value read we use an atomic rmw access to avoid costly - * cli/sti. Otherwise we use an even cheaper single atomic write - * to the APIC. - */ - unsigned int cfg; - - /* - * Wait for idle. + * Wait for the previous ICR command to complete. Use + * safe_apic_wait_icr_idle() for the NMI vector as there have been + * issues where otherwise the system hangs when the panic CPU tries + * to stop the others before launching the kdump kernel. */ if (unlikely(vector == NMI_VECTOR)) safe_apic_wait_icr_idle(); else __xapic_wait_icr_idle(); - /* - * No need to touch the target chip field. Also the destination - * mode is ignored when a shorthand is used. - */ - cfg = __prepare_ICR(shortcut, vector, 0); - - /* - * Send the IPI. The write to APIC_ICR fires this off. - */ - native_apic_mem_write(APIC_ICR, cfg); + /* Destination field (ICR2) and the destination mode are ignored */ + native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0)); } /* * This is used to send an IPI with no shorthand notation (the destination is * specified in bits 56 to 63 of the ICR). */ -void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest) +void __default_send_IPI_dest_field(unsigned int dest_mask, int vector, + unsigned int dest_mode) { - unsigned long cfg; - - /* - * Wait for idle. - */ + /* See comment in __default_send_IPI_shortcut() */ if (unlikely(vector == NMI_VECTOR)) safe_apic_wait_icr_idle(); else __xapic_wait_icr_idle(); - /* - * prepare target chip field - */ - cfg = __prepare_ICR2(mask); - native_apic_mem_write(APIC_ICR2, cfg); - - /* - * program the ICR - */ - cfg = __prepare_ICR(0, vector, dest); - - /* - * Send the IPI. The write to APIC_ICR fires this off. - */ - native_apic_mem_write(APIC_ICR, cfg); + /* Set the IPI destination field in the ICR */ + native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask)); + /* Send it with the proper destination mode */ + native_apic_mem_write(APIC_ICR, __prepare_ICR(0, vector, dest_mode)); } void default_send_IPI_single_phys(int cpu, int vector) diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 0c477110baaf..5b0a0e78ba6c 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -44,8 +44,6 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, void default_init_apic_ldr(void); -void __default_send_IPI_shortcut(unsigned int shortcut, int vector); - /* * This is used to send an IPI with no shorthand notation (the destination is * specified in bits 56 to 63 of the ICR). -- cgit v1.2.3 From cfebd0077f3fc083cc139d7851e2068d058a89dd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:04 -0700 Subject: x86/apic: Consolidate wait_icr_idle() implementations Two copies and also needlessly public. Move it into ipi.c so it can be inlined. Rename it to apic_mem_wait_icr_idle(). Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 1 - arch/x86/kernel/apic/apic.c | 6 ------ arch/x86/kernel/apic/apic_flat_64.c | 4 ++-- arch/x86/kernel/apic/bigsmp_32.c | 2 +- arch/x86/kernel/apic/ipi.c | 6 +++--- arch/x86/kernel/apic/local.h | 2 ++ arch/x86/kernel/apic/probe_32.c | 2 +- 7 files changed, 9 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 4fb71b7d597a..1499865ebae7 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -98,7 +98,6 @@ static inline u32 native_apic_mem_read(u32 reg) return *((volatile u32 *)(APIC_BASE + reg)); } -extern void native_apic_wait_icr_idle(void); extern u32 native_safe_apic_wait_icr_idle(void); extern void native_apic_icr_write(u32 low, u32 id); extern u64 native_apic_icr_read(void); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 4ee95cb2178e..ab26a616f710 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -240,12 +240,6 @@ static void __init apic_disable(void) apic = &apic_noop; } -void native_apic_wait_icr_idle(void) -{ - while (apic_read(APIC_ICR) & APIC_ICR_BUSY) - cpu_relax(); -} - u32 native_safe_apic_wait_icr_idle(void) { u32 send_status; diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index a0c875d6ef3e..57d3afb65e95 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -111,7 +111,7 @@ static struct apic apic_flat __ro_after_init = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = native_apic_wait_icr_idle, + .wait_icr_idle = apic_mem_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, }; @@ -187,7 +187,7 @@ static struct apic apic_physflat __ro_after_init = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = native_apic_wait_icr_idle, + .wait_icr_idle = apic_mem_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index ffff294f4b44..57077fc2f8a5 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -108,7 +108,7 @@ static struct apic apic_bigsmp __ro_after_init = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = native_apic_wait_icr_idle, + .wait_icr_idle = apic_mem_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 614ac557f5e4..e0e05673a4f4 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -102,7 +102,7 @@ static inline int __prepare_ICR2(unsigned int mask) return SET_XAPIC_DEST_FIELD(mask); } -static inline void __xapic_wait_icr_idle(void) +void apic_mem_wait_icr_idle(void) { while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY) cpu_relax(); @@ -137,7 +137,7 @@ static void __default_send_IPI_shortcut(unsigned int shortcut, int vector) if (unlikely(vector == NMI_VECTOR)) safe_apic_wait_icr_idle(); else - __xapic_wait_icr_idle(); + apic_mem_wait_icr_idle(); /* Destination field (ICR2) and the destination mode are ignored */ native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0)); @@ -154,7 +154,7 @@ void __default_send_IPI_dest_field(unsigned int dest_mask, int vector, if (unlikely(vector == NMI_VECTOR)) safe_apic_wait_icr_idle(); else - __xapic_wait_icr_idle(); + apic_mem_wait_icr_idle(); /* Set the IPI destination field in the ICR */ native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask)); diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 5b0a0e78ba6c..2eb49d434564 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -44,6 +44,8 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, void default_init_apic_ldr(void); +void apic_mem_wait_icr_idle(void); + /* * This is used to send an IPI with no shorthand notation (the destination is * specified in bits 56 to 63 of the ICR). diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 81c69d766a30..52f3c6f2c6e2 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -64,7 +64,7 @@ static struct apic apic_default __ro_after_init = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = native_apic_wait_icr_idle, + .wait_icr_idle = apic_mem_wait_icr_idle, .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, }; -- cgit v1.2.3 From ee513d9da315592c27daa3cf5ce9ec2747b2bacd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:04 -0700 Subject: x86/apic: Allow apic::wait_icr_idle() to be NULL Nuke more NOOP callbacks and make the invocation conditional. Will be replaced with a static call later. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 9 ++------- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 7 ------- arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 1 - arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/xen/apic.c | 5 ----- 7 files changed, 2 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 1499865ebae7..397cd5ff2ebc 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -206,12 +206,6 @@ static inline u32 native_apic_msr_read(u32 reg) return (u32)msr; } -static inline void native_x2apic_wait_icr_idle(void) -{ - /* no need to wait for icr idle in x2apic */ - return; -} - static inline u32 native_safe_x2apic_wait_icr_idle(void) { /* no need to wait for icr idle in x2apic */ @@ -376,7 +370,8 @@ static inline void apic_icr_write(u32 low, u32 high) static inline void apic_wait_icr_idle(void) { - apic->wait_icr_idle(); + if (apic->wait_icr_idle) + apic->wait_icr_idle(); } static inline u32 safe_apic_wait_icr_idle(void) diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index c73a7a811bb9..ef31a2e829f1 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -20,7 +20,6 @@ static void noop_send_IPI_mask_allbutself(const struct cpumask *cpumask, int vec static void noop_send_IPI_allbutself(int vector) { } static void noop_send_IPI_all(int vector) { } static void noop_send_IPI_self(int vector) { } -static void noop_apic_wait_icr_idle(void) { } static void noop_apic_icr_write(u32 low, u32 id) { } static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) @@ -105,6 +104,5 @@ struct apic apic_noop __ro_after_init = { .eoi_write = noop_apic_write, .icr_read = noop_apic_icr_read, .icr_write = noop_apic_icr_write, - .wait_icr_idle = noop_apic_wait_icr_idle, .safe_wait_icr_idle = noop_safe_apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index e8e13d7c84d7..647ed774b346 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -223,11 +223,6 @@ static int numachip2_acpi_madt_oem_check(char *oem_id, char *oem_table_id) return 1; } -/* APIC IPIs are queued */ -static void numachip_apic_wait_icr_idle(void) -{ -} - /* APIC NMI IPIs are queued */ static u32 numachip_safe_apic_wait_icr_idle(void) { @@ -269,7 +264,6 @@ static const struct apic apic_numachip1 __refconst = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = numachip_apic_wait_icr_idle, .safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle, }; @@ -310,7 +304,6 @@ static const struct apic apic_numachip2 __refconst = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .wait_icr_idle = numachip_apic_wait_icr_idle, .safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 705b33d7072a..7ceef70312b5 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -266,7 +266,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .wait_icr_idle = native_x2apic_wait_icr_idle, .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index e0fdbf5522cd..c1ab67859134 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -180,7 +180,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .wait_icr_idle = native_x2apic_wait_icr_idle, .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 9e25e3033d0d..5cb68f657f79 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -854,7 +854,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .wait_icr_idle = native_x2apic_wait_icr_idle, .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index f1cd7f2b3ac0..e709bf0be86a 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -120,10 +120,6 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb) return initial_apic_id >> index_msb; } -static void xen_noop(void) -{ -} - static int xen_cpu_present_to_apicid(int cpu) { if (cpu_present(cpu)) @@ -165,7 +161,6 @@ static struct apic xen_pv_apic = { .icr_read = xen_apic_icr_read, .icr_write = xen_apic_icr_write, - .wait_icr_idle = xen_noop, .safe_wait_icr_idle = xen_safe_apic_wait_icr_idle, }; -- cgit v1.2.3 From 13d779fd26afd6e0a7aec899ab8a0eea6620fee7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:05 -0700 Subject: x86/apic: Allow apic::safe_wait_icr_idle() to be NULL Remove tons of NOOP callbacks by making the invocation of safe_wait_icr_idle() conditional in the inline wrapper. Will be replaced by a static_call_cond() later. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 8 +------- arch/x86/kernel/apic/apic_noop.c | 6 ------ arch/x86/kernel/apic/apic_numachip.c | 8 -------- arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 1 - arch/x86/kernel/apic/x2apic_uv_x.c | 1 - arch/x86/xen/apic.c | 6 ------ 7 files changed, 1 insertion(+), 30 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 397cd5ff2ebc..dd0a05dbe980 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -206,12 +206,6 @@ static inline u32 native_apic_msr_read(u32 reg) return (u32)msr; } -static inline u32 native_safe_x2apic_wait_icr_idle(void) -{ - /* no need to wait for icr idle in x2apic */ - return 0; -} - static inline void native_x2apic_icr_write(u32 low, u32 id) { wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low); @@ -376,7 +370,7 @@ static inline void apic_wait_icr_idle(void) static inline u32 safe_apic_wait_icr_idle(void) { - return apic->safe_wait_icr_idle(); + return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0; } extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)); diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index ef31a2e829f1..36c1e287ad05 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -27,11 +27,6 @@ static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) return -1; } -static u32 noop_safe_apic_wait_icr_idle(void) -{ - return 0; -} - static u64 noop_apic_icr_read(void) { return 0; @@ -104,5 +99,4 @@ struct apic apic_noop __ro_after_init = { .eoi_write = noop_apic_write, .icr_read = noop_apic_icr_read, .icr_write = noop_apic_icr_write, - .safe_wait_icr_idle = noop_safe_apic_wait_icr_idle, }; diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 647ed774b346..e879dbb57696 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -223,12 +223,6 @@ static int numachip2_acpi_madt_oem_check(char *oem_id, char *oem_table_id) return 1; } -/* APIC NMI IPIs are queued */ -static u32 numachip_safe_apic_wait_icr_idle(void) -{ - return 0; -} - static const struct apic apic_numachip1 __refconst = { .name = "NumaConnect system", .probe = numachip1_probe, @@ -264,7 +258,6 @@ static const struct apic apic_numachip1 __refconst = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle, }; apic_driver(apic_numachip1); @@ -304,7 +297,6 @@ static const struct apic apic_numachip2 __refconst = { .eoi_write = native_apic_mem_write, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, - .safe_wait_icr_idle = numachip_safe_apic_wait_icr_idle, }; apic_driver(apic_numachip2); diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 7ceef70312b5..355da47e3d3a 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -266,7 +266,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; apic_driver(apic_x2apic_cluster); diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index c1ab67859134..2c9a884efd6b 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -180,7 +180,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; apic_driver(apic_x2apic_phys); diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 5cb68f657f79..f532c5856dbb 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -854,7 +854,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .eoi_write = native_apic_msr_eoi_write, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, - .safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle, }; #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_LENGTH 3 diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index e709bf0be86a..0f71b2e79c76 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -92,11 +92,6 @@ static void xen_apic_icr_write(u32 low, u32 id) WARN_ON(1); } -static u32 xen_safe_apic_wait_icr_idle(void) -{ - return 0; -} - static int xen_apic_probe_pv(void) { if (xen_pv_domain()) @@ -161,7 +156,6 @@ static struct apic xen_pv_apic = { .icr_read = xen_apic_icr_read, .icr_write = xen_apic_icr_write, - .safe_wait_icr_idle = xen_safe_apic_wait_icr_idle, }; static void __init xen_apic_check(void) -- cgit v1.2.3 From e7b6a023d2c90fcd540a29bbf5bd4e1b816e613c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:06 -0700 Subject: x86/apic: Move safe wait_icr_idle() next to apic_mem_wait_icr_idle() Move it next to apic_mem_wait_icr_idle(), rename it so that it's clear what it does and rewrite it in readable form. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 1 - arch/x86/kernel/apic/apic.c | 17 ----------------- arch/x86/kernel/apic/apic_flat_64.c | 4 ++-- arch/x86/kernel/apic/bigsmp_32.c | 2 +- arch/x86/kernel/apic/ipi.c | 19 +++++++++++++++++-- arch/x86/kernel/apic/local.h | 1 + arch/x86/kernel/apic/probe_32.c | 2 +- 7 files changed, 22 insertions(+), 24 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index dd0a05dbe980..49b6c7ed2747 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -98,7 +98,6 @@ static inline u32 native_apic_mem_read(u32 reg) return *((volatile u32 *)(APIC_BASE + reg)); } -extern u32 native_safe_apic_wait_icr_idle(void); extern void native_apic_icr_write(u32 low, u32 id); extern u64 native_apic_icr_read(void); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index ab26a616f710..80a31f22a7cb 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -240,23 +240,6 @@ static void __init apic_disable(void) apic = &apic_noop; } -u32 native_safe_apic_wait_icr_idle(void) -{ - u32 send_status; - int timeout; - - timeout = 0; - do { - send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY; - if (!send_status) - break; - inc_irq_stat(icr_read_retry_count); - udelay(100); - } while (timeout++ < 1000); - - return send_status; -} - void native_apic_icr_write(u32 low, u32 id) { unsigned long flags; diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 57d3afb65e95..1da865f9c657 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -112,7 +112,7 @@ static struct apic apic_flat __ro_after_init = { .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, - .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, + .safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout, }; /* @@ -188,7 +188,7 @@ static struct apic apic_physflat __ro_after_init = { .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, - .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, + .safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout, }; /* diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 57077fc2f8a5..32984bfb3ce6 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -109,7 +109,7 @@ static struct apic apic_bigsmp __ro_after_init = { .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, - .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, + .safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout, }; bool __init apic_bigsmp_possible(bool cmdline_override) diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index e0e05673a4f4..460df029e511 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include + #include #include "local.h" @@ -102,6 +104,19 @@ static inline int __prepare_ICR2(unsigned int mask) return SET_XAPIC_DEST_FIELD(mask); } +u32 apic_mem_wait_icr_idle_timeout(void) +{ + int cnt; + + for (cnt = 0; cnt < 1000; cnt++) { + if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY)) + return 0; + inc_irq_stat(icr_read_retry_count); + udelay(100); + } + return APIC_ICR_BUSY; +} + void apic_mem_wait_icr_idle(void) { while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY) @@ -135,7 +150,7 @@ static void __default_send_IPI_shortcut(unsigned int shortcut, int vector) * to stop the others before launching the kdump kernel. */ if (unlikely(vector == NMI_VECTOR)) - safe_apic_wait_icr_idle(); + apic_mem_wait_icr_idle_timeout(); else apic_mem_wait_icr_idle(); @@ -152,7 +167,7 @@ void __default_send_IPI_dest_field(unsigned int dest_mask, int vector, { /* See comment in __default_send_IPI_shortcut() */ if (unlikely(vector == NMI_VECTOR)) - safe_apic_wait_icr_idle(); + apic_mem_wait_icr_idle_timeout(); else apic_mem_wait_icr_idle(); diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 2eb49d434564..98cfe7860d90 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -45,6 +45,7 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, void default_init_apic_ldr(void); void apic_mem_wait_icr_idle(void); +u32 apic_mem_wait_icr_idle_timeout(void); /* * This is used to send an IPI with no shorthand notation (the destination is diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 52f3c6f2c6e2..a861a773ce5c 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -65,7 +65,7 @@ static struct apic apic_default __ro_after_init = { .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, - .safe_wait_icr_idle = native_safe_apic_wait_icr_idle, + .safe_wait_icr_idle = apic_mem_wait_icr_idle_timeout, }; apic_driver(apic_default); -- cgit v1.2.3 From 194ac709819c1fe281397ce1b2894f2bb6413599 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:07 -0700 Subject: x86/apic/uv: Get rid of wrapper callbacks Why on earth makes a wrapper around some common function sense? Just to be able to slap some vendor name on it... Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/x2apic_uv_x.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index f532c5856dbb..f0580a6b7595 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -783,11 +783,6 @@ static int uv_apic_id_valid(u32 apicid) return 1; } -static u32 apic_uv_calc_apicid(unsigned int cpu) -{ - return apic_default_calc_apicid(cpu); -} - static unsigned int x2apic_get_apic_id(unsigned long id) { return id; @@ -838,7 +833,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .get_apic_id = x2apic_get_apic_id, .set_apic_id = set_apic_id, - .calc_dest_apicid = apic_uv_calc_apicid, + .calc_dest_apicid = apic_default_calc_apicid, .send_IPI = uv_send_IPI_one, .send_IPI_mask = uv_send_IPI_mask, -- cgit v1.2.3 From 96ae35c75bdd8a327c686cf39030d8ed7f82f558 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:07 -0700 Subject: x86/apic/x2apic: Share all common IPI functions Yet more copy and pasta gone. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/local.h | 4 +++- arch/x86/kernel/apic/x2apic_cluster.c | 10 -------- arch/x86/kernel/apic/x2apic_phys.c | 44 +++++++++++++++++------------------ arch/x86/kernel/apic/x2apic_uv_x.c | 14 +++-------- 4 files changed, 28 insertions(+), 44 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 98cfe7860d90..a250675cd97e 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -19,8 +19,10 @@ void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest); unsigned int x2apic_get_apic_id(unsigned long id); u32 x2apic_set_apic_id(unsigned int id); int x2apic_phys_pkg_id(int initial_apicid, int index_msb); + +void x2apic_send_IPI_all(int vector); +void x2apic_send_IPI_allbutself(int vector); void x2apic_send_IPI_self(int vector); -void __x2apic_send_IPI_shorthand(int vector, u32 which); /* IPI */ diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 355da47e3d3a..afd26764047a 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -83,16 +83,6 @@ x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT); } -static void x2apic_send_IPI_allbutself(int vector) -{ - __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLBUT); -} - -static void x2apic_send_IPI_all(int vector) -{ - __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC); -} - static u32 x2apic_calc_apicid(unsigned int cpu) { return x86_cpu_to_logical_apicid[cpu]; diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 2c9a884efd6b..c40d19b140bc 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -81,16 +81,36 @@ static void __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT); } -static void x2apic_send_IPI_allbutself(int vector) +static void __x2apic_send_IPI_shorthand(int vector, u32 which) +{ + unsigned long cfg = __prepare_ICR(which, vector, 0); + + /* x2apic MSRs are special and need a special fence: */ + weak_wrmsr_fence(); + native_x2apic_icr_write(cfg, 0); +} + +void x2apic_send_IPI_allbutself(int vector) { __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLBUT); } -static void x2apic_send_IPI_all(int vector) +void x2apic_send_IPI_all(int vector) { __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC); } +void x2apic_send_IPI_self(int vector) +{ + apic_write(APIC_SELF_IPI, vector); +} + +void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) +{ + unsigned long cfg = __prepare_ICR(0, vector, dest); + native_x2apic_icr_write(cfg, apicid); +} + static int x2apic_phys_probe(void) { if (!x2apic_mode) @@ -111,21 +131,6 @@ int x2apic_apic_id_valid(u32 apicid) return 1; } -void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) -{ - unsigned long cfg = __prepare_ICR(0, vector, dest); - native_x2apic_icr_write(cfg, apicid); -} - -void __x2apic_send_IPI_shorthand(int vector, u32 which) -{ - unsigned long cfg = __prepare_ICR(which, vector, 0); - - /* x2apic MSRs are special and need a special fence: */ - weak_wrmsr_fence(); - native_x2apic_icr_write(cfg, 0); -} - unsigned int x2apic_get_apic_id(unsigned long id) { return id; @@ -141,11 +146,6 @@ int x2apic_phys_pkg_id(int initial_apicid, int index_msb) return initial_apicid >> index_msb; } -void x2apic_send_IPI_self(int vector) -{ - apic_write(APIC_SELF_IPI, vector); -} - static struct apic apic_x2apic_phys __ro_after_init = { .name = "physical x2apic", diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index f0580a6b7595..a8ed237bcde7 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -25,6 +25,8 @@ #include #include +#include "local.h" + static enum uv_system_type uv_system_type; static int uv_hubbed_system; static int uv_hubless_system; @@ -783,11 +785,6 @@ static int uv_apic_id_valid(u32 apicid) return 1; } -static unsigned int x2apic_get_apic_id(unsigned long id) -{ - return id; -} - static u32 set_apic_id(unsigned int id) { return id; @@ -803,11 +800,6 @@ static int uv_phys_pkg_id(int initial_apicid, int index_msb) return uv_read_apic_id() >> index_msb; } -static void uv_send_IPI_self(int vector) -{ - apic_write(APIC_SELF_IPI, vector); -} - static int uv_probe(void) { return apic == &apic_x2apic_uv_x; @@ -840,7 +832,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .send_IPI_mask_allbutself = uv_send_IPI_mask_allbutself, .send_IPI_allbutself = uv_send_IPI_allbutself, .send_IPI_all = uv_send_IPI_all, - .send_IPI_self = uv_send_IPI_self, + .send_IPI_self = x2apic_send_IPI_self, .wakeup_secondary_cpu = uv_wakeup_secondary, -- cgit v1.2.3 From d65eb3195862d600b120b3f6481e016c312158cb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:08 -0700 Subject: x86/apic/64: Uncopypaste probing No need for the same thing twice. Also prepares for simplifying the APIC ID validation checks. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/probe_64.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c index 7c9f1a8a83c8..a82bb52d5c69 100644 --- a/arch/x86/kernel/apic/probe_64.c +++ b/arch/x86/kernel/apic/probe_64.c @@ -13,6 +13,15 @@ #include "local.h" +static __init void apic_install_driver(struct apic *driver) +{ + if (apic == driver) + return; + + apic = driver; + pr_info("Switched APIC routing to %s:\n", apic->name); +} + /* Select the appropriate APIC driver */ void __init x86_64_probe_apic(void) { @@ -22,11 +31,7 @@ void __init x86_64_probe_apic(void) for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { if ((*drv)->probe && (*drv)->probe()) { - if (apic != *drv) { - apic = *drv; - pr_info("Switched APIC routing to %s.\n", - apic->name); - } + apic_install_driver(*drv); break; } } @@ -38,11 +43,7 @@ int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id) for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { if ((*drv)->acpi_madt_oem_check(oem_id, oem_table_id)) { - if (apic != *drv) { - apic = *drv; - pr_info("Setting APIC routing to %s.\n", - apic->name); - } + apic_install_driver(*drv); return 1; } } -- cgit v1.2.3 From 9132d720eb8a40be5eaa539ed940beaf2bd2e421 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:09 -0700 Subject: x86/apic: Wrap APIC ID validation into an inline Prepare for removing the callback and making this as simple comparison to an upper limit, which is the obvious solution to do for limit checks... Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 5 +++++ arch/x86/kernel/acpi/boot.c | 2 +- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/smpboot.c | 5 ++--- arch/x86/mm/srat.c | 5 ++--- 5 files changed, 11 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 49b6c7ed2747..5a01d51a2970 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -372,6 +372,11 @@ static inline u32 safe_apic_wait_icr_idle(void) return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0; } +static inline bool apic_id_valid(u32 apic_id) +{ + return apic->apic_id_valid(apic_id); +} + extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)); #else /* CONFIG_X86_LOCAL_APIC */ diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index cca1a0709605..7f5b257aaee3 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -235,7 +235,7 @@ acpi_parse_x2apic(union acpi_subtable_headers *header, const unsigned long end) * to not preallocating memory for all NR_CPUS * when we use CPU hotplug. */ - if (!apic->apic_id_valid(apic_id)) { + if (!apic_id_valid(apic_id)) { if (enabled) pr_warn("x2apic entry ignored\n"); return 0; diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 71feae718a87..2ee3f5a20f3d 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -691,7 +691,7 @@ static int x86_vector_select(struct irq_domain *d, struct irq_fwspec *fwspec, * if IRQ remapping is enabled. APIC IDs above 15 bits are * only permitted if IRQ remapping is enabled, so check that. */ - if (apic->apic_id_valid(32768)) + if (apic_id_valid(32768)) return 0; return x86_fwspec_is_ioapic(fwspec) || x86_fwspec_is_hpet(fwspec); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index df1c5f01411b..734997481238 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1064,9 +1064,8 @@ int native_kick_ap(unsigned int cpu, struct task_struct *tidle) pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu); - if (apicid == BAD_APICID || - !physid_isset(apicid, phys_cpu_present_map) || - !apic->apic_id_valid(apicid)) { + if (apicid == BAD_APICID || !physid_isset(apicid, phys_cpu_present_map) || + !apic_id_valid(apicid)) { pr_err("%s: bad cpu %d\n", __func__, cpu); return -EINVAL; } diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c index dac07e4f5834..9c52a95937ad 100644 --- a/arch/x86/mm/srat.c +++ b/arch/x86/mm/srat.c @@ -40,9 +40,8 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) return; pxm = pa->proximity_domain; apic_id = pa->apic_id; - if (!apic->apic_id_valid(apic_id)) { - printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n", - pxm, apic_id); + if (!apic_id_valid(apic_id)) { + pr_info("SRAT: PXM %u -> X2APIC 0x%04x ignored\n", pxm, apic_id); return; } node = acpi_map_pxm_to_node(pxm); -- cgit v1.2.3 From d92e5e7cf580ac59678fa40fd99aa8eb00f1ff52 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:10 -0700 Subject: x86/apic: Add max_apic_id member There is really no point to have a callback which compares numbers. Add a field which allows each APIC to store the maximum APIC ID supported and fill it in for all APIC incarnations. The next step will remove the callback. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 3 +++ arch/x86/kernel/apic/apic_flat_64.c | 2 ++ arch/x86/kernel/apic/apic_noop.c | 1 + arch/x86/kernel/apic/apic_numachip.c | 2 ++ arch/x86/kernel/apic/bigsmp_32.c | 1 + arch/x86/kernel/apic/probe_32.c | 1 + arch/x86/kernel/apic/x2apic_cluster.c | 1 + arch/x86/kernel/apic/x2apic_phys.c | 1 + arch/x86/kernel/apic/x2apic_uv_x.c | 1 + arch/x86/xen/apic.c | 1 + 10 files changed, 14 insertions(+) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 5a01d51a2970..42d3240f4f89 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -277,6 +277,9 @@ struct apic { u64 (*icr_read)(void); void (*icr_write)(u32 low, u32 high); + /* The limit of the APIC ID space. */ + u32 max_apic_id; + /* Probe, setup and smpboot functions */ int (*probe)(void); int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 1da865f9c657..b5e85bb021a1 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -94,6 +94,7 @@ static struct apic apic_flat __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = flat_phys_pkg_id, + .max_apic_id = 0xFE, .get_apic_id = flat_get_apic_id, .set_apic_id = set_apic_id, @@ -170,6 +171,7 @@ static struct apic apic_physflat __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = flat_phys_pkg_id, + .max_apic_id = 0xFE, .get_apic_id = flat_get_apic_id, .set_apic_id = set_apic_id, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 36c1e287ad05..d49c10d31318 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -80,6 +80,7 @@ struct apic apic_noop __ro_after_init = { .phys_pkg_id = noop_phys_pkg_id, + .max_apic_id = 0xFE, .get_apic_id = noop_get_apic_id, .set_apic_id = NULL, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index e879dbb57696..43cbc24dbdfd 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -239,6 +239,7 @@ static const struct apic apic_numachip1 __refconst = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, + .max_apic_id = UINT_MAX, .get_apic_id = numachip1_get_apic_id, .set_apic_id = numachip1_set_apic_id, @@ -278,6 +279,7 @@ static const struct apic apic_numachip2 __refconst = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, + .max_apic_id = UINT_MAX, .get_apic_id = numachip2_get_apic_id, .set_apic_id = numachip2_set_apic_id, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 32984bfb3ce6..5f7a94b97e17 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -91,6 +91,7 @@ static struct apic apic_bigsmp __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = bigsmp_phys_pkg_id, + .max_apic_id = 0xFE, .get_apic_id = bigsmp_get_apic_id, .set_apic_id = NULL, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index a861a773ce5c..7c4b5f716bb6 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -47,6 +47,7 @@ static struct apic apic_default __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = default_phys_pkg_id, + .max_apic_id = 0xFE, .get_apic_id = default_get_apic_id, .set_apic_id = NULL, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index afd26764047a..036dd1c3a60b 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -239,6 +239,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = x2apic_phys_pkg_id, + .max_apic_id = UINT_MAX, .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index c40d19b140bc..d6495acaf6af 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -163,6 +163,7 @@ static struct apic apic_x2apic_phys __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = x2apic_phys_pkg_id, + .max_apic_id = UINT_MAX, .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index a8ed237bcde7..7d4eaa889e64 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -822,6 +822,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = uv_phys_pkg_id, + .max_apic_id = UINT_MAX, .get_apic_id = x2apic_get_apic_id, .set_apic_id = set_apic_id, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 0f71b2e79c76..572814b29c6d 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -138,6 +138,7 @@ static struct apic xen_pv_apic = { .cpu_present_to_apicid = xen_cpu_present_to_apicid, .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ + .max_apic_id = UINT_MAX, .get_apic_id = xen_get_apic_id, .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ -- cgit v1.2.3 From a9d608c22aeecaa6cd34717a8576318ae2295379 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:10 -0700 Subject: x86/apic: Simplify X2APIC ID validation Currently, x2apic_max_apicid==0 means that there is no max APIC id limit. But, this means that 0 needs to be special-cased. Designate UINT_MAX to mean unlimited so that a plain old less than or equal compare works and there is no special-casing. Replace the 0 initialization with UINT_MAX. [ dhansen: muck with changelog ] Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/x2apic_phys.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index d6495acaf6af..d292ec311c19 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -8,7 +8,7 @@ int x2apic_phys; static struct apic apic_x2apic_phys; -static u32 x2apic_max_apicid __ro_after_init; +static u32 x2apic_max_apicid __ro_after_init = UINT_MAX; void __init x2apic_set_max_apicid(u32 apicid) { @@ -125,10 +125,7 @@ static int x2apic_phys_probe(void) /* Common x2apic functions, also used by x2apic_cluster */ int x2apic_apic_id_valid(u32 apicid) { - if (x2apic_max_apicid && apicid > x2apic_max_apicid) - return 0; - - return 1; + return apicid <= x2apic_max_apicid; } unsigned int x2apic_get_apic_id(unsigned long id) -- cgit v1.2.3 From b5a5ce58d3d22ce560305ef9c70a2ec21fb887ba Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:11 -0700 Subject: x86/apic: Prepare x2APIC for using apic::max_apic_id In order to remove the apic::apic_id_valid() callback and switch to checking apic::max_apic_id, it is required to update apic::max_apic_id when the APIC initialization code overrides it via x2apic_set_max_apicid(). Make the existing booleans a bitfield and add a flag which lets the update function and the core code which switches the driver detect whether the apic instance wants to have that update or not and apply it if required. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 7 ++++--- arch/x86/kernel/apic/local.h | 1 + arch/x86/kernel/apic/probe_64.c | 4 ++++ arch/x86/kernel/apic/x2apic_cluster.c | 1 + arch/x86/kernel/apic/x2apic_phys.c | 5 ++++- 5 files changed, 14 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 42d3240f4f89..2e79c13d3c65 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -266,10 +266,11 @@ struct apic { void (*send_IPI_all)(int vector); void (*send_IPI_self)(int vector); - u32 disable_esr; - enum apic_delivery_modes delivery_mode; - bool dest_mode_logical; + + u32 disable_esr : 1, + dest_mode_logical : 1, + x2apic_set_max_apicid : 1; u32 (*calc_dest_apicid)(unsigned int cpu); diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index a250675cd97e..e576b3a4d29c 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -23,6 +23,7 @@ int x2apic_phys_pkg_id(int initial_apicid, int index_msb); void x2apic_send_IPI_all(int vector); void x2apic_send_IPI_allbutself(int vector); void x2apic_send_IPI_self(int vector); +extern u32 x2apic_max_apicid; /* IPI */ diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c index a82bb52d5c69..a12a379280a8 100644 --- a/arch/x86/kernel/apic/probe_64.c +++ b/arch/x86/kernel/apic/probe_64.c @@ -19,6 +19,10 @@ static __init void apic_install_driver(struct apic *driver) return; apic = driver; + + if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid) + apic->max_apic_id = x2apic_max_apicid; + pr_info("Switched APIC routing to %s:\n", apic->name); } diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 036dd1c3a60b..a164764cb166 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -240,6 +240,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .phys_pkg_id = x2apic_phys_pkg_id, .max_apic_id = UINT_MAX, + .x2apic_set_max_apicid = true, .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index d292ec311c19..3aabfd336f1e 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -8,11 +8,13 @@ int x2apic_phys; static struct apic apic_x2apic_phys; -static u32 x2apic_max_apicid __ro_after_init = UINT_MAX; +u32 x2apic_max_apicid __ro_after_init = UINT_MAX; void __init x2apic_set_max_apicid(u32 apicid) { x2apic_max_apicid = apicid; + if (apic->x2apic_set_max_apicid) + apic->max_apic_id = apicid; } static int __init set_x2apic_phys_mode(char *arg) @@ -161,6 +163,7 @@ static struct apic apic_x2apic_phys __ro_after_init = { .phys_pkg_id = x2apic_phys_pkg_id, .max_apic_id = UINT_MAX, + .x2apic_set_max_apicid = true, .get_apic_id = x2apic_get_apic_id, .set_apic_id = x2apic_set_apic_id, -- cgit v1.2.3 From d8666cf780203fe016f13e7d5d1af50811dc01b5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:12 -0700 Subject: x86/apic: Sanitize APIC ID range validation Now that everything has apic::max_apic_id set and the eventual update for the x2APIC case is in place, switch the apic_id_valid() helper to use apic::max_apic_id and remove the apic::apic_id_valid() callback. [ dhansen: Fix subject typo ] Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 3 +-- arch/x86/kernel/apic/apic_common.c | 5 ----- arch/x86/kernel/apic/apic_flat_64.c | 2 -- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 8 -------- arch/x86/kernel/apic/bigsmp_32.c | 1 - arch/x86/kernel/apic/local.h | 1 - arch/x86/kernel/apic/probe_32.c | 1 - arch/x86/kernel/apic/x2apic_cluster.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 7 ------- arch/x86/kernel/apic/x2apic_uv_x.c | 6 ------ arch/x86/xen/apic.c | 6 ------ 12 files changed, 1 insertion(+), 42 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 2e79c13d3c65..ccf4e3665d08 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -284,7 +284,6 @@ struct apic { /* Probe, setup and smpboot functions */ int (*probe)(void); int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); - int (*apic_id_valid)(u32 apicid); bool (*apic_id_registered)(void); bool (*check_apicid_used)(physid_mask_t *map, int apicid); @@ -378,7 +377,7 @@ static inline u32 safe_apic_wait_icr_idle(void) static inline bool apic_id_valid(u32 apic_id) { - return apic->apic_id_valid(apic_id); + return apic_id <= apic->max_apic_id; } extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)); diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index bbb5d5a6c820..7bc5d9bf59cd 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -37,11 +37,6 @@ int default_cpu_present_to_apicid(int mps_cpu) } EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid); -int default_apic_id_valid(u32 apicid) -{ - return (apicid < 255); -} - bool default_apic_id_registered(void) { return physid_isset(read_apic_id(), phys_cpu_present_map); diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index b5e85bb021a1..1e268a0251f8 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -80,7 +80,6 @@ static struct apic apic_flat __ro_after_init = { .name = "flat", .probe = flat_probe, .acpi_madt_oem_check = flat_acpi_madt_oem_check, - .apic_id_valid = default_apic_id_valid, .apic_id_registered = default_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, @@ -158,7 +157,6 @@ static struct apic apic_physflat __ro_after_init = { .name = "physical flat", .probe = physflat_probe, .acpi_madt_oem_check = physflat_acpi_madt_oem_check, - .apic_id_valid = default_apic_id_valid, .apic_id_registered = default_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index d49c10d31318..bd6a95ce75de 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -67,8 +67,6 @@ struct apic apic_noop __ro_after_init = { .probe = noop_probe, .acpi_madt_oem_check = NULL, - .apic_id_valid = default_apic_id_valid, - .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 43cbc24dbdfd..39ebe1a3458b 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -56,12 +56,6 @@ static u32 numachip2_set_apic_id(unsigned int id) return id << 24; } -static int numachip_apic_id_valid(u32 apicid) -{ - /* Trust what bootloader passes in MADT */ - return 1; -} - static int numachip_phys_pkg_id(int initial_apic_id, int index_msb) { return initial_apic_id >> index_msb; @@ -227,7 +221,6 @@ static const struct apic apic_numachip1 __refconst = { .name = "NumaConnect system", .probe = numachip1_probe, .acpi_madt_oem_check = numachip1_acpi_madt_oem_check, - .apic_id_valid = numachip_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, @@ -267,7 +260,6 @@ static const struct apic apic_numachip2 __refconst = { .name = "NumaConnect2 system", .probe = numachip2_probe, .acpi_madt_oem_check = numachip2_acpi_madt_oem_check, - .apic_id_valid = numachip_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 5f7a94b97e17..ffcd114c96b9 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -79,7 +79,6 @@ static struct apic apic_bigsmp __ro_after_init = { .name = "bigsmp", .probe = probe_bigsmp, - .apic_id_valid = default_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index e576b3a4d29c..ec219c659c7d 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -14,7 +14,6 @@ #include /* X2APIC */ -int x2apic_apic_id_valid(u32 apicid); void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest); unsigned int x2apic_get_apic_id(unsigned long id); u32 x2apic_set_apic_id(unsigned int id); diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 7c4b5f716bb6..d9f52f2f3e16 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -33,7 +33,6 @@ static struct apic apic_default __ro_after_init = { .name = "default", .probe = probe_default, - .apic_id_valid = default_apic_id_valid, .apic_id_registered = default_apic_id_registered, .delivery_mode = APIC_DELIVERY_MODE_FIXED, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index a164764cb166..12b9c310f3f5 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -226,7 +226,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .name = "cluster x2apic", .probe = x2apic_cluster_probe, .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, - .apic_id_valid = x2apic_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 3aabfd336f1e..fcb96b9652ba 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -124,12 +124,6 @@ static int x2apic_phys_probe(void) return apic == &apic_x2apic_phys; } -/* Common x2apic functions, also used by x2apic_cluster */ -int x2apic_apic_id_valid(u32 apicid) -{ - return apicid <= x2apic_max_apicid; -} - unsigned int x2apic_get_apic_id(unsigned long id) { return id; @@ -150,7 +144,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .name = "physical x2apic", .probe = x2apic_phys_probe, .acpi_madt_oem_check = x2apic_acpi_madt_oem_check, - .apic_id_valid = x2apic_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 7d4eaa889e64..26891ac881f9 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -780,11 +780,6 @@ static void uv_send_IPI_all(int vector) uv_send_IPI_mask(cpu_online_mask, vector); } -static int uv_apic_id_valid(u32 apicid) -{ - return 1; -} - static u32 set_apic_id(unsigned int id) { return id; @@ -810,7 +805,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .name = "UV large system", .probe = uv_probe, .acpi_madt_oem_check = uv_acpi_madt_oem_check, - .apic_id_valid = uv_apic_id_valid, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = false, diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 572814b29c6d..661eecfdc4ba 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -105,11 +105,6 @@ static int xen_madt_oem_check(char *oem_id, char *oem_table_id) return xen_pv_domain(); } -static int xen_id_always_valid(u32 apicid) -{ - return 1; -} - static int xen_phys_pkg_id(int initial_apic_id, int index_msb) { return initial_apic_id >> index_msb; @@ -127,7 +122,6 @@ static struct apic xen_pv_apic = { .name = "Xen PV", .probe = xen_apic_probe_pv, .acpi_madt_oem_check = xen_madt_oem_check, - .apic_id_valid = xen_id_always_valid, /* .delivery_mode and .dest_mode_logical not used by XENPV */ -- cgit v1.2.3 From 1fddf1dcc58406e72551ca6aa6f0a81c438abb07 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:13 -0700 Subject: x86/apic: Remove pointless NULL initializations Wasted space for no value. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic_flat_64.c | 2 -- arch/x86/kernel/apic/apic_noop.c | 2 -- arch/x86/kernel/apic/apic_numachip.c | 4 ---- arch/x86/kernel/apic/probe_32.c | 1 - arch/x86/kernel/apic/x2apic_phys.c | 2 -- arch/x86/kernel/apic/x2apic_uv_x.c | 2 -- 6 files changed, 13 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 1e268a0251f8..f65419dfdb0f 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -87,9 +87,7 @@ static struct apic apic_flat __ro_after_init = { .disable_esr = 0, - .check_apicid_used = NULL, .init_apic_ldr = default_init_apic_ldr, - .ioapic_phys_id_map = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = flat_phys_pkg_id, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index bd6a95ce75de..28b693b0f72d 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -65,7 +65,6 @@ static void noop_apic_write(u32 reg, u32 val) struct apic apic_noop __ro_after_init = { .name = "noop", .probe = noop_probe, - .acpi_madt_oem_check = NULL, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, @@ -80,7 +79,6 @@ struct apic apic_noop __ro_after_init = { .max_apic_id = 0xFE, .get_apic_id = noop_get_apic_id, - .set_apic_id = NULL, .calc_dest_apicid = apic_flat_calc_apicid, diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 39ebe1a3458b..9df018716dcd 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -227,8 +227,6 @@ static const struct apic apic_numachip1 __refconst = { .disable_esr = 0, - .check_apicid_used = NULL, - .ioapic_phys_id_map = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, @@ -266,8 +264,6 @@ static const struct apic apic_numachip2 __refconst = { .disable_esr = 0, - .check_apicid_used = NULL, - .ioapic_phys_id_map = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = numachip_phys_pkg_id, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index d9f52f2f3e16..5facf4266bc5 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -48,7 +48,6 @@ static struct apic apic_default __ro_after_init = { .max_apic_id = 0xFE, .get_apic_id = default_get_apic_id, - .set_apic_id = NULL, .calc_dest_apicid = apic_flat_calc_apicid, diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index fcb96b9652ba..47ee5c679a45 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -150,8 +150,6 @@ static struct apic apic_x2apic_phys __ro_after_init = { .disable_esr = 0, - .check_apicid_used = NULL, - .ioapic_phys_id_map = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = x2apic_phys_pkg_id, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 26891ac881f9..56925147d8ac 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -811,8 +811,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .disable_esr = 0, - .check_apicid_used = NULL, - .ioapic_phys_id_map = NULL, .cpu_present_to_apicid = default_cpu_present_to_apicid, .phys_pkg_id = uv_phys_pkg_id, -- cgit v1.2.3 From 3600ceb0df845beb7a204ab599aaff1213129314 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:14 -0700 Subject: x86/apic/noop: Tidy up the code First of all apic_noop can't be probed because it's not registered. So there is no point for implementing a probe callback. The machine is rightfully to die when that is invoked. Remove the gunk and tidy up the other space consuming dummy callbacks. This gunk should simply die. Nothing should ever invoke APIC callbacks once this is installed, But that's a differrent story for another round of cleanups. The comment on top of this file which was intentionally left in place tells exactly why this is needed: voodoo programming. In fact the kernel of today should just outright refuse to boot on a system with no (functional) local APIC. That would spare tons of #ifdeffery and other nonsense. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/kernel/apic/apic_noop.c | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index 28b693b0f72d..e33bea0a2f49 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -8,6 +8,10 @@ * Though in case if apic is disabled (for some reason) we try * to not uglify the caller's code and allow to call (some) apic routines * like self-ipi, etc... + * + * FIXME: Remove this gunk. The above argument which was intentionally left + * in place is silly to begin with because none of the callbacks except for + * APIC::read/write() have a WARN_ON_ONCE() in them. Sigh... */ #include #include @@ -21,35 +25,10 @@ static void noop_send_IPI_allbutself(int vector) { } static void noop_send_IPI_all(int vector) { } static void noop_send_IPI_self(int vector) { } static void noop_apic_icr_write(u32 low, u32 id) { } - -static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) -{ - return -1; -} - -static u64 noop_apic_icr_read(void) -{ - return 0; -} - -static int noop_phys_pkg_id(int cpuid_apic, int index_msb) -{ - return 0; -} - -static unsigned int noop_get_apic_id(unsigned long x) -{ - return 0; -} - -static int noop_probe(void) -{ - /* - * NOOP apic should not ever be - * enabled via probe routine - */ - return 0; -} +static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) { return -1; } +static u64 noop_apic_icr_read(void) { return 0; } +static int noop_phys_pkg_id(int cpuid_apic, int index_msb) { return 0; } +static unsigned int noop_get_apic_id(unsigned long x) { return 0; } static u32 noop_apic_read(u32 reg) { @@ -64,7 +43,6 @@ static void noop_apic_write(u32 reg, u32 val) struct apic apic_noop __ro_after_init = { .name = "noop", - .probe = noop_probe, .delivery_mode = APIC_DELIVERY_MODE_FIXED, .dest_mode_logical = true, -- cgit v1.2.3 From 185c8f33a048bd04fdedd08e7bd7861a85158834 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:15 -0700 Subject: x86/apic: Remove pointless arguments from [native_]eoi_write() Every callsite hands in the same constants which is a pointless exercise and cannot be optimized by the compiler due to the indirect calls. Use the constants in the eoi() callbacks and remove the arguments. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Reviewed-by: Wei Liu Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/hyperv/hv_apic.c | 6 +++--- arch/x86/include/asm/apic.h | 17 +++++++++++------ arch/x86/kernel/apic/apic.c | 8 ++++---- arch/x86/kernel/apic/apic_flat_64.c | 4 ++-- arch/x86/kernel/apic/apic_noop.c | 3 ++- arch/x86/kernel/apic/apic_numachip.c | 4 ++-- arch/x86/kernel/apic/bigsmp_32.c | 2 +- arch/x86/kernel/apic/probe_32.c | 2 +- arch/x86/kernel/apic/x2apic_cluster.c | 2 +- arch/x86/kernel/apic/x2apic_phys.c | 2 +- arch/x86/kernel/apic/x2apic_uv_x.c | 2 +- arch/x86/kernel/kvm.c | 6 +++--- arch/x86/xen/apic.c | 7 ++++++- 13 files changed, 38 insertions(+), 27 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index 1fbda2f94184..5513f60b3cdd 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -86,14 +86,14 @@ static void hv_apic_write(u32 reg, u32 val) } } -static void hv_apic_eoi_write(u32 reg, u32 val) +static void hv_apic_eoi_write(void) { struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1)) return; - wrmsr(HV_X64_MSR_EOI, val, 0); + wrmsr(HV_X64_MSR_EOI, APIC_EOI_ACK, 0); } static bool cpu_is_self(int cpu) @@ -310,7 +310,7 @@ void __init hv_apic_init(void) * lazy EOI when available, but the same accessor works for * both xapic and x2apic because the field layout is the same. */ - apic_set_eoi_write(hv_apic_eoi_write); + apic_set_eoi_cb(hv_apic_eoi_write); if (!x2apic_enabled()) { apic->read = hv_apic_read; apic->write = hv_apic_write; diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index ccf4e3665d08..790f15d89415 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -98,6 +98,11 @@ static inline u32 native_apic_mem_read(u32 reg) return *((volatile u32 *)(APIC_BASE + reg)); } +static inline void native_apic_mem_eoi(void) +{ + native_apic_mem_write(APIC_EOI, APIC_EOI_ACK); +} + extern void native_apic_icr_write(u32 low, u32 id); extern u64 native_apic_icr_read(void); @@ -189,7 +194,7 @@ static inline void native_apic_msr_write(u32 reg, u32 v) wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0); } -static inline void native_apic_msr_eoi_write(u32 reg, u32 v) +static inline void native_apic_msr_eoi(void) { __wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0); } @@ -250,8 +255,8 @@ struct irq_data; */ struct apic { /* Hotpath functions first */ - void (*eoi_write)(u32 reg, u32 v); - void (*native_eoi_write)(u32 reg, u32 v); + void (*eoi)(void); + void (*native_eoi)(void); void (*write)(u32 reg, u32 v); u32 (*read)(u32 reg); @@ -351,7 +356,7 @@ static inline void apic_write(u32 reg, u32 val) static inline void apic_eoi(void) { - apic->eoi_write(APIC_EOI, APIC_EOI_ACK); + apic->eoi(); } static inline u64 apic_icr_read(void) @@ -380,7 +385,7 @@ static inline bool apic_id_valid(u32 apic_id) return apic_id <= apic->max_apic_id; } -extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)); +extern void __init apic_set_eoi_cb(void (*eoi)(void)); #else /* CONFIG_X86_LOCAL_APIC */ @@ -391,7 +396,7 @@ static inline u64 apic_icr_read(void) { return 0; } static inline void apic_icr_write(u32 low, u32 high) { } static inline void apic_wait_icr_idle(void) { } static inline u32 safe_apic_wait_icr_idle(void) { return 0; } -static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {} +static inline void apic_set_eoi_cb(void (*eoi)(void)) {} #endif /* CONFIG_X86_LOCAL_APIC */ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 80a31f22a7cb..33d5ed693f54 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2502,15 +2502,15 @@ void __init acpi_wake_cpu_handler_update(wakeup_cpu_handler handler) * interrupts disabled, so we know this does not race with actual APIC driver * use. */ -void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) +void __init apic_set_eoi_cb(void (*eoi)(void)) { struct apic **drv; for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { /* Should happen once for each apic */ - WARN_ON((*drv)->eoi_write == eoi_write); - (*drv)->native_eoi_write = (*drv)->eoi_write; - (*drv)->eoi_write = eoi_write; + WARN_ON((*drv)->eoi == eoi); + (*drv)->native_eoi = (*drv)->eoi; + (*drv)->eoi = eoi; } } diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index f65419dfdb0f..5ce1b3740a72 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -106,7 +106,7 @@ static struct apic apic_flat __ro_after_init = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, @@ -182,7 +182,7 @@ static struct apic apic_physflat __ro_after_init = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c index e33bea0a2f49..966d7cf10b95 100644 --- a/arch/x86/kernel/apic/apic_noop.c +++ b/arch/x86/kernel/apic/apic_noop.c @@ -29,6 +29,7 @@ static int noop_wakeup_secondary_cpu(int apicid, unsigned long start_eip) { retu static u64 noop_apic_icr_read(void) { return 0; } static int noop_phys_pkg_id(int cpuid_apic, int index_msb) { return 0; } static unsigned int noop_get_apic_id(unsigned long x) { return 0; } +static void noop_apic_eoi(void) { } static u32 noop_apic_read(u32 reg) { @@ -71,7 +72,7 @@ struct apic apic_noop __ro_after_init = { .read = noop_apic_read, .write = noop_apic_write, - .eoi_write = noop_apic_write, + .eoi = noop_apic_eoi, .icr_read = noop_apic_icr_read, .icr_write = noop_apic_icr_write, }; diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 9df018716dcd..63f3d7be9dc7 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -247,7 +247,7 @@ static const struct apic apic_numachip1 __refconst = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, }; @@ -284,7 +284,7 @@ static const struct apic apic_numachip2 __refconst = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, }; diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index ffcd114c96b9..23c29a0a4988 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -105,7 +105,7 @@ static struct apic apic_bigsmp __ro_after_init = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 5facf4266bc5..3671cc1c6814 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -60,7 +60,7 @@ static struct apic apic_default __ro_after_init = { .read = native_apic_mem_read, .write = native_apic_mem_write, - .eoi_write = native_apic_mem_write, + .eoi = native_apic_mem_eoi, .icr_read = native_apic_icr_read, .icr_write = native_apic_icr_write, .wait_icr_idle = apic_mem_wait_icr_idle, diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index 12b9c310f3f5..affbff65e497 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -254,7 +254,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = { .read = native_apic_msr_read, .write = native_apic_msr_write, - .eoi_write = native_apic_msr_eoi_write, + .eoi = native_apic_msr_eoi, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, }; diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 47ee5c679a45..788cdb4ee394 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -169,7 +169,7 @@ static struct apic apic_x2apic_phys __ro_after_init = { .read = native_apic_msr_read, .write = native_apic_msr_write, - .eoi_write = native_apic_msr_eoi_write, + .eoi = native_apic_msr_eoi, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, }; diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 56925147d8ac..1cedb2519938 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -831,7 +831,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { .read = native_apic_msr_read, .write = native_apic_msr_write, - .eoi_write = native_apic_msr_eoi_write, + .eoi = native_apic_msr_eoi, .icr_read = native_x2apic_icr_read, .icr_write = native_x2apic_icr_write, }; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 1cceac5984da..c2e3e7aabb6a 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -332,7 +332,7 @@ static void kvm_register_steal_time(void) static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED; -static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val) +static notrace void kvm_guest_apic_eoi_write(void) { /** * This relies on __test_and_clear_bit to modify the memory @@ -343,7 +343,7 @@ static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val) */ if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi))) return; - apic->native_eoi_write(APIC_EOI, APIC_EOI_ACK); + apic->native_eoi(); } static void kvm_guest_cpu_init(void) @@ -825,7 +825,7 @@ static void __init kvm_guest_init(void) } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) - apic_set_eoi_write(kvm_guest_apic_eoi_write); + apic_set_eoi_cb(kvm_guest_apic_eoi_write); if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { static_branch_enable(&kvm_async_pf_enabled); diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 661eecfdc4ba..804a26b7c85e 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -81,6 +81,11 @@ static void xen_apic_write(u32 reg, u32 val) WARN(1,"register: %x, value: %x\n", reg, val); } +static void xen_apic_eoi(void) +{ + WARN_ON_ONCE(1); +} + static u64 xen_apic_icr_read(void) { return 0; @@ -147,7 +152,7 @@ static struct apic xen_pv_apic = { #endif .read = xen_apic_read, .write = xen_apic_write, - .eoi_write = xen_apic_write, + .eoi = xen_apic_eoi, .icr_read = xen_apic_icr_read, .icr_write = xen_apic_icr_write, -- cgit v1.2.3 From 670c04add6e1a22de7c59e282c138ddcf6c9e5a2 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Wed, 9 Aug 2023 08:16:46 -0700 Subject: x86/apic: Nuke ack_APIC_irq() Yet another wrapper of a wrapper gone along with the outdated comment that this compiles to a single instruction. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Reviewed-by: Wei Liu Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/hyperv/hv_init.c | 2 +- arch/x86/include/asm/apic.h | 10 ---------- arch/x86/kernel/apic/apic.c | 10 +++++----- arch/x86/kernel/apic/io_apic.c | 4 ++-- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/cpu/acrn.c | 2 +- arch/x86/kernel/cpu/mce/amd.c | 2 +- arch/x86/kernel/cpu/mce/threshold.c | 2 +- arch/x86/kernel/cpu/mshyperv.c | 4 ++-- arch/x86/kernel/irq.c | 14 +++++++------- arch/x86/kernel/irq_work.c | 2 +- arch/x86/kernel/kvm.c | 2 +- arch/x86/kernel/smp.c | 8 ++++---- arch/x86/xen/enlighten_hvm.c | 2 +- 14 files changed, 28 insertions(+), 38 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 6c04b52f139b..fc56fc80712b 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -161,7 +161,7 @@ static inline bool hv_reenlightenment_available(void) DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment) { - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(irq_hv_reenlightenment_count); schedule_delayed_work(&hv_reenlightenment_work, HZ/10); } diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 790f15d89415..6ad35741653d 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -402,16 +402,6 @@ static inline void apic_set_eoi_cb(void (*eoi)(void)) {} extern void apic_ack_irq(struct irq_data *data); -static inline void ack_APIC_irq(void) -{ - /* - * ack_APIC_irq() actually gets compiled as a single instruction - * ... yummie. - */ - apic_eoi(); -} - - static inline bool lapic_vector_set_in_irr(unsigned int vector) { u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 33d5ed693f54..b8b801e1f76f 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1076,7 +1076,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) { struct pt_regs *old_regs = set_irq_regs(regs); - ack_APIC_irq(); + apic_eoi(); trace_local_timer_entry(LOCAL_TIMER_VECTOR); local_apic_timer_interrupt(); trace_local_timer_exit(LOCAL_TIMER_VECTOR); @@ -1480,7 +1480,7 @@ static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) * per set bit. */ for_each_set_bit(bit, isr->map, APIC_IR_BITS) - ack_APIC_irq(); + apic_eoi(); return true; } @@ -1492,7 +1492,7 @@ static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr) * interrupt from previous kernel might still have ISR bit set. * * Most probably by now the CPU has serviced that pending interrupt and it - * might not have done the ack_APIC_irq() because it thought, interrupt + * might not have done the apic_eoi() because it thought, interrupt * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear * the ISR bit and cpu thinks it has already serviced the interrupt. Hence * a vector might get locked. It was noticed for timer irq (vector @@ -2147,7 +2147,7 @@ static noinline void handle_spurious_interrupt(u8 vector) if (v & (1 << (vector & 0x1f))) { pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n", vector, smp_processor_id()); - ack_APIC_irq(); + apic_eoi(); } else { pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n", vector, smp_processor_id()); @@ -2198,7 +2198,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */ apic_write(APIC_ESR, 0); v = apic_read(APIC_ESR); - ack_APIC_irq(); + apic_eoi(); atomic_inc(&irq_err_count); apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x", diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index ff2d27e2089a..00da6cf6b07d 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1823,7 +1823,7 @@ static void ioapic_ack_level(struct irq_data *irq_data) * We must acknowledge the irq before we move it or the acknowledge will * not propagate properly. */ - ack_APIC_irq(); + apic_eoi(); /* * Tail end of clearing remote IRR bit (either by delivering the EOI @@ -2046,7 +2046,7 @@ static void unmask_lapic_irq(struct irq_data *data) static void ack_lapic_irq(struct irq_data *data) { - ack_APIC_irq(); + apic_eoi(); } static struct irq_chip lapic_chip __read_mostly = { diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 2ee3f5a20f3d..4a262c61c631 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -907,7 +907,7 @@ static int apic_retrigger_irq(struct irq_data *irqd) void apic_ack_irq(struct irq_data *irqd) { irq_move_irq(irqd); - ack_APIC_irq(); + apic_eoi(); } void apic_ack_edge(struct irq_data *irqd) diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 485441b7f030..bfeb18fad63f 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -51,7 +51,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) * will block the interrupt whose vector is lower than * HYPERVISOR_CALLBACK_VECTOR. */ - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(irq_hv_callback_count); if (acrn_intr_handler) diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index c4ec4ca47e11..c267f43de39e 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -759,7 +759,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error) inc_irq_stat(irq_deferred_error_count); deferred_error_int_vector(); trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR); - ack_APIC_irq(); + apic_eoi(); } /* diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c index 6a059a035021..ef4e7bb5fd88 100644 --- a/arch/x86/kernel/cpu/mce/threshold.c +++ b/arch/x86/kernel/cpu/mce/threshold.c @@ -27,5 +27,5 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_threshold) inc_irq_stat(irq_threshold_count); mce_threshold_vector(); trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR); - ack_APIC_irq(); + apic_eoi(); } diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index c7969e806c64..0100468e72ca 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -119,7 +119,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback) vmbus_handler(); if (ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED) - ack_APIC_irq(); + apic_eoi(); set_irq_regs(old_regs); } @@ -147,7 +147,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) if (hv_stimer0_handler) hv_stimer0_handler(); add_interrupt_randomness(HYPERV_STIMER0_VECTOR); - ack_APIC_irq(); + apic_eoi(); set_irq_regs(old_regs); } diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 9f668d2f3d11..11761c124545 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -49,7 +49,7 @@ void ack_bad_irq(unsigned int irq) * completely. * But only ack when the APIC is enabled -AK */ - ack_APIC_irq(); + apic_eoi(); } #define irq_stats(x) (&per_cpu(irq_stat, x)) @@ -256,7 +256,7 @@ DEFINE_IDTENTRY_IRQ(common_interrupt) if (likely(!IS_ERR_OR_NULL(desc))) { handle_irq(desc, regs); } else { - ack_APIC_irq(); + apic_eoi(); if (desc == VECTOR_UNUSED) { pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n", @@ -280,7 +280,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) { struct pt_regs *old_regs = set_irq_regs(regs); - ack_APIC_irq(); + apic_eoi(); trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); inc_irq_stat(x86_platform_ipis); if (x86_platform_ipi_callback) @@ -310,7 +310,7 @@ EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler); */ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) { - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(kvm_posted_intr_ipis); } @@ -319,7 +319,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) */ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) { - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(kvm_posted_intr_wakeup_ipis); kvm_posted_intr_wakeup_handler(); } @@ -329,7 +329,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) */ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) { - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(kvm_posted_intr_nested_ipis); } #endif @@ -401,6 +401,6 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_thermal) inc_irq_stat(irq_thermal_count); smp_thermal_vector(); trace_thermal_apic_exit(THERMAL_APIC_VECTOR); - ack_APIC_irq(); + apic_eoi(); } #endif diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index 890d4778cd35..ee53419f825e 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c @@ -16,7 +16,7 @@ #ifdef CONFIG_X86_LOCAL_APIC DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work) { - ack_APIC_irq(); + apic_eoi(); trace_irq_work_entry(IRQ_WORK_VECTOR); inc_irq_stat(apic_irq_work_irqs); irq_work_run(); diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c2e3e7aabb6a..f5339e31b440 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -291,7 +291,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) struct pt_regs *old_regs = set_irq_regs(regs); u32 token; - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(irq_hv_callback_count); diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 7eb18ca7bd45..6299bf8a4d7f 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -135,7 +135,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs) */ DEFINE_IDTENTRY_SYSVEC(sysvec_reboot) { - ack_APIC_irq(); + apic_eoi(); cpu_emergency_disable_virtualization(); stop_this_cpu(NULL); } @@ -268,7 +268,7 @@ done: */ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi) { - ack_APIC_irq(); + apic_eoi(); trace_reschedule_entry(RESCHEDULE_VECTOR); inc_irq_stat(irq_resched_count); scheduler_ipi(); @@ -277,7 +277,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi) DEFINE_IDTENTRY_SYSVEC(sysvec_call_function) { - ack_APIC_irq(); + apic_eoi(); trace_call_function_entry(CALL_FUNCTION_VECTOR); inc_irq_stat(irq_call_count); generic_smp_call_function_interrupt(); @@ -286,7 +286,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function) DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single) { - ack_APIC_irq(); + apic_eoi(); trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); inc_irq_stat(irq_call_count); generic_smp_call_function_single_interrupt(); diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index a6820ca940bf..9a192f51f1b0 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -132,7 +132,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback) struct pt_regs *old_regs = set_irq_regs(regs); if (xen_percpu_upcall) - ack_APIC_irq(); + apic_eoi(); inc_irq_stat(irq_hv_callback_count); -- cgit v1.2.3 From 0fa075769cd4af9c568044973e7bdf430cc7c158 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:17 -0700 Subject: x86/apic: Wrap apic->native_eoi() into a helper Prepare for converting the hotpath APIC callbacks to static calls. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 6 ++++++ arch/x86/kernel/kvm.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 6ad35741653d..5d98c4b1a735 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -359,6 +359,11 @@ static inline void apic_eoi(void) apic->eoi(); } +static inline void apic_native_eoi(void) +{ + apic->native_eoi(); +} + static inline u64 apic_icr_read(void) { return apic->icr_read(); @@ -397,6 +402,7 @@ static inline void apic_icr_write(u32 low, u32 high) { } static inline void apic_wait_icr_idle(void) { } static inline u32 safe_apic_wait_icr_idle(void) { return 0; } static inline void apic_set_eoi_cb(void (*eoi)(void)) {} +static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); } #endif /* CONFIG_X86_LOCAL_APIC */ diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index f5339e31b440..54625a4f6142 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -343,7 +343,7 @@ static notrace void kvm_guest_apic_eoi_write(void) */ if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi))) return; - apic->native_eoi(); + apic_native_eoi(); } static void kvm_guest_cpu_init(void) -- cgit v1.2.3 From 3af1e415e4d43128b72af615b346b832694377d3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:18 -0700 Subject: x86/apic: Provide common init infrastructure In preparation for converting the hotpath APIC callbacks to static keys, provide common initialization infrastructure. Lift apic_install_drivers() from probe_64.c and convert all places which switch the apic instance by storing the pointer to use apic_install_driver() as a first step. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 2 ++ arch/x86/kernel/apic/Makefile | 2 +- arch/x86/kernel/apic/apic.c | 31 +----------------------- arch/x86/kernel/apic/apic_flat_64.c | 6 +---- arch/x86/kernel/apic/bigsmp_32.c | 6 ++--- arch/x86/kernel/apic/init.c | 47 +++++++++++++++++++++++++++++++++++++ arch/x86/kernel/apic/probe_32.c | 5 ++-- arch/x86/kernel/apic/probe_64.c | 13 ---------- arch/x86/xen/apic.c | 10 +++----- 9 files changed, 59 insertions(+), 63 deletions(-) create mode 100644 arch/x86/kernel/apic/init.c (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 5d98c4b1a735..90814ec84540 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -344,6 +344,8 @@ extern int lapic_can_unplug_cpu(void); #ifdef CONFIG_X86_LOCAL_APIC +void __init apic_install_driver(struct apic *driver); + static inline u32 apic_read(u32 reg) { return apic->read(reg); diff --git a/arch/x86/kernel/apic/Makefile b/arch/x86/kernel/apic/Makefile index a6fcaf16cdbf..2ee867d796d9 100644 --- a/arch/x86/kernel/apic/Makefile +++ b/arch/x86/kernel/apic/Makefile @@ -7,7 +7,7 @@ # In particualr, smp_apic_timer_interrupt() is called in random places. KCOV_INSTRUMENT := n -obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_common.o apic_noop.o ipi.o vector.o +obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_common.o apic_noop.o ipi.o vector.o init.o obj-y += hw_nmi.o obj-$(CONFIG_X86_IO_APIC) += io_apic.o diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b8b801e1f76f..55f2b9667433 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -236,8 +236,7 @@ static int modern_apic(void) */ static void __init apic_disable(void) { - pr_info("APIC: switched to apic NOOP\n"); - apic = &apic_noop; + apic_install_driver(&apic_noop); } void native_apic_icr_write(u32 low, u32 id) @@ -2486,34 +2485,6 @@ u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) } EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); -#ifdef CONFIG_X86_64 -void __init acpi_wake_cpu_handler_update(wakeup_cpu_handler handler) -{ - struct apic **drv; - - for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) - (*drv)->wakeup_secondary_cpu_64 = handler; -} -#endif - -/* - * Override the generic EOI implementation with an optimized version. - * Only called during early boot when only one CPU is active and with - * interrupts disabled, so we know this does not race with actual APIC driver - * use. - */ -void __init apic_set_eoi_cb(void (*eoi)(void)) -{ - struct apic **drv; - - for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { - /* Should happen once for each apic */ - WARN_ON((*drv)->eoi == eoi); - (*drv)->native_eoi = (*drv)->eoi; - (*drv)->eoi = eoi; - } -} - static void __init apic_bsp_up_setup(void) { #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index 5ce1b3740a72..032a84e2c3cc 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c @@ -143,11 +143,7 @@ static int physflat_acpi_madt_oem_check(char *oem_id, char *oem_table_id) static int physflat_probe(void) { - if (apic == &apic_physflat || num_possible_cpus() > 8 || - jailhouse_paravirt()) - return 1; - - return 0; + return apic == &apic_physflat || num_possible_cpus() > 8 || jailhouse_paravirt(); } static struct apic apic_physflat __ro_after_init = { diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c index 23c29a0a4988..0e5535add4b5 100644 --- a/arch/x86/kernel/apic/bigsmp_32.c +++ b/arch/x86/kernel/apic/bigsmp_32.c @@ -119,10 +119,8 @@ bool __init apic_bigsmp_possible(bool cmdline_override) void __init apic_bigsmp_force(void) { - if (apic != &apic_bigsmp) { - apic = &apic_bigsmp; - pr_info("Overriding APIC driver with bigsmp\n"); - } + if (apic != &apic_bigsmp) + apic_install_driver(&apic_bigsmp); } apic_driver(apic_bigsmp); diff --git a/arch/x86/kernel/apic/init.c b/arch/x86/kernel/apic/init.c new file mode 100644 index 000000000000..25cf39b855db --- /dev/null +++ b/arch/x86/kernel/apic/init.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define pr_fmt(fmt) "APIC: " fmt + +#include + +#include "local.h" + +void __init apic_install_driver(struct apic *driver) +{ + if (apic == driver) + return; + + apic = driver; + + if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid) + apic->max_apic_id = x2apic_max_apicid; + + pr_info("Switched APIC routing to: %s\n", driver->name); +} + +#ifdef CONFIG_X86_64 +void __init acpi_wake_cpu_handler_update(wakeup_cpu_handler handler) +{ + struct apic **drv; + + for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) + (*drv)->wakeup_secondary_cpu_64 = handler; +} +#endif + +/* + * Override the generic EOI implementation with an optimized version. + * Only called during early boot when only one CPU is active and with + * interrupts disabled, so we know this does not race with actual APIC driver + * use. + */ +void __init apic_set_eoi_cb(void (*eoi)(void)) +{ + struct apic **drv; + + for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { + /* Should happen once for each apic */ + WARN_ON((*drv)->eoi == eoi); + (*drv)->native_eoi = (*drv)->eoi; + (*drv)->eoi = eoi; + } +} diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c index 3671cc1c6814..9a06df6cdd68 100644 --- a/arch/x86/kernel/apic/probe_32.c +++ b/arch/x86/kernel/apic/probe_32.c @@ -82,7 +82,7 @@ static int __init parse_apic(char *arg) for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { if (!strcmp((*drv)->name, arg)) { - apic = *drv; + apic_install_driver(*drv); cmdline_apic = 1; return 0; } @@ -129,7 +129,7 @@ void __init x86_32_probe_apic(void) for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { if ((*drv)->probe()) { - apic = *drv; + apic_install_driver(*drv); break; } } @@ -137,5 +137,4 @@ void __init x86_32_probe_apic(void) if (drv == __apicdrivers_end) panic("Didn't find an APIC driver"); } - printk(KERN_INFO "Using APIC driver %s\n", apic->name); } diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c index a12a379280a8..ecdf0c4121e1 100644 --- a/arch/x86/kernel/apic/probe_64.c +++ b/arch/x86/kernel/apic/probe_64.c @@ -13,19 +13,6 @@ #include "local.h" -static __init void apic_install_driver(struct apic *driver) -{ - if (apic == driver) - return; - - apic = driver; - - if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid) - apic->max_apic_id = x2apic_max_apicid; - - pr_info("Switched APIC routing to %s:\n", apic->name); -} - /* Select the appropriate APIC driver */ void __init x86_64_probe_apic(void) { diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 804a26b7c85e..1838aefc632f 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -160,20 +160,16 @@ static struct apic xen_pv_apic = { static void __init xen_apic_check(void) { - if (apic == &xen_pv_apic) - return; - - pr_info("Switched APIC routing from %s to %s.\n", apic->name, - xen_pv_apic.name); - apic = &xen_pv_apic; + apic_install_driver(&xen_pv_apic); } + void __init xen_init_apic(void) { x86_apic_ops.io_apic_read = xen_io_apic_read; /* On PV guests the APIC CPUID bit is disabled so none of the * routines end up executing. */ if (!xen_initial_domain()) - apic = &xen_pv_apic; + apic_install_driver(&xen_pv_apic); x86_platform.apic_post_init = xen_apic_check; } -- cgit v1.2.3 From 3b5244bef15e0ec2b51ae5ea4182e1b674d01551 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Tue, 8 Aug 2023 15:04:18 -0700 Subject: x86/xen/apic: Use standard apic driver mechanism for Xen PV Instead of setting the Xen PV apic driver very early during boot, just use the standard apic driver probing by setting an appropriate x86_init.irqs.intr_mode_init callback. At the same time eliminate xen_apic_check() which has never been used. The #ifdef CONFIG_X86_LOCAL_APIC around the call of xen_init_apic() can be removed, too, as CONFIG_XEN depends on CONFIG_X86_LOCAL_APIC. Signed-off-by: Juergen Gross Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) Link: https://lore.kernel.org/lkml/aa086365-fd02-210f-67c6-5c9175c0dfee@suse.com --- arch/x86/xen/apic.c | 11 ----------- arch/x86/xen/enlighten_pv.c | 4 +--- 2 files changed, 1 insertion(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 1838aefc632f..315ffd8d377f 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -158,19 +158,8 @@ static struct apic xen_pv_apic = { .icr_write = xen_apic_icr_write, }; -static void __init xen_apic_check(void) -{ - apic_install_driver(&xen_pv_apic); -} - void __init xen_init_apic(void) { x86_apic_ops.io_apic_read = xen_io_apic_read; - /* On PV guests the APIC CPUID bit is disabled so none of the - * routines end up executing. */ - if (!xen_initial_domain()) - apic_install_driver(&xen_pv_apic); - - x86_platform.apic_post_init = xen_apic_check; } apic_driver(xen_pv_apic); diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 93b658248d01..c393c44892ac 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -1326,7 +1326,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si) x86_init.resources.memory_setup = xen_memory_setup; x86_init.irqs.intr_mode_select = x86_init_noop; - x86_init.irqs.intr_mode_init = x86_init_noop; + x86_init.irqs.intr_mode_init = x86_64_probe_apic; x86_init.oem.arch_setup = xen_arch_setup; x86_init.oem.banner = xen_banner; x86_init.hyper.init_platform = xen_pv_init_platform; @@ -1366,12 +1366,10 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si) xen_init_capabilities(); -#ifdef CONFIG_X86_LOCAL_APIC /* * set up the basic apic ops. */ xen_init_apic(); -#endif machine_ops = xen_machine_ops; -- cgit v1.2.3 From bef4f379e953af49a9bd81790954e78fcb264920 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:19 -0700 Subject: x86/apic: Provide apic_update_callback() There are already two variants of update mechanism for particular callbacks and virtualization just writes into the data structure. Provide an interface and use a shadow data structure to preserve callbacks so they can be reapplied when the APIC driver is replaced. The extra data structure is intentional as any new callback needs to be also updated in the core code. This also prepares for static calls. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 28 ++++++++++++++++++++++++++++ arch/x86/kernel/apic/init.c | 39 ++++++++++++++++++++++++++++++++++++++- arch/x86/kernel/setup.c | 2 ++ 3 files changed, 68 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 90814ec84540..6e279c1b380d 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -308,6 +308,23 @@ struct apic { char *name; }; +struct apic_override { + void (*eoi)(void); + void (*native_eoi)(void); + void (*write)(u32 reg, u32 v); + u32 (*read)(u32 reg); + void (*send_IPI)(int cpu, int vector); + void (*send_IPI_mask)(const struct cpumask *mask, int vector); + void (*send_IPI_mask_allbutself)(const struct cpumask *msk, int vec); + void (*send_IPI_allbutself)(int vector); + void (*send_IPI_all)(int vector); + void (*send_IPI_self)(int vector); + u64 (*icr_read)(void); + void (*icr_write)(u32 low, u32 high); + int (*wakeup_secondary_cpu)(int apicid, unsigned long start_eip); + int (*wakeup_secondary_cpu_64)(int apicid, unsigned long start_eip); +}; + /* * Pointer to the local APIC driver in use on this system (there's * always just one such driver in use - the kernel decides via an @@ -343,9 +360,17 @@ extern int lapic_can_unplug_cpu(void); #endif #ifdef CONFIG_X86_LOCAL_APIC +extern struct apic_override __x86_apic_override; +void __init apic_setup_apic_calls(void); void __init apic_install_driver(struct apic *driver); +#define apic_update_callback(_callback, _fn) { \ + __x86_apic_override._callback = _fn; \ + apic->_callback = _fn; \ + pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn); \ +} + static inline u32 apic_read(u32 reg) { return apic->read(reg); @@ -405,6 +430,9 @@ static inline void apic_wait_icr_idle(void) { } static inline u32 safe_apic_wait_icr_idle(void) { return 0; } static inline void apic_set_eoi_cb(void (*eoi)(void)) {} static inline void apic_native_eoi(void) { WARN_ON_ONCE(1); } +static inline void apic_setup_apic_calls(void) { } + +#define apic_update_callback(_callback, _fn) do { } while (0) #endif /* CONFIG_X86_LOCAL_APIC */ diff --git a/arch/x86/kernel/apic/init.c b/arch/x86/kernel/apic/init.c index 25cf39b855db..dab3afa8a86a 100644 --- a/arch/x86/kernel/apic/init.c +++ b/arch/x86/kernel/apic/init.c @@ -5,6 +5,37 @@ #include "local.h" +/* The container for function call overrides */ +struct apic_override __x86_apic_override __initdata; + +#define apply_override(__cb) \ + if (__x86_apic_override.__cb) \ + apic->__cb = __x86_apic_override.__cb + +static __init void restore_override_callbacks(void) +{ + apply_override(eoi); + apply_override(native_eoi); + apply_override(write); + apply_override(read); + apply_override(send_IPI); + apply_override(send_IPI_mask); + apply_override(send_IPI_mask_allbutself); + apply_override(send_IPI_allbutself); + apply_override(send_IPI_all); + apply_override(send_IPI_self); + apply_override(icr_read); + apply_override(icr_write); + apply_override(wakeup_secondary_cpu); + apply_override(wakeup_secondary_cpu_64); +} + +void __init apic_setup_apic_calls(void) +{ + /* Ensure that the default APIC has native_eoi populated */ + apic->native_eoi = apic->eoi; +} + void __init apic_install_driver(struct apic *driver) { if (apic == driver) @@ -15,6 +46,13 @@ void __init apic_install_driver(struct apic *driver) if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid) apic->max_apic_id = x2apic_max_apicid; + /* Copy the original eoi() callback as KVM/HyperV might overwrite it */ + if (!apic->native_eoi) + apic->native_eoi = apic->eoi; + + /* Apply any already installed callback overrides */ + restore_override_callbacks(); + pr_info("Switched APIC routing to: %s\n", driver->name); } @@ -41,7 +79,6 @@ void __init apic_set_eoi_cb(void (*eoi)(void)) for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { /* Should happen once for each apic */ WARN_ON((*drv)->eoi == eoi); - (*drv)->native_eoi = (*drv)->eoi; (*drv)->eoi = eoi; } } diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index f83d02b4fb25..b9145a63da77 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1017,6 +1017,8 @@ void __init setup_arch(char **cmdline_p) x86_report_nx(); + apic_setup_apic_calls(); + if (acpi_mps_check()) { #ifdef CONFIG_X86_LOCAL_APIC apic_is_disabled = true; -- cgit v1.2.3 From 2744a7ce34a776444225f20ae11ead6e980a129a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:20 -0700 Subject: x86/apic: Replace acpi_wake_cpu_handler_update() and apic_set_eoi_cb() Switch them over to apic_update_callback() and remove the code. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Reviewed-by: Wei Liu Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/hyperv/hv_apic.c | 2 +- arch/x86/hyperv/hv_vtl.c | 2 +- arch/x86/include/asm/apic.h | 3 --- arch/x86/kernel/acpi/boot.c | 2 +- arch/x86/kernel/apic/init.c | 27 --------------------------- arch/x86/kernel/kvm.c | 4 ++-- 6 files changed, 5 insertions(+), 35 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index 5513f60b3cdd..bfb02f68c0f1 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -310,7 +310,7 @@ void __init hv_apic_init(void) * lazy EOI when available, but the same accessor works for * both xapic and x2apic because the field layout is the same. */ - apic_set_eoi_cb(hv_apic_eoi_write); + apic_update_callback(eoi, hv_apic_eoi_write); if (!x2apic_enabled()) { apic->read = hv_apic_read; apic->write = hv_apic_write; diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index 85d38b9f3586..9976b771a23a 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -222,7 +222,7 @@ static int __init hv_vtl_early_init(void) "Please add 'noxsave' to the kernel command line.\n"); real_mode_header = &hv_vtl_real_mode_header; - apic->wakeup_secondary_cpu_64 = hv_vtl_wakeup_secondary_cpu; + apic_update_callback(wakeup_secondary_cpu_64, hv_vtl_wakeup_secondary_cpu); return 0; } diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 6e279c1b380d..1742f972d254 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -417,8 +417,6 @@ static inline bool apic_id_valid(u32 apic_id) return apic_id <= apic->max_apic_id; } -extern void __init apic_set_eoi_cb(void (*eoi)(void)); - #else /* CONFIG_X86_LOCAL_APIC */ static inline u32 apic_read(u32 reg) { return 0; } @@ -478,7 +476,6 @@ static inline unsigned int read_apic_id(void) #ifdef CONFIG_X86_64 typedef int (*wakeup_cpu_handler)(int apicid, unsigned long start_eip); -extern void acpi_wake_cpu_handler_update(wakeup_cpu_handler handler); extern int default_acpi_madt_oem_check(char *, char *); extern void x86_64_probe_apic(void); #else diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 7f5b257aaee3..2374989f3956 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -1174,7 +1174,7 @@ static int __init acpi_parse_mp_wake(union acpi_subtable_headers *header, acpi_mp_wake_mailbox_paddr = mp_wake->base_address; - acpi_wake_cpu_handler_update(acpi_wakeup_cpu); + apic_update_callback(wakeup_secondary_cpu_64, acpi_wakeup_cpu); return 0; } diff --git a/arch/x86/kernel/apic/init.c b/arch/x86/kernel/apic/init.c index dab3afa8a86a..d7f4aca52f26 100644 --- a/arch/x86/kernel/apic/init.c +++ b/arch/x86/kernel/apic/init.c @@ -55,30 +55,3 @@ void __init apic_install_driver(struct apic *driver) pr_info("Switched APIC routing to: %s\n", driver->name); } - -#ifdef CONFIG_X86_64 -void __init acpi_wake_cpu_handler_update(wakeup_cpu_handler handler) -{ - struct apic **drv; - - for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) - (*drv)->wakeup_secondary_cpu_64 = handler; -} -#endif - -/* - * Override the generic EOI implementation with an optimized version. - * Only called during early boot when only one CPU is active and with - * interrupts disabled, so we know this does not race with actual APIC driver - * use. - */ -void __init apic_set_eoi_cb(void (*eoi)(void)) -{ - struct apic **drv; - - for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) { - /* Should happen once for each apic */ - WARN_ON((*drv)->eoi == eoi); - (*drv)->eoi = eoi; - } -} diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 54625a4f6142..c86cedbb0bd5 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -332,7 +332,7 @@ static void kvm_register_steal_time(void) static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED; -static notrace void kvm_guest_apic_eoi_write(void) +static notrace __maybe_unused void kvm_guest_apic_eoi_write(void) { /** * This relies on __test_and_clear_bit to modify the memory @@ -825,7 +825,7 @@ static void __init kvm_guest_init(void) } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) - apic_set_eoi_cb(kvm_guest_apic_eoi_write); + apic_update_callback(eoi, kvm_guest_apic_eoi_write); if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) { static_branch_enable(&kvm_async_pf_enabled); -- cgit v1.2.3 From d6f361ea706710f8d582bb176793b779a3b8b2ca Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:20 -0700 Subject: x86/apic: Convert other overrides to apic_update_callback() Convert all places which just assign a new function directly to the apic callback to use apic_update_callback() which prepares for using static calls. Mark snp_set_wakeup_secondary_cpu() and kvm_setup_pv_ipi() __init, as they are only invoked from init code and otherwise trigger a section mismatch as they are now invoking a __init function. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/hyperv/hv_apic.c | 20 ++++++++++---------- arch/x86/kernel/kvm.c | 6 +++--- arch/x86/kernel/sev.c | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index bfb02f68c0f1..72d9931da3a2 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -288,12 +288,12 @@ void __init hv_apic_init(void) */ orig_apic = *apic; - apic->send_IPI = hv_send_ipi; - apic->send_IPI_mask = hv_send_ipi_mask; - apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself; - apic->send_IPI_allbutself = hv_send_ipi_allbutself; - apic->send_IPI_all = hv_send_ipi_all; - apic->send_IPI_self = hv_send_ipi_self; + apic_update_callback(send_IPI, hv_send_ipi); + apic_update_callback(send_IPI_mask, hv_send_ipi_mask); + apic_update_callback(send_IPI_mask_allbutself, hv_send_ipi_mask_allbutself); + apic_update_callback(send_IPI_allbutself, hv_send_ipi_allbutself); + apic_update_callback(send_IPI_all, hv_send_ipi_all); + apic_update_callback(send_IPI_self, hv_send_ipi_self); } if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) { @@ -312,10 +312,10 @@ void __init hv_apic_init(void) */ apic_update_callback(eoi, hv_apic_eoi_write); if (!x2apic_enabled()) { - apic->read = hv_apic_read; - apic->write = hv_apic_write; - apic->icr_write = hv_apic_icr_write; - apic->icr_read = hv_apic_icr_read; + apic_update_callback(read, hv_apic_read); + apic_update_callback(write, hv_apic_write); + apic_update_callback(icr_write, hv_apic_icr_write); + apic_update_callback(icr_read, hv_apic_icr_read); } } } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c86cedbb0bd5..6a36db4f79fd 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -622,10 +622,10 @@ late_initcall(setup_efi_kvm_sev_migration); /* * Set the IPI entry points */ -static void kvm_setup_pv_ipi(void) +static __init void kvm_setup_pv_ipi(void) { - apic->send_IPI_mask = kvm_send_ipi_mask; - apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself; + apic_update_callback(send_IPI_mask, kvm_send_ipi_mask); + apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself); pr_info("setup PV IPIs\n"); } diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c index 1ee7bed453de..a2b50ae61b72 100644 --- a/arch/x86/kernel/sev.c +++ b/arch/x86/kernel/sev.c @@ -1089,7 +1089,7 @@ static int wakeup_cpu_via_vmgexit(int apic_id, unsigned long start_ip) return ret; } -void snp_set_wakeup_secondary_cpu(void) +void __init snp_set_wakeup_secondary_cpu(void) { if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) return; @@ -1099,7 +1099,7 @@ void snp_set_wakeup_secondary_cpu(void) * required method to start APs under SNP. If the hypervisor does * not support AP creation, then no APs will be started. */ - apic->wakeup_secondary_cpu = wakeup_cpu_via_vmgexit; + apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit); } int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh) -- cgit v1.2.3 From ac72b92d8cec9ee96f15301a4d5a3dd455c0a7cf Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:21 -0700 Subject: x86/xen/apic: Mark apic __ro_after_init Nothing can change it post init. Remove the 32bit callbacks and comments as XENPV is strictly 64bit. While at it mop up the whitespace damage which causes eyebleed due to an editor which is highlighting it. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) Cc: Juergen Gross --- arch/x86/xen/apic.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) (limited to 'arch') diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 315ffd8d377f..7ad91225fdf4 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -123,43 +123,41 @@ static int xen_cpu_present_to_apicid(int cpu) return BAD_APICID; } -static struct apic xen_pv_apic = { - .name = "Xen PV", - .probe = xen_apic_probe_pv, +static struct apic xen_pv_apic __ro_after_init = { + .name = "Xen PV", + .probe = xen_apic_probe_pv, .acpi_madt_oem_check = xen_madt_oem_check, /* .delivery_mode and .dest_mode_logical not used by XENPV */ .disable_esr = 0, - .check_apicid_used = default_check_apicid_used, /* Used on 32-bit */ - .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ .cpu_present_to_apicid = xen_cpu_present_to_apicid, .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ .max_apic_id = UINT_MAX, - .get_apic_id = xen_get_apic_id, - .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ + .get_apic_id = xen_get_apic_id, + .set_apic_id = xen_set_apic_id, .calc_dest_apicid = apic_flat_calc_apicid, #ifdef CONFIG_SMP - .send_IPI_mask = xen_send_IPI_mask, - .send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself, - .send_IPI_allbutself = xen_send_IPI_allbutself, - .send_IPI_all = xen_send_IPI_all, - .send_IPI_self = xen_send_IPI_self, + .send_IPI_mask = xen_send_IPI_mask, + .send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself, + .send_IPI_allbutself = xen_send_IPI_allbutself, + .send_IPI_all = xen_send_IPI_all, + .send_IPI_self = xen_send_IPI_self, #endif .read = xen_apic_read, .write = xen_apic_write, .eoi = xen_apic_eoi, - .icr_read = xen_apic_icr_read, - .icr_write = xen_apic_icr_write, + .icr_read = xen_apic_icr_read, + .icr_write = xen_apic_icr_write, }; +apic_driver(xen_pv_apic); void __init xen_init_apic(void) { x86_apic_ops.io_apic_read = xen_io_apic_read; } -apic_driver(xen_pv_apic); -- cgit v1.2.3 From 54271fb0b78511048aa7910d593994f2ca055e68 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:21 -0700 Subject: x86/apic: Mark all hotpath APIC callback wrappers __always_inline There is no value for instrumentation to look at those wrappers and with the upcoming conversion to static calls even less so. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 1742f972d254..28192a221197 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -371,48 +371,48 @@ void __init apic_install_driver(struct apic *driver); pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn); \ } -static inline u32 apic_read(u32 reg) +static __always_inline u32 apic_read(u32 reg) { return apic->read(reg); } -static inline void apic_write(u32 reg, u32 val) +static __always_inline void apic_write(u32 reg, u32 val) { apic->write(reg, val); } -static inline void apic_eoi(void) +static __always_inline void apic_eoi(void) { apic->eoi(); } -static inline void apic_native_eoi(void) +static __always_inline void apic_native_eoi(void) { apic->native_eoi(); } -static inline u64 apic_icr_read(void) +static __always_inline u64 apic_icr_read(void) { return apic->icr_read(); } -static inline void apic_icr_write(u32 low, u32 high) +static __always_inline void apic_icr_write(u32 low, u32 high) { apic->icr_write(low, high); } -static inline void apic_wait_icr_idle(void) +static __always_inline void apic_wait_icr_idle(void) { if (apic->wait_icr_idle) apic->wait_icr_idle(); } -static inline u32 safe_apic_wait_icr_idle(void) +static __always_inline u32 safe_apic_wait_icr_idle(void) { return apic->safe_wait_icr_idle ? apic->safe_wait_icr_idle() : 0; } -static inline bool apic_id_valid(u32 apic_id) +static __always_inline bool apic_id_valid(u32 apic_id) { return apic_id <= apic->max_apic_id; } -- cgit v1.2.3 From 28b8235238fa39a1b5c5820e7f5b14e7f104aac0 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Wed, 9 Aug 2023 08:21:45 -0700 Subject: x86/apic: Wrap IPI calls into helper functions Move them to one place so the static call conversion gets simpler. No functional change. [ dhansen: merge against recent x86/apic changes ] Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/hyperv/hv_spinlock.c | 2 +- arch/x86/include/asm/apic.h | 30 ++++++++++++++++++++++++++++++ arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/apic/hw_nmi.c | 4 +++- arch/x86/kernel/apic/ipi.c | 16 ++++++++-------- arch/x86/kernel/apic/vector.c | 2 +- arch/x86/kernel/cpu/mce/inject.c | 3 +-- arch/x86/kernel/irq_work.c | 2 +- arch/x86/kernel/nmi_selftest.c | 2 +- arch/x86/kernel/smp.c | 2 +- arch/x86/kvm/vmx/posted_intr.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/platform/uv/uv_nmi.c | 2 +- 13 files changed, 51 insertions(+), 20 deletions(-) (limited to 'arch') diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index 91cfe698bde0..737d6f7a6155 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -20,7 +20,7 @@ static bool __initdata hv_pvspin = true; static void hv_qlock_kick(int cpu) { - apic->send_IPI(cpu, X86_PLATFORM_IPI_VECTOR); + __apic_send_IPI(cpu, X86_PLATFORM_IPI_VECTOR); } static void hv_qlock_wait(u8 *byte, u8 val) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 28192a221197..62471e20fce0 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -401,6 +401,36 @@ static __always_inline void apic_icr_write(u32 low, u32 high) apic->icr_write(low, high); } +static __always_inline void __apic_send_IPI(int cpu, int vector) +{ + apic->send_IPI(cpu, vector); +} + +static __always_inline void __apic_send_IPI_mask(const struct cpumask *mask, int vector) +{ + apic->send_IPI_mask(mask, vector); +} + +static __always_inline void __apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) +{ + apic->send_IPI_mask_allbutself(mask, vector); +} + +static __always_inline void __apic_send_IPI_allbutself(int vector) +{ + apic->send_IPI_allbutself(vector); +} + +static __always_inline void __apic_send_IPI_all(int vector) +{ + apic->send_IPI_all(vector); +} + +static __always_inline void __apic_send_IPI_self(int vector) +{ + apic->send_IPI_self(vector); +} + static __always_inline void apic_wait_icr_idle(void) { if (apic->wait_icr_idle) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 55f2b9667433..760adac3d1a8 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -502,7 +502,7 @@ static int lapic_timer_set_oneshot(struct clock_event_device *evt) static void lapic_timer_broadcast(const struct cpumask *mask) { #ifdef CONFIG_SMP - apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR); + __apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR); #endif } diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index 34a992e275ef..07a3dd14a6da 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -21,6 +21,8 @@ #include #include +#include "local.h" + #ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF u64 hw_nmi_get_sample_period(int watchdog_thresh) { @@ -31,7 +33,7 @@ u64 hw_nmi_get_sample_period(int watchdog_thresh) #ifdef arch_trigger_cpumask_backtrace static void nmi_raise_cpu_backtrace(cpumask_t *mask) { - apic->send_IPI_mask(mask, NMI_VECTOR); + __apic_send_IPI_mask(mask, NMI_VECTOR); } void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 460df029e511..b54b2a6a4c32 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -54,9 +54,9 @@ void apic_send_IPI_allbutself(unsigned int vector) return; if (static_branch_likely(&apic_use_ipi_shorthand)) - apic->send_IPI_allbutself(vector); + __apic_send_IPI_allbutself(vector); else - apic->send_IPI_mask_allbutself(cpu_online_mask, vector); + __apic_send_IPI_mask_allbutself(cpu_online_mask, vector); } /* @@ -70,12 +70,12 @@ void native_smp_send_reschedule(int cpu) WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu); return; } - apic->send_IPI(cpu, RESCHEDULE_VECTOR); + __apic_send_IPI(cpu, RESCHEDULE_VECTOR); } void native_send_call_func_single_ipi(int cpu) { - apic->send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR); + __apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR); } void native_send_call_func_ipi(const struct cpumask *mask) @@ -87,14 +87,14 @@ void native_send_call_func_ipi(const struct cpumask *mask) goto sendmask; if (cpumask_test_cpu(cpu, mask)) - apic->send_IPI_all(CALL_FUNCTION_VECTOR); + __apic_send_IPI_all(CALL_FUNCTION_VECTOR); else if (num_online_cpus() > 1) - apic->send_IPI_allbutself(CALL_FUNCTION_VECTOR); + __apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR); return; } sendmask: - apic->send_IPI_mask(mask, CALL_FUNCTION_VECTOR); + __apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR); } #endif /* CONFIG_SMP */ @@ -221,7 +221,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask, */ void default_send_IPI_single(int cpu, int vector) { - apic->send_IPI_mask(cpumask_of(cpu), vector); + __apic_send_IPI_mask(cpumask_of(cpu), vector); } void default_send_IPI_allbutself(int vector) diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 4a262c61c631..319448d87b99 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -898,7 +898,7 @@ static int apic_retrigger_irq(struct irq_data *irqd) unsigned long flags; raw_spin_lock_irqsave(&vector_lock, flags); - apic->send_IPI(apicd->cpu, apicd->vector); + __apic_send_IPI(apicd->cpu, apicd->vector); raw_spin_unlock_irqrestore(&vector_lock, flags); return 1; diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 12cf2e7ca33c..4d8d4bcf915d 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -270,8 +270,7 @@ static void __maybe_unused raise_mce(struct mce *m) mce_irq_ipi, NULL, 0); preempt_enable(); } else if (m->inject_flags & MCJ_NMI_BROADCAST) - apic->send_IPI_mask(mce_inject_cpumask, - NMI_VECTOR); + __apic_send_IPI_mask(mce_inject_cpumask, NMI_VECTOR); } start = jiffies; while (!cpumask_empty(mce_inject_cpumask)) { diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index ee53419f825e..b0a24deab4a1 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c @@ -28,7 +28,7 @@ void arch_irq_work_raise(void) if (!arch_irq_work_has_interrupt()) return; - apic->send_IPI_self(IRQ_WORK_VECTOR); + __apic_send_IPI_self(IRQ_WORK_VECTOR); apic_wait_icr_idle(); } #endif diff --git a/arch/x86/kernel/nmi_selftest.c b/arch/x86/kernel/nmi_selftest.c index a1a96df3dff1..e93a8545c74d 100644 --- a/arch/x86/kernel/nmi_selftest.c +++ b/arch/x86/kernel/nmi_selftest.c @@ -75,7 +75,7 @@ static void __init test_nmi_ipi(struct cpumask *mask) /* sync above data before sending NMI */ wmb(); - apic->send_IPI_mask(mask, NMI_VECTOR); + __apic_send_IPI_mask(mask, NMI_VECTOR); /* Don't wait longer than a second */ timeout = USEC_PER_SEC; diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 6299bf8a4d7f..6eb06d001bcc 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -237,7 +237,7 @@ static void native_stop_other_cpus(int wait) pr_emerg("Shutting down cpus with NMI\n"); for_each_cpu(cpu, &cpus_stop_mask) - apic->send_IPI(cpu, NMI_VECTOR); + __apic_send_IPI(cpu, NMI_VECTOR); } /* * Don't wait longer than 10 ms if the caller didn't diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 94c38bea60e7..af662312fd07 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -175,7 +175,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) * scheduled out). */ if (pi_test_on(&new)) - apic->send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); + __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); local_irq_restore(flags); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index df461f387e20..b483a8baaacf 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4179,7 +4179,7 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, */ if (vcpu != kvm_get_running_vcpu()) - apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); + __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); return; } #endif diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c index a60af0230e27..85f9aa02075a 100644 --- a/arch/x86/platform/uv/uv_nmi.c +++ b/arch/x86/platform/uv/uv_nmi.c @@ -601,7 +601,7 @@ static void uv_nmi_nr_cpus_ping(void) for_each_cpu(cpu, uv_nmi_cpu_mask) uv_cpu_nmi_per(cpu).pinging = 1; - apic->send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI); + __apic_send_IPI_mask(uv_nmi_cpu_mask, APIC_DM_NMI); } /* Clean up flags for CPU's that ignored both NMI and ping */ -- cgit v1.2.3 From 3b7c27e6789911359867a9e8d3d7889fc94a3d55 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:23 -0700 Subject: x86/apic: Provide static call infrastructure for APIC callbacks Declare and define the static calls for the hotpath APIC callbacks. Note this deliberately uses STATIC_CALL_NULL() because otherwise it would be required to have the definitions in the 32bit and the 64bit default APIC implementations and it's hard to keep the calls in sync. The other option would be to have stub functions for each callback type. Not pretty either So the NULL capable calls are used and filled in during early boot after the static key infrastructure has been initialized. The calls will be static_call() except for the wait_irc_idle() callback which is valid to be NULL for X2APIC systems. Update the calls when a new APIC driver is installed and when a callback override is invoked. Export the trampolines for the two calls which are used in KVM and MCE error inject modules. Test the setup and let the next step convert the inline wrappers to make it effective. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Sohil Mehta Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) --- arch/x86/include/asm/apic.h | 21 ++++++++++++++++++ arch/x86/kernel/apic/init.c | 53 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 62471e20fce0..ddafd14f9872 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -3,6 +3,7 @@ #define _ASM_X86_APIC_H #include +#include #include #include @@ -368,9 +369,29 @@ void __init apic_install_driver(struct apic *driver); #define apic_update_callback(_callback, _fn) { \ __x86_apic_override._callback = _fn; \ apic->_callback = _fn; \ + static_call_update(apic_call_##_callback, _fn); \ pr_info("APIC: %s() replaced with %ps()\n", #_callback, _fn); \ } +#define DECLARE_APIC_CALL(__cb) \ + DECLARE_STATIC_CALL(apic_call_##__cb, *apic->__cb) + +DECLARE_APIC_CALL(eoi); +DECLARE_APIC_CALL(native_eoi); +DECLARE_APIC_CALL(icr_read); +DECLARE_APIC_CALL(icr_write); +DECLARE_APIC_CALL(read); +DECLARE_APIC_CALL(send_IPI); +DECLARE_APIC_CALL(send_IPI_mask); +DECLARE_APIC_CALL(send_IPI_mask_allbutself); +DECLARE_APIC_CALL(send_IPI_allbutself); +DECLARE_APIC_CALL(send_IPI_all); +DECLARE_APIC_CALL(send_IPI_self); +DECLARE_APIC_CALL(wait_icr_idle); +DECLARE_APIC_CALL(wakeup_secondary_cpu); +DECLARE_APIC_CALL(wakeup_secondary_cpu_64); +DECLARE_APIC_CALL(write); + static __always_inline u32 apic_read(u32 reg) { return apic->read(reg); diff --git a/arch/x86/kernel/apic/init.c b/arch/x86/kernel/apic/init.c index d7f4aca52f26..821e2e536f19 100644 --- a/arch/x86/kernel/apic/init.c +++ b/arch/x86/kernel/apic/init.c @@ -5,6 +5,34 @@ #include "local.h" +/* + * Use DEFINE_STATIC_CALL_NULL() to avoid having to provide stub functions + * for each callback. The callbacks are setup during boot and all except + * wait_icr_idle() must be initialized before usage. The IPI wrappers + * use static_call() and not static_call_cond() to catch any fails. + */ +#define DEFINE_APIC_CALL(__cb) \ + DEFINE_STATIC_CALL_NULL(apic_call_##__cb, *apic->__cb) + +DEFINE_APIC_CALL(eoi); +DEFINE_APIC_CALL(native_eoi); +DEFINE_APIC_CALL(icr_read); +DEFINE_APIC_CALL(icr_write); +DEFINE_APIC_CALL(read); +DEFINE_APIC_CALL(send_IPI); +DEFINE_APIC_CALL(send_IPI_mask); +DEFINE_APIC_CALL(send_IPI_mask_allbutself); +DEFINE_APIC_CALL(send_IPI_allbutself); +DEFINE_APIC_CALL(send_IPI_all); +DEFINE_APIC_CALL(send_IPI_self); +DEFINE_APIC_CALL(wait_icr_idle); +DEFINE_APIC_CALL(wakeup_secondary_cpu); +DEFINE_APIC_CALL(wakeup_secondary_cpu_64); +DEFINE_APIC_CALL(write); + +EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_mask); +EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_self); + /* The container for function call overrides */ struct apic_override __x86_apic_override __initdata; @@ -30,10 +58,34 @@ static __init void restore_override_callbacks(void) apply_override(wakeup_secondary_cpu_64); } +#define update_call(__cb) \ + static_call_update(apic_call_##__cb, *apic->__cb) + +static __init void update_static_calls(void) +{ + update_call(eoi); + update_call(native_eoi); + update_call(write); + update_call(read); + update_call(send_IPI); + update_call(send_IPI_mask); + update_call(send_IPI_mask_allbutself); + update_call(send_IPI_allbutself); + update_call(send_IPI_all); + update_call(send_IPI_self); + update_call(icr_read); + update_call(icr_write); + update_call(wait_icr_idle); + update_call(wakeup_secondary_cpu); + update_call(wakeup_secondary_cpu_64); +} + void __init apic_setup_apic_calls(void) { /* Ensure that the default APIC has native_eoi populated */ apic->native_eoi = apic->eoi; + update_static_calls(); + pr_info("Static calls initialized\n"); } void __init apic_install_driver(struct apic *driver) @@ -52,6 +104,7 @@ void __init apic_install_driver(struct apic *driver) /* Apply any already installed callback overrides */ restore_override_callbacks(); + update_static_calls(); pr_info("Switched APIC routing to: %s\n", driver->name); } -- cgit v1.2.3 From f8542a55499a69a859c84866b66f0df43933e563 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 8 Aug 2023 15:04:24 -0700 Subject: x86/apic: Turn on static calls Convert all the APIC callback inline wrappers from apic->foo() to static_call(apic_call_foo)(), except for the safe_wait_icr_idle() one which is only used during SMP bringup when sending INIT/SIPI. That really can do the conditional callback. The regular wait_icr_idle() matters as it is used in irq_work_raise(), so X2APIC machines spare the conditional. Signed-off-by: Thomas Gleixner Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Juergen Gross # Xen PV (dom0 and unpriv. guest) Tested-by: Michael Kelley Tested-by: Sohil Mehta --- arch/x86/include/asm/apic.h | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index ddafd14f9872..5af4ec1a0f71 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -394,68 +394,67 @@ DECLARE_APIC_CALL(write); static __always_inline u32 apic_read(u32 reg) { - return apic->read(reg); + return static_call(apic_call_read)(reg); } static __always_inline void apic_write(u32 reg, u32 val) { - apic->write(reg, val); + static_call(apic_call_write)(reg, val); } static __always_inline void apic_eoi(void) { - apic->eoi(); + static_call(apic_call_eoi)(); } static __always_inline void apic_native_eoi(void) { - apic->native_eoi(); + static_call(apic_call_native_eoi)(); } static __always_inline u64 apic_icr_read(void) { - return apic->icr_read(); + return static_call(apic_call_icr_read)(); } static __always_inline void apic_icr_write(u32 low, u32 high) { - apic->icr_write(low, high); + static_call(apic_call_icr_write)(low, high); } static __always_inline void __apic_send_IPI(int cpu, int vector) { - apic->send_IPI(cpu, vector); + static_call(apic_call_send_IPI)(cpu, vector); } static __always_inline void __apic_send_IPI_mask(const struct cpumask *mask, int vector) { - apic->send_IPI_mask(mask, vector); + static_call_mod(apic_call_send_IPI_mask)(mask, vector); } static __always_inline void __apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector) { - apic->send_IPI_mask_allbutself(mask, vector); + static_call(apic_call_send_IPI_mask_allbutself)(mask, vector); } static __always_inline void __apic_send_IPI_allbutself(int vector) { - apic->send_IPI_allbutself(vector); + static_call(apic_call_send_IPI_allbutself)(vector); } static __always_inline void __apic_send_IPI_all(int vector) { - apic->send_IPI_all(vector); + static_call(apic_call_send_IPI_all)(vector); } static __always_inline void __apic_send_IPI_self(int vector) { - apic->send_IPI_self(vector); + static_call_mod(apic_call_send_IPI_self)(vector); } static __always_inline void apic_wait_icr_idle(void) { - if (apic->wait_icr_idle) - apic->wait_icr_idle(); + static_call_cond(apic_call_wait_icr_idle)(); } static __always_inline u32 safe_apic_wait_icr_idle(void) -- cgit v1.2.3