diff options
| author | Mark Brown <broonie@kernel.org> | 2026-07-28 17:48:37 +0100 |
|---|---|---|
| committer | Mark Brown <broonie@kernel.org> | 2026-07-28 17:48:38 +0100 |
| commit | 514804e862d3ff0b4353597b5ea84b6c39c09b9f (patch) | |
| tree | 0937084a6cab11d18cb773fcfe8f9d7551b76d79 /arch/x86/kernel | |
| parent | 1254e4c63053bfa54261442cc86f3f8e8c03465e (diff) | |
| parent | 38eccba72916c4939010bc616a88efb88fb81baf (diff) | |
| download | linux-next-514804e862d3ff0b4353597b5ea84b6c39c09b9f.tar.gz linux-next-514804e862d3ff0b4353597b5ea84b6c39c09b9f.zip | |
Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
Diffstat (limited to 'arch/x86/kernel')
56 files changed, 685 insertions, 792 deletions
diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index be4c5e9e5ff6..bbade0da5130 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -241,7 +241,6 @@ EXPORT_SYMBOL_GPL(amd_detect_prefcore); */ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) { - enum x86_topology_cpu_type core_type = get_topology_cpu_type(&cpu_data(cpu)); bool prefcore; int ret; u32 tmp; @@ -273,16 +272,18 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) /* detect if running on heterogeneous design */ if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { - switch (core_type) { + switch (cpu_data(cpu).topo.cpu_type) { case TOPO_CPU_TYPE_UNKNOWN: + case TOPO_CPU_TYPE_ANY: pr_warn("Undefined core type found for cpu %d\n", cpu); break; case TOPO_CPU_TYPE_PERFORMANCE: /* use the max scale for performance cores */ *numerator = CPPC_HIGHEST_PERF_PERFORMANCE; return 0; + case TOPO_CPU_TYPE_LOW_POWER: case TOPO_CPU_TYPE_EFFICIENCY: - /* use the highest perf value for efficiency cores */ + /* use the highest perf value for efficiency and low-power cores */ ret = amd_get_highest_perf(cpu, &tmp); if (ret) return ret; diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 91fa262f0e30..8dfe98784bf9 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -61,6 +61,7 @@ int x86_acpi_suspend_lowlevel(void) { struct wakeup_header *header = (struct wakeup_header *) __va(real_mode_header->wakeup_header); + struct msr val; if (header->signature != WAKEUP_HEADER_SIGNATURE) { printk(KERN_ERR "wakeup header does not match\n"); @@ -82,13 +83,10 @@ int x86_acpi_suspend_lowlevel(void) * with 2-MB L2 Cache and Intel® Processor A100 and A110 on 90 * nm process with 512-KB L2 Cache Specification Update". */ - if (!rdmsr_safe(MSR_EFER, - &header->pmode_efer_low, - &header->pmode_efer_high) && - !wrmsr_safe(MSR_EFER, - header->pmode_efer_low, - header->pmode_efer_high)) + if (!rdmsrq_safe(MSR_EFER, &val.q) && !wrmsrq_safe(MSR_EFER, val.q)) header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_EFER); + header->pmode_efer_low = val.l; + header->pmode_efer_high = val.h; #endif /* !CONFIG_64BIT */ header->pmode_cr0 = read_cr0(); @@ -96,14 +94,12 @@ int x86_acpi_suspend_lowlevel(void) header->pmode_cr4 = __read_cr4(); header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4); } - if (!rdmsr_safe(MSR_IA32_MISC_ENABLE, - &header->pmode_misc_en_low, - &header->pmode_misc_en_high) && - !wrmsr_safe(MSR_IA32_MISC_ENABLE, - header->pmode_misc_en_low, - header->pmode_misc_en_high)) + if (!rdmsrq_safe(MSR_IA32_MISC_ENABLE, &val.q) && + !wrmsrq_safe(MSR_IA32_MISC_ENABLE, val.q)) header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE); + header->pmode_misc_en_low = val.l; + header->pmode_misc_en_high = val.h; header->realmode_flags = acpi_realmode_flags; header->real_magic = 0x12345678; diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62936a3bde19..da97a5c74425 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -41,15 +41,6 @@ static int __init debug_alt(char *str) } __setup("debug-alternative", debug_alt); -static int noreplace_smp; - -static int __init setup_noreplace_smp(char *str) -{ - noreplace_smp = 1; - return 1; -} -__setup("noreplace-smp", setup_noreplace_smp); - #define DPRINTK(type, fmt, args...) \ do { \ if (debug_alternative & DA_##type) \ @@ -1775,8 +1766,8 @@ static int cfi_rewrite_callers(s32 *start, s32 *end) #define FINEIBT_WARN(_f, _v) \ WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v) -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { int ret; @@ -2088,8 +2079,8 @@ bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type) #else /* !CONFIG_FINEIBT: */ -static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi, bool builtin) +static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi, bool builtin) { if (IS_ENABLED(CONFIG_CFI) && builtin) pr_info("CFI: Using standard kCFI\n"); @@ -2101,167 +2092,14 @@ static void poison_cfi(void *addr) { } #endif /* !CONFIG_FINEIBT */ -void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, - s32 *start_cfi, s32 *end_cfi) +void __init_or_module apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, + s32 *start_cfi, s32 *end_cfi) { return __apply_fineibt(start_retpoline, end_retpoline, start_cfi, end_cfi, /* .builtin = */ false); } -#ifdef CONFIG_SMP -static void alternatives_smp_lock(const s32 *start, const s32 *end, - u8 *text, u8 *text_end) -{ - const s32 *poff; - - for (poff = start; poff < end; poff++) { - u8 *ptr = (u8 *)poff + *poff; - - if (!*poff || ptr < text || ptr >= text_end) - continue; - /* turn DS segment override prefix into lock prefix */ - if (*ptr == 0x3e) - text_poke(ptr, ((unsigned char []){0xf0}), 1); - } -} - -static void alternatives_smp_unlock(const s32 *start, const s32 *end, - u8 *text, u8 *text_end) -{ - const s32 *poff; - - for (poff = start; poff < end; poff++) { - u8 *ptr = (u8 *)poff + *poff; - - if (!*poff || ptr < text || ptr >= text_end) - continue; - /* turn lock prefix into DS segment override prefix */ - if (*ptr == 0xf0) - text_poke(ptr, ((unsigned char []){0x3E}), 1); - } -} - -struct smp_alt_module { - /* what is this ??? */ - struct module *mod; - char *name; - - /* ptrs to lock prefixes */ - const s32 *locks; - const s32 *locks_end; - - /* .text segment, needed to avoid patching init code ;) */ - u8 *text; - u8 *text_end; - - struct list_head next; -}; -static LIST_HEAD(smp_alt_modules); -static bool uniproc_patched = false; /* protected by text_mutex */ - -void __init_or_module alternatives_smp_module_add(struct module *mod, - char *name, - void *locks, void *locks_end, - void *text, void *text_end) -{ - struct smp_alt_module *smp; - - mutex_lock(&text_mutex); - if (!uniproc_patched) - goto unlock; - - if (num_possible_cpus() == 1) - /* Don't bother remembering, we'll never have to undo it. */ - goto smp_unlock; - - smp = kzalloc_obj(*smp); - if (NULL == smp) - /* we'll run the (safe but slow) SMP code then ... */ - goto unlock; - - smp->mod = mod; - smp->name = name; - smp->locks = locks; - smp->locks_end = locks_end; - smp->text = text; - smp->text_end = text_end; - DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", - smp->locks, smp->locks_end, - smp->text, smp->text_end, smp->name); - - list_add_tail(&smp->next, &smp_alt_modules); -smp_unlock: - alternatives_smp_unlock(locks, locks_end, text, text_end); -unlock: - mutex_unlock(&text_mutex); -} - -void __init_or_module alternatives_smp_module_del(struct module *mod) -{ - struct smp_alt_module *item; - - mutex_lock(&text_mutex); - list_for_each_entry(item, &smp_alt_modules, next) { - if (mod != item->mod) - continue; - list_del(&item->next); - kfree(item); - break; - } - mutex_unlock(&text_mutex); -} - -void alternatives_enable_smp(void) -{ - struct smp_alt_module *mod; - - /* Why bother if there are no other CPUs? */ - BUG_ON(num_possible_cpus() == 1); - - mutex_lock(&text_mutex); - - if (uniproc_patched) { - pr_info("switching to SMP code\n"); - BUG_ON(num_online_cpus() != 1); - clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); - clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); - list_for_each_entry(mod, &smp_alt_modules, next) - alternatives_smp_lock(mod->locks, mod->locks_end, - mod->text, mod->text_end); - uniproc_patched = false; - } - mutex_unlock(&text_mutex); -} - -/* - * Return 1 if the address range is reserved for SMP-alternatives. - * Must hold text_mutex. - */ -int alternatives_text_reserved(void *start, void *end) -{ - struct smp_alt_module *mod; - const s32 *poff; - u8 *text_start = start; - u8 *text_end = end; - - lockdep_assert_held(&text_mutex); - - list_for_each_entry(mod, &smp_alt_modules, next) { - if (mod->text > text_end || mod->text_end < text_start) - continue; - for (poff = mod->locks; poff < mod->locks_end; poff++) { - const u8 *ptr = (const u8 *)poff + *poff; - - if (text_start <= ptr && text_end > ptr) - return 1; - } - } - - return 0; -} -#endif /* CONFIG_SMP */ - /* * Self-test for the INT3 based CALL emulation code. * @@ -2440,40 +2278,12 @@ void __init alternative_instructions(void) ibt_restore(ibt); -#ifdef CONFIG_SMP - /* Patch to UP if other cpus not imminent. */ - if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { - uniproc_patched = true; - alternatives_smp_module_add(NULL, "core kernel", - __smp_locks, __smp_locks_end, - _text, _etext); - } -#endif - restart_nmi(); alternatives_patched = 1; alt_reloc_selftest(); } -#ifdef CONFIG_SMP -/* - * With CONFIG_DEFERRED_STRUCT_PAGE_INIT enabled we can free_init_pages() only - * after the deferred initialization of the memory map is complete. - */ -static int __init free_smp_locks(void) -{ - if (!uniproc_patched || num_possible_cpus() == 1) { - free_init_pages("SMP alternatives", - (unsigned long)__smp_locks, - (unsigned long)__smp_locks_end); - } - - return 0; -} -arch_initcall(free_smp_locks); -#endif - /** * text_poke_early - Update instructions on a live kernel at boot time * @addr: address to modify diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index aa1e19979aa8..90025451ace2 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1191,11 +1191,11 @@ void disable_local_APIC(void) * restore the disabled state. */ if (enabled_via_apicbase) { - unsigned int l, h; + struct msr val; - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_ENABLE; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_ENABLE; + wrmsrq(MSR_IA32_APICBASE, val.q); } #endif } @@ -1960,7 +1960,8 @@ static bool __init detect_init_APIC(void) static bool __init apic_verify(unsigned long addr) { - u32 features, h, l; + struct msr val; + u32 features; /* * The APIC feature bit should now be enabled @@ -1975,9 +1976,9 @@ static bool __init apic_verify(unsigned long addr) /* The BIOS may have set up the APIC at some other address */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (l & MSR_IA32_APICBASE_ENABLE) - addr = l & MSR_IA32_APICBASE_BASE; + rdmsrq(MSR_IA32_APICBASE, val.q); + if (val.l & MSR_IA32_APICBASE_ENABLE) + addr = val.l & MSR_IA32_APICBASE_BASE; } register_lapic_address(addr); @@ -1987,7 +1988,7 @@ static bool __init apic_verify(unsigned long addr) bool __init apic_force_enable(unsigned long addr) { - u32 h, l; + struct msr val; if (apic_is_disabled) return false; @@ -1998,12 +1999,12 @@ bool __init apic_force_enable(unsigned long addr) * and AMD K7 (Model > 1) or later. */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - if (!(l & MSR_IA32_APICBASE_ENABLE)) { + rdmsrq(MSR_IA32_APICBASE, val.q); + if (!(val.l & MSR_IA32_APICBASE_ENABLE)) { pr_info("Local APIC disabled by BIOS -- reenabling.\n"); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | addr; - wrmsr(MSR_IA32_APICBASE, l, h); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | addr; + wrmsrq(MSR_IA32_APICBASE, val.q); enabled_via_apicbase = 1; } } @@ -2442,7 +2443,7 @@ static int lapic_suspend(void *data) static void lapic_resume(void *data) { - unsigned int l, h; + struct msr val; unsigned long flags; int maxlvt; @@ -2475,10 +2476,10 @@ static void lapic_resume(void *data) * SMP! We'll need to do this as part of the CPU restore! */ if (boot_cpu_data.x86 >= 6) { - rdmsr(MSR_IA32_APICBASE, l, h); - l &= ~MSR_IA32_APICBASE_BASE; - l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; - wrmsr(MSR_IA32_APICBASE, l, h); + rdmsrq(MSR_IA32_APICBASE, val.q); + val.l &= ~MSR_IA32_APICBASE_BASE; + val.l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; + wrmsrq(MSR_IA32_APICBASE, val.q); } } diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c index 5c5be2d58242..a60c8960bbfd 100644 --- a/arch/x86/kernel/apic/apic_numachip.c +++ b/arch/x86/kernel/apic/apic_numachip.c @@ -31,7 +31,7 @@ static u32 numachip1_get_apic_id(u32 x) unsigned long value; unsigned int id = (x >> 24) & 0xff; - if (static_cpu_has(X86_FEATURE_NODEID_MSR)) { + if (cpu_feature_enabled(X86_FEATURE_NODEID_MSR)) { rdmsrq(MSR_FAM10H_NODE_ID, value); id |= (value << 2) & 0xff00; } diff --git a/arch/x86/kernel/apic/local.h b/arch/x86/kernel/apic/local.h index 998efd442063..090dd71837aa 100644 --- a/arch/x86/kernel/apic/local.h +++ b/arch/x86/kernel/apic/local.h @@ -44,9 +44,7 @@ static inline unsigned int __prepare_ICR(unsigned int shortcut, int vector, #ifdef CONFIG_X86_X2APIC static inline void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest) { - unsigned long cfg = __prepare_ICR(0, vector, dest); - - native_x2apic_icr_write(cfg, apicid); + native_x2apic_icr_write(__prepare_ICR(0, vector, dest), apicid); } #endif diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c index 10f79026e8e3..090647cc5a78 100644 --- a/arch/x86/kernel/apic/x2apic_phys.c +++ b/arch/x86/kernel/apic/x2apic_phys.c @@ -85,11 +85,9 @@ static void static void __x2apic_send_IPI_shorthand(int vector, u32 which) { - unsigned long cfg = __prepare_ICR(which, vector, 0); - /* x2apic MSRs are special and need a special fence: */ weak_wrmsr_fence(); - native_x2apic_icr_write(cfg, 0); + native_x2apic_icr_write(__prepare_ICR(which, vector, 0), 0); } void x2apic_send_IPI_allbutself(int vector) diff --git a/arch/x86/kernel/apic/x2apic_savic.c b/arch/x86/kernel/apic/x2apic_savic.c index dbc5678bc3b6..4bc6d7e018a5 100644 --- a/arch/x86/kernel/apic/x2apic_savic.c +++ b/arch/x86/kernel/apic/x2apic_savic.c @@ -243,10 +243,7 @@ static void savic_write(u32 reg, u32 data) static void send_ipi(u32 dest, unsigned int vector, unsigned int dsh) { - unsigned int icr_low; - - icr_low = __prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL); - savic_icr_write(icr_low, dest); + savic_icr_write(__prepare_ICR(dsh, vector, APIC_DEST_PHYSICAL), dest); } static void savic_send_ipi(int cpu, int vector) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 487ac147e11f..169e373418bb 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -113,7 +113,7 @@ static void init_amd_k5(struct cpuinfo_x86 *c) static void init_amd_k6(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; int mbytes = get_num_physpages() >> (20-PAGE_SHIFT); if (c->x86_model < 6) { @@ -160,13 +160,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 508) mbytes = 508; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0x0000FFFF) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0x0000FFFF) == 0) { unsigned long flags; - l = (1<<0)|((mbytes/4)<<1); + val.l = (1 << 0) | ((mbytes / 4) << 1); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling old style K6 write allocation for %d Mb\n", mbytes); @@ -181,13 +181,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c) if (mbytes > 4092) mbytes = 4092; - rdmsr(MSR_K6_WHCR, l, h); - if ((l&0xFFFF0000) == 0) { + rdmsrq(MSR_K6_WHCR, val.q); + if ((val.l & 0xFFFF0000) == 0) { unsigned long flags; - l = ((mbytes>>2)<<22)|(1<<16); + val.l = ((mbytes >> 2) << 22) | (1 << 16); local_irq_save(flags); wbinvd(); - wrmsr(MSR_K6_WHCR, l, h); + wrmsrq(MSR_K6_WHCR, val.q); local_irq_restore(flags); pr_info("Enabling new style K6 write allocation for %d Mb\n", mbytes); @@ -207,7 +207,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c) static void init_amd_k7(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_32 - u32 l, h; + struct msr val; /* * Bit 15 of Athlon specific MSR 15, needs to be 0 @@ -228,11 +228,12 @@ static void init_amd_k7(struct cpuinfo_x86 *c) * As per AMD technical note 27212 0.2 */ if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) { - rdmsr(MSR_K7_CLK_CTL, l, h); - if ((l & 0xfff00000) != 0x20000000) { + rdmsrq(MSR_K7_CLK_CTL, val.q); + if ((val.l & 0xfff00000) != 0x20000000) { pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n", - l, ((l & 0x000fffff)|0x20000000)); - wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h); + val.l, ((val.l & 0x000fffff) | 0x20000000)); + val.l = (val.l & 0x000fffff) | 0x20000000; + wrmsrq(MSR_K7_CLK_CTL, val.q); } } @@ -614,12 +615,13 @@ clear_sev: static void early_init_amd(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; if (c->x86 >= 0xf) set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index d9af230c0512..ddf0db5326fa 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -192,8 +192,8 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. */ - if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && - !static_cpu_has(X86_FEATURE_VIRT_SSBD)) + if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) && + !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) return; /* @@ -201,7 +201,7 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) * virtual MSR value. If its not permanently enabled, evaluate * current's TIF_SSBD thread flag. */ - if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) + if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) hostval = SPEC_CTRL_SSBD; else hostval = ssbd_tif_to_spec_ctrl(ti->flags); @@ -2499,8 +2499,8 @@ static void __init ssb_apply_mitigation(void) * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may * use a completely different MSR and bit dependent on family. */ - if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && - !static_cpu_has(X86_FEATURE_AMD_SSBD)) { + if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) && + !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { x86_amd_ssb_disable(); } else { x86_spec_ctrl_base |= SPEC_CTRL_SSBD; diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 51a95b07831f..13ed16527905 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -654,7 +654,7 @@ static DEFINE_RAW_SPINLOCK(cache_disable_lock); */ static void maybe_flush_caches(void) { - if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) + if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) wbinvd(); } diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c index 681d2da49341..513fa1f640f9 100644 --- a/arch/x86/kernel/cpu/centaur.c +++ b/arch/x86/kernel/cpu/centaur.c @@ -22,7 +22,7 @@ static void init_c3(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Centaur Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -30,17 +30,17 @@ static void init_c3(struct cpuinfo_x86 *c) /* enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= ACE_FCR; /* enable ACE unit */ - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + /* enable ACE unit */ + wrmsrq(MSR_VIA_FCR, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_VIA_RNG, lo, hi); - lo |= RNG_ENABLE; /* enable RNG unit */ - wrmsr(MSR_VIA_RNG, lo, hi); + rdmsrq(MSR_VIA_RNG, msr); + /* enable RNG unit */ + wrmsrq(MSR_VIA_RNG, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } @@ -52,9 +52,8 @@ static void init_c3(struct cpuinfo_x86 *c) #ifdef CONFIG_X86_32 /* Cyrix III family needs CX8 & PGE explicitly enabled. */ if (c->x86_model >= 6 && c->x86_model <= 13) { - rdmsr(MSR_VIA_FCR, lo, hi); - lo |= (1<<1 | 1<<7); - wrmsr(MSR_VIA_FCR, lo, hi); + rdmsrq(MSR_VIA_FCR, msr); + wrmsrq(MSR_VIA_FCR, msr | (1 << 1 | 1 << 7)); set_cpu_cap(c, X86_FEATURE_CX8); } @@ -115,8 +114,9 @@ static void init_centaur(struct cpuinfo_x86 *c) char *name; u32 fcr_set = 0; u32 fcr_clr = 0; - u32 lo, hi, newlo; + u32 newlo; u32 aa, bb, cc, dd; + struct msr val; #endif early_init_centaur(c); init_intel_cacheinfo(c); @@ -169,15 +169,16 @@ static void init_centaur(struct cpuinfo_x86 *c) name = "??"; } - rdmsr(MSR_IDT_FCR1, lo, hi); - newlo = (lo|fcr_set) & (~fcr_clr); + rdmsrq(MSR_IDT_FCR1, val.q); + newlo = (val.l | fcr_set) & (~fcr_clr); - if (newlo != lo) { + if (newlo != val.l) { pr_info("Centaur FCR was 0x%X now 0x%X\n", - lo, newlo); - wrmsr(MSR_IDT_FCR1, newlo, hi); + val.l, newlo); + val.l = newlo; + wrmsrq(MSR_IDT_FCR1, val.q); } else { - pr_info("Centaur FCR is 0x%X\n", lo); + pr_info("Centaur FCR is 0x%X\n", val.l); } /* Emulate MTRRs using Centaur's MCR. */ set_cpu_cap(c, X86_FEATURE_CENTAUR_MCR); diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..e84ddf6bb10d 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -339,16 +339,16 @@ bool cpuid_feature(void) static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c) { - unsigned long lo, hi; + struct msr val; if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr) return; /* Disable processor serial number: */ - rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi); - lo |= 0x200000; - wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi); + rdmsrq(MSR_IA32_BBL_CR_CTL, val.q); + val.l |= 0x200000; + wrmsrq(MSR_IA32_BBL_CR_CTL, val.q); pr_notice("CPU serial number disabled.\n"); clear_cpu_cap(c, X86_FEATURE_PN); @@ -1251,9 +1251,6 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \ X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues) -#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \ - X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues) - #define VULNBL_AMD(family, blacklist) \ VULNBL(AMD, family, X86_MODEL_ANY, blacklist) @@ -1316,11 +1313,9 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), - VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE), - VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE), @@ -1388,7 +1383,21 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return true; /* Only consult the blacklist when there is no enumeration: */ - return cpu_matches(cpu_vuln_blacklist, RFDS); + if (!cpu_matches(cpu_vuln_blacklist, RFDS)) + return false; + + /* + * ADL and RPL are affected only if they have Atom CPUs. Hybrids have + * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not + * present. + */ + if ((boot_cpu_data.x86_model == 0x97 || + boot_cpu_data.x86_model == 0xB7) && + boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM && + !boot_cpu_has(X86_FEATURE_HYBRID_CPU)) + return false; + + return true; } static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) @@ -1651,7 +1660,7 @@ static inline bool parse_set_clear_cpuid(char *arg, bool set) int taint = 0; while (arg) { - bool found __maybe_unused = false; + bool found = false; unsigned int bit; opt = strsep(&arg, ","); @@ -2299,8 +2308,10 @@ static inline void idt_syscall_init(void) /* May not be marked __init: used by software suspend */ void syscall_init(void) { + struct msr val = { .h = (__USER32_CS << 16) | __KERNEL_CS }; + /* The default user and kernel segments */ - wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS); + wrmsrq(MSR_STAR, val.q); /* * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index d69757246bde..10a92927a515 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -25,7 +25,8 @@ enum vmx_feature_leafs { static void init_vmx_capabilities(struct cpuinfo_x86 *c) { - u32 supported, funcs, ept, vpid, ign, low, high; + struct msr val; + u32 supported, funcs, ept, vpid; BUILD_BUG_ON(NVMXINTS != NR_VMX_FEATURE_WORDS); @@ -39,25 +40,31 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c) * as they exist on any CPU that supports VMX, i.e. we want the WARN if * the RDMSR faults. */ - rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, ign, supported); + rdmsrq(MSR_IA32_VMX_PROCBASED_CTLS, val.q); + supported = val.h; c->vmx_capability[PRIMARY_CTLS] = supported; - rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &ign, &supported); + rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &val.q); + supported = val.h; c->vmx_capability[SECONDARY_CTLS] = supported; /* All 64 bits of tertiary controls MSR are allowed-1 settings. */ - rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &low, &high); - c->vmx_capability[TERTIARY_CTLS_LOW] = low; - c->vmx_capability[TERTIARY_CTLS_HIGH] = high; + rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &val.q); + c->vmx_capability[TERTIARY_CTLS_LOW] = val.l; + c->vmx_capability[TERTIARY_CTLS_HIGH] = val.h; - rdmsr(MSR_IA32_VMX_PINBASED_CTLS, ign, supported); - rdmsr_safe(MSR_IA32_VMX_VMFUNC, &ign, &funcs); + rdmsrq(MSR_IA32_VMX_PINBASED_CTLS, val.q); + supported = val.h; + rdmsrq_safe(MSR_IA32_VMX_VMFUNC, &val.q); + funcs = val.h; /* * Except for EPT+VPID, which enumerates support for both in a single * MSR, low for EPT, high for VPID. */ - rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, &ept, &vpid); + rdmsrq_safe(MSR_IA32_VMX_EPT_VPID_CAP, &val.q); + ept = val.l; + vpid = val.h; /* Pin, EPT, VPID and VM-Func are merged into a single word. */ WARN_ON_ONCE(supported >> 16); diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index 3e8891a9caf2..ec51c2b9a257 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -125,11 +125,12 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c) static void early_init_hygon(struct cpuinfo_x86 *c) { - u32 dummy; + u64 val; set_cpu_cap(c, X86_FEATURE_K8); - rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); + rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val); + c->microcode = (u32)val; /* * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index abb3984336eb..4297ceb2cb24 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -199,6 +199,41 @@ void intel_unlock_cpuid_leafs(struct cpuinfo_x86 *c) c->cpuid_level = cpuid_eax(0); } +/* + * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 + * structures are populated. + */ +static u32 intel_cpuid_vfm(void) +{ + u32 eax = cpuid_eax(1); + u32 fam = x86_family(eax); + u32 model = x86_model(eax); + + return IFM(fam, model); +} + +u32 intel_get_platform_id(void) +{ + unsigned int val[2]; + + if (x86_hypervisor_present) + return 0; + + /* + * This can be called early. Use CPUID directly instead of + * relying on cpuinfo_x86 which may not be fully initialized. + * The PII does not have MSR_IA32_PLATFORM_ID. Everything + * before _it_ has no microcode (for Linux at least). + */ + if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) + return 0; + + /* get processor flags from MSR 0x17 */ + native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); + + return (val[1] >> 18) & 7; +} + static void early_init_intel(struct cpuinfo_x86 *c) { u64 misc_enable; @@ -542,12 +577,12 @@ static void init_intel(struct cpuinfo_x86 *c) set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC); if (boot_cpu_has(X86_FEATURE_DS)) { - unsigned int l1, l2; + u64 l; - rdmsr(MSR_IA32_MISC_ENABLE, l1, l2); - if (!(l1 & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) + rdmsrq(MSR_IA32_MISC_ENABLE, l); + if (!(l & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_BTS); - if (!(l1 & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) + if (!(l & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL)) set_cpu_cap(c, X86_FEATURE_PEBS); } diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c index 4604802692da..7ab077f0cc66 100644 --- a/arch/x86/kernel/cpu/match.c +++ b/arch/x86/kernel/cpu/match.c @@ -6,34 +6,6 @@ #include <linux/slab.h> /** - * x86_match_vendor_cpu_type - helper function to match the hardware defined - * cpu-type for a single entry in the x86_cpu_id - * table. Note, this function does not match the - * generic cpu-types TOPO_CPU_TYPE_EFFICIENCY and - * TOPO_CPU_TYPE_PERFORMANCE. - * @c: Pointer to the cpuinfo_x86 structure of the CPU to match. - * @m: Pointer to the x86_cpu_id entry to match against. - * - * Return: true if the cpu-type matches, false otherwise. - */ -static bool x86_match_vendor_cpu_type(struct cpuinfo_x86 *c, const struct x86_cpu_id *m) -{ - if (m->type == X86_CPU_TYPE_ANY) - return true; - - /* Hybrid CPUs are special, they are assumed to match all cpu-types */ - if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) - return true; - - if (c->x86_vendor == X86_VENDOR_INTEL) - return m->type == c->topo.intel_type; - if (c->x86_vendor == X86_VENDOR_AMD) - return m->type == c->topo.amd_type; - - return false; -} - -/** * x86_match_cpu - match current CPU against an array of x86_cpu_ids * @match: Pointer to array of x86_cpu_ids. Last entry terminated with * {}. @@ -81,7 +53,7 @@ const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id *match) continue; if (m->feature != X86_FEATURE_ANY && !cpu_has(c, m->feature)) continue; - if (!x86_match_vendor_cpu_type(c, m)) + if (m->type != X86_CPU_TYPE_ANY && c->topo.cpu_type != m->type) continue; return m; } diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 36e0df4d1342..f916fb4c5d13 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -280,11 +280,11 @@ static void smca_configure(unsigned int bank, unsigned int cpu) u8 *bank_counts = this_cpu_ptr(smca_bank_counts); const struct smca_hwid *s_hwid; unsigned int i, hwid_mcatype; - u32 high, low; + struct msr val; u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank); /* Set appropriate bits in MCA_CONFIG */ - if (!rdmsr_safe(smca_config, &low, &high)) { + if (!rdmsrq_safe(smca_config, &val.q)) { /* * OS is required to set the MCAX bit to acknowledge that it is * now using the new MSR ranges and new registers under each @@ -294,7 +294,7 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * * MCA_CONFIG[MCAX] is bit 32 (0 in the high portion of the MSR.) */ - high |= BIT(0); + val.h |= BIT(0); /* * SMCA sets the Deferred Error Interrupt type per bank. @@ -307,9 +307,9 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * APIC based interrupt. First, check that no interrupt has been * set. */ - if ((low & BIT(5)) && !((high >> 5) & 0x3) && data->dfr_intr_en) { + if ((val.l & BIT(5)) && !((val.h >> 5) & 0x3) && data->dfr_intr_en) { __set_bit(bank, data->dfr_intr_banks); - high |= BIT(5); + val.h |= BIT(5); } /* @@ -324,33 +324,33 @@ static void smca_configure(unsigned int bank, unsigned int cpu) * The OS should set this to inform the platform that the OS is ready * to handle the MCA Thresholding interrupt. */ - if ((low & BIT(10)) && data->thr_intr_en) { + if ((val.l & BIT(10)) && data->thr_intr_en) { __set_bit(bank, data->thr_intr_banks); - high |= BIT(8); + val.h |= BIT(8); } - this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(low & BIT(8)); + this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(val.l & BIT(8)); - if (low & MCI_CONFIG_PADDRV) + if (val.l & MCI_CONFIG_PADDRV) this_cpu_ptr(smca_banks)[bank].paddrv = 1; - wrmsr(smca_config, low, high); + wrmsrq(smca_config, val.q); } - if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) { + if (rdmsrq_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &val.q)) { pr_warn("Failed to read MCA_IPID for bank %d\n", bank); return; } - hwid_mcatype = HWID_MCATYPE(high & MCI_IPID_HWID, - (high & MCI_IPID_MCATYPE) >> 16); + hwid_mcatype = HWID_MCATYPE(val.h & MCI_IPID_HWID, + (val.h & MCI_IPID_MCATYPE) >> 16); for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) { s_hwid = &smca_hwid_mcatypes[i]; if (hwid_mcatype == s_hwid->hwid_mcatype) { this_cpu_ptr(smca_banks)[bank].hwid = s_hwid; - this_cpu_ptr(smca_banks)[bank].id = low; + this_cpu_ptr(smca_banks)[bank].id = val.l; this_cpu_ptr(smca_banks)[bank].sysfs_id = bank_counts[s_hwid->bank_type]++; break; } @@ -432,50 +432,50 @@ static bool lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi) static void threshold_restart_block(void *_tr) { struct thresh_restart *tr = _tr; - u32 hi, lo; + struct msr val; /* sysfs write might race against an offline operation */ if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off) return; - rdmsr(tr->b->address, lo, hi); + rdmsrq(tr->b->address, val.q); /* * Reset error count and overflow bit. * This is done during init or after handling an interrupt. */ - if (hi & MASK_OVERFLOW_HI || tr->set_lvt_off) { - hi &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); - hi |= THRESHOLD_MAX - tr->b->threshold_limit; + if (val.h & MASK_OVERFLOW_HI || tr->set_lvt_off) { + val.h &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI); + val.h |= THRESHOLD_MAX - tr->b->threshold_limit; } else if (tr->old_limit) { /* change limit w/o reset */ - int new_count = (hi & THRESHOLD_MAX) + + int new_count = (val.h & THRESHOLD_MAX) + (tr->old_limit - tr->b->threshold_limit); - hi = (hi & ~MASK_ERR_COUNT_HI) | + val.h = (val.h & ~MASK_ERR_COUNT_HI) | (new_count & THRESHOLD_MAX); } /* clear IntType */ - hi &= ~MASK_INT_TYPE_HI; + val.h &= ~MASK_INT_TYPE_HI; if (!tr->b->interrupt_capable) goto done; if (tr->set_lvt_off) { - if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) { + if (lvt_off_valid(tr->b, tr->lvt_off, val.l, val.h)) { /* set new lvt offset */ - hi &= ~MASK_LVTOFF_HI; - hi |= tr->lvt_off << 20; + val.h &= ~MASK_LVTOFF_HI; + val.h |= tr->lvt_off << 20; } } if (tr->b->interrupt_enable) - hi |= INT_TYPE_APIC; + val.h |= INT_TYPE_APIC; done: - hi |= MASK_COUNT_EN_HI; - wrmsr(tr->b->address, lo, hi); + val.h |= MASK_COUNT_EN_HI; + wrmsrq(tr->b->address, val.q); } static void threshold_restart_bank(unsigned int bank, bool intr_en) @@ -726,7 +726,8 @@ static void smca_enable_interrupt_vectors(void) void mce_amd_feature_init(struct cpuinfo_x86 *c) { unsigned int bank, block, cpu = smp_processor_id(); - u32 low = 0, high = 0, address = 0; + struct msr val = { .q = 0 }; + u32 address = 0; int offset = -1; amd_apply_cpu_quirks(c); @@ -746,21 +747,21 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c) disable_err_thresholding(c, bank); for (block = 0; block < NR_BLOCKS; ++block) { - address = get_block_address(address, low, high, bank, block, cpu); + address = get_block_address(address, val.l, val.h, bank, block, cpu); if (!address) break; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) break; - if (!(high & MASK_VALID_HI)) + if (!(val.h & MASK_VALID_HI)) continue; - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) continue; - offset = prepare_threshold_block(bank, block, address, offset, high); + offset = prepare_threshold_block(bank, block, address, offset, val.h); } } } @@ -1083,24 +1084,24 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb u32 address) { struct threshold_block *b = NULL; - u32 low, high; + struct msr val; int err; if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS)) return 0; - if (rdmsr_safe(address, &low, &high)) + if (rdmsrq_safe(address, &val.q)) return 0; - if (!(high & MASK_VALID_HI)) { + if (!(val.h & MASK_VALID_HI)) { if (block) goto recurse; else return 0; } - if (!(high & MASK_CNTP_HI) || - (high & MASK_LOCKED_HI)) + if (!(val.h & MASK_CNTP_HI) || + (val.h & MASK_LOCKED_HI)) goto recurse; b = kzalloc_obj(struct threshold_block); @@ -1112,7 +1113,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb b->cpu = cpu; b->address = address; b->interrupt_enable = 0; - b->interrupt_capable = lvt_interrupt_supported(bank, high); + b->interrupt_capable = lvt_interrupt_supported(bank, val.h); b->threshold_limit = get_thr_limit(); if (b->interrupt_capable) { @@ -1124,13 +1125,13 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb list_add(&b->miscj, &tb->miscj); - mce_threshold_block_init(b, (high & MASK_LVTOFF_HI) >> 20); + mce_threshold_block_init(b, (val.h & MASK_LVTOFF_HI) >> 20); err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(cpu, bank, b)); if (err) goto out_free; recurse: - address = get_block_address(address, low, high, bank, ++block, cpu); + address = get_block_address(address, val.l, val.h, bank, ++block, cpu); if (!address) return 0; diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 9bba1e2f03af..017aaf57ba47 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1866,7 +1866,7 @@ static void __mcheck_cpu_init_generic(void) rdmsrq(MSR_IA32_MCG_CAP, cap); if (cap & MCG_CTL_P) - wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); + wrmsrq(MSR_IA32_MCG_CTL, ~0ULL); } static void __mcheck_cpu_init_prepare_banks(void) diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c index 2272ad53fc33..3c2b6cc918b1 100644 --- a/arch/x86/kernel/cpu/mce/p5.c +++ b/arch/x86/kernel/cpu/mce/p5.c @@ -23,16 +23,16 @@ int mce_p5_enabled __read_mostly; /* Machine check handler for Pentium class Intel CPUs: */ noinstr void pentium_machine_check(struct pt_regs *regs) { - u32 loaddr, hi, lotype; + u64 addr, type; instrumentation_begin(); - rdmsr(MSR_IA32_P5_MC_ADDR, loaddr, hi); - rdmsr(MSR_IA32_P5_MC_TYPE, lotype, hi); + rdmsrq(MSR_IA32_P5_MC_ADDR, addr); + rdmsrq(MSR_IA32_P5_MC_TYPE, type); pr_emerg("CPU#%d: Machine Check Exception: 0x%8X (type 0x%8X).\n", - smp_processor_id(), loaddr, lotype); + smp_processor_id(), (u32)addr, (u32)type); - if (lotype & (1<<5)) { + if (type & (1<<5)) { pr_emerg("CPU#%d: Possible thermal failure (CPU on fire ?).\n", smp_processor_id()); } @@ -44,7 +44,7 @@ noinstr void pentium_machine_check(struct pt_regs *regs) /* Set up machine check reporting for processors with Intel style MCE: */ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) { - u32 l, h; + u64 __maybe_unused q; /* Default P5 to off as its often misconnected: */ if (!mce_p5_enabled) @@ -55,8 +55,8 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) return; /* Read registers before enabling: */ - rdmsr(MSR_IA32_P5_MC_ADDR, l, h); - rdmsr(MSR_IA32_P5_MC_TYPE, l, h); + rdmsrq(MSR_IA32_P5_MC_ADDR, q); + rdmsrq(MSR_IA32_P5_MC_TYPE, q); pr_info("Intel old style machine check architecture supported.\n"); /* Enable MCE: */ diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c index 6c99f2941909..7040243533d9 100644 --- a/arch/x86/kernel/cpu/mce/winchip.c +++ b/arch/x86/kernel/cpu/mce/winchip.c @@ -28,12 +28,12 @@ noinstr void winchip_machine_check(struct pt_regs *regs) /* Set up machine check reporting on the Winchip C6 series */ void winchip_mcheck_init(struct cpuinfo_x86 *c) { - u32 lo, hi; + struct msr val; - rdmsr(MSR_IDT_FCR1, lo, hi); - lo |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ - lo &= ~(1<<4); /* Enable MCE */ - wrmsr(MSR_IDT_FCR1, lo, hi); + rdmsrq(MSR_IDT_FCR1, val.q); + val.l |= (1<<2); /* Enable EIERRINT (int 18 MCE) */ + val.l &= ~(1<<4); /* Enable MCE */ + wrmsrq(MSR_IDT_FCR1, val.q); cr4_set_bits(X86_CR4_MCE); diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 531dfb771c8b..6cdc410e7547 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -136,8 +136,7 @@ struct cont_desc { * Microcode patch container file is prepended to the initrd in cpio * format. See Documentation/arch/x86/microcode.rst */ -static const char -ucode_path[] __maybe_unused = "kernel/x86/microcode/AuthenticAMD.bin"; +static const char ucode_path[] = "kernel/x86/microcode/AuthenticAMD.bin"; /* * This is CPUID(1).EAX on the BSP. It is used in two ways: diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index f4a444e6114d..1142183c950c 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -121,42 +121,6 @@ static inline unsigned int exttable_size(struct extended_sigtable *et) return et->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE; } - -/* - * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86 - * structures are populated. - */ -static u32 intel_cpuid_vfm(void) -{ - u32 eax = cpuid_eax(1); - u32 fam = x86_family(eax); - u32 model = x86_model(eax); - - return IFM(fam, model); -} - -u32 intel_get_platform_id(void) -{ - unsigned int val[2]; - - if (x86_hypervisor_present) - return 0; - - /* - * This can be called early. Use CPUID directly instead of - * relying on cpuinfo_x86 which may not be fully initialized. - * The PII does not have MSR_IA32_PLATFORM_ID. Everything - * before _it_ has no microcode (for Linux at least). - */ - if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH) - return 0; - - /* get processor flags from MSR 0x17 */ - native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]); - - return (val[1] >> 18) & 7; -} - void intel_collect_cpu_info(struct cpu_signature *sig) { sig->sig = cpuid_eax(1); diff --git a/arch/x86/kernel/cpu/mtrr/amd.c b/arch/x86/kernel/cpu/mtrr/amd.c index ef3e8e42b782..a73715d6f05c 100644 --- a/arch/x86/kernel/cpu/mtrr/amd.c +++ b/arch/x86/kernel/cpu/mtrr/amd.c @@ -10,20 +10,23 @@ static void amd_get_mtrr(unsigned int reg, unsigned long *base, unsigned long *size, mtrr_type *type) { - unsigned long low, high; + unsigned long val; + struct msr msr; - rdmsr(MSR_K6_UWCCR, low, high); + rdmsrq(MSR_K6_UWCCR, msr.q); /* Upper dword is region 1, lower is region 0 */ if (reg == 1) - low = high; + val = msr.h; + else + val = msr.l; /* The base masks off on the right alignment */ - *base = (low & 0xFFFE0000) >> PAGE_SHIFT; + *base = (val & 0xFFFE0000) >> PAGE_SHIFT; *type = 0; - if (low & 1) + if (val & 1) *type = MTRR_TYPE_UNCACHABLE; - if (low & 2) + if (val & 2) *type = MTRR_TYPE_WRCOMB; - if (!(low & 3)) { + if (!(val & 3)) { *size = 0; return; } @@ -42,8 +45,8 @@ amd_get_mtrr(unsigned int reg, unsigned long *base, * +1 000 0000 0000 0100 * *128K ... */ - low = (~low) & 0x1FFFC; - *size = (low + 4) << (15 - PAGE_SHIFT); + val = (~val) & 0x1FFFC; + *size = (val + 4) << (15 - PAGE_SHIFT); } /** @@ -59,12 +62,16 @@ amd_get_mtrr(unsigned int reg, unsigned long *base, static void amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { + struct msr msr; u32 regs[2]; /* * Low is MTRR0, High MTRR 1 */ - rdmsr(MSR_K6_UWCCR, regs[0], regs[1]); + rdmsrq(MSR_K6_UWCCR, msr.q); + regs[0] = msr.l; + regs[1] = msr.h; + /* * Blank to disable */ @@ -89,7 +96,9 @@ amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type * disable local interrupts, write back the cache, set the mtrr */ wbinvd(); - wrmsr(MSR_K6_UWCCR, regs[0], regs[1]); + msr.l = regs[0]; + msr.h = regs[1]; + wrmsrq(MSR_K6_UWCCR, msr.q); } static int diff --git a/arch/x86/kernel/cpu/mtrr/centaur.c b/arch/x86/kernel/cpu/mtrr/centaur.c index 6f6c3ae92943..e32cca1caf59 100644 --- a/arch/x86/kernel/cpu/mtrr/centaur.c +++ b/arch/x86/kernel/cpu/mtrr/centaur.c @@ -65,26 +65,26 @@ static void centaur_set_mcr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { - unsigned long low, high; + struct msr val; if (size == 0) { /* Disable */ - high = low = 0; + val.q = 0; } else { - high = base << PAGE_SHIFT; + val.h = base << PAGE_SHIFT; if (centaur_mcr_type == 0) { /* Only support write-combining... */ - low = -size << PAGE_SHIFT | 0x1f; + val.l = -size << PAGE_SHIFT | 0x1f; } else { if (type == MTRR_TYPE_UNCACHABLE) - low = -size << PAGE_SHIFT | 0x02; /* NC */ + val.l = -size << PAGE_SHIFT | 0x02; /* NC */ else - low = -size << PAGE_SHIFT | 0x09; /* WWO, WC */ + val.l = -size << PAGE_SHIFT | 0x09; /* WWO, WC */ } } - centaur_mcr[reg].high = high; - centaur_mcr[reg].low = low; - wrmsr(MSR_IDT_MCR0 + reg, low, high); + centaur_mcr[reg].high = val.h; + centaur_mcr[reg].low = val.l; + wrmsrq(MSR_IDT_MCR0 + reg, val.q); } static int diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c index e3eee9ae4141..cd1a6dec4064 100644 --- a/arch/x86/kernel/cpu/mtrr/cleanup.c +++ b/arch/x86/kernel/cpu/mtrr/cleanup.c @@ -658,8 +658,8 @@ static int __init mtrr_search_optimal_index(void) int __init mtrr_cleanup(void) { unsigned long x_remove_base, x_remove_size; - unsigned long base, size, def, dummy; - u64 chunk_size, gran_size; + u64 def, chunk_size, gran_size; + unsigned long base, size; mtrr_type type; int index_good; int i; @@ -670,7 +670,7 @@ int __init mtrr_cleanup(void) if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1) return 0; - rdmsr(MSR_MTRRdefType, def, dummy); + rdmsrq(MSR_MTRRdefType, def); def &= 0xff; if (def != MTRR_TYPE_UNCACHABLE) return 0; @@ -806,7 +806,7 @@ early_param("disable_mtrr_trim", disable_mtrr_trim_setup); int __init amd_special_default_mtrr(void) { - u32 l, h; + u64 q; if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) @@ -814,13 +814,13 @@ int __init amd_special_default_mtrr(void) if (boot_cpu_data.x86 < 0xf) return 0; /* In case some hypervisor doesn't pass SYSCFG through: */ - if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0) + if (rdmsrq_safe(MSR_AMD64_SYSCFG, &q) < 0) return 0; /* * Memory between 4GB and top of mem is forced WB by this magic bit. * Reserved before K8RevF, but should be zero there. */ - if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == + if ((q & (Tom2Enabled | Tom2ForceMemTypeWB)) == (Tom2Enabled | Tom2ForceMemTypeWB)) return 1; return 0; @@ -854,9 +854,9 @@ real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) */ int __init mtrr_trim_uncached_memory(unsigned long end_pfn) { - unsigned long i, base, size, highest_pfn = 0, def, dummy; + unsigned long i, base, size, highest_pfn = 0; mtrr_type type; - u64 total_trim_size; + u64 def, total_trim_size; /* extra one for all 0 */ int num[MTRR_NUM_TYPES + 1]; @@ -870,7 +870,7 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn) if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim) return 0; - rdmsr(MSR_MTRRdefType, def, dummy); + rdmsrq(MSR_MTRRdefType, def); def &= MTRR_DEF_TYPE_TYPE; if (def != MTRR_TYPE_UNCACHABLE) return 0; diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index 3a8317060732..67cf69f24b00 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c @@ -103,7 +103,7 @@ u32 phys_hi_rsvd; */ static inline void k8_check_syscfg_dram_mod_en(void) { - u32 lo, hi; + struct msr val; if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0x0f))) @@ -112,13 +112,13 @@ static inline void k8_check_syscfg_dram_mod_en(void) if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - rdmsr(MSR_AMD64_SYSCFG, lo, hi); - if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) { + rdmsrq(MSR_AMD64_SYSCFG, val.q); + if (val.l & K8_MTRRFIXRANGE_DRAM_MODIFY) { pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]" " not cleared by BIOS, clearing this bit\n", smp_processor_id()); - lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; - mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi); + val.l &= ~K8_MTRRFIXRANGE_DRAM_MODIFY; + mtrr_wrmsr(MSR_AMD64_SYSCFG, val.l, val.h); } } @@ -557,8 +557,14 @@ u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform) static void get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr) { - rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); - rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); + struct msr val; + + rdmsrq(MTRRphysBase_MSR(index), val.q); + vr->base_lo = val.l; + vr->base_hi = val.h; + rdmsrq(MTRRphysMask_MSR(index), val.q); + vr->mask_lo = val.l; + vr->mask_hi = val.h; } /* Fill the MSR pair relating to a var range */ @@ -577,17 +583,17 @@ void fill_mtrr_var_range(unsigned int index, static void get_fixed_ranges(mtrr_type *frs) { - unsigned int *p = (unsigned int *)frs; + u64 *p = (u64 *)frs; int i; k8_check_syscfg_dram_mod_en(); - rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]); + rdmsrq(MSR_MTRRfix64K_00000, p[0]); for (i = 0; i < 2; i++) - rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]); + rdmsrq(MSR_MTRRfix16K_80000 + i, p[1 + i]); for (i = 0; i < 8; i++) - rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]); + rdmsrq(MSR_MTRRfix4K_C0000 + i, p[3 + i]); } void mtrr_save_fixed_ranges(void *info) @@ -689,31 +695,26 @@ static void __init print_mtrr_state(void) bool __init get_mtrr_state(void) { struct mtrr_var_range *vrs; - unsigned lo, dummy; unsigned int i; + u64 q; vrs = mtrr_state.var_ranges; - rdmsr(MSR_MTRRcap, lo, dummy); - mtrr_state.have_fixed = lo & MTRR_CAP_FIX; + rdmsrq(MSR_MTRRcap, q); + mtrr_state.have_fixed = q & MTRR_CAP_FIX; for (i = 0; i < num_var_ranges; i++) get_mtrr_var_range(i, &vrs[i]); if (mtrr_state.have_fixed) get_fixed_ranges(mtrr_state.fixed_ranges); - rdmsr(MSR_MTRRdefType, lo, dummy); - mtrr_state.def_type = lo & MTRR_DEF_TYPE_TYPE; - mtrr_state.enabled = (lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT; + rdmsrq(MSR_MTRRdefType, q); + mtrr_state.def_type = q & MTRR_DEF_TYPE_TYPE; + mtrr_state.enabled = (q & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT; if (amd_special_default_mtrr()) { - unsigned low, high; - /* TOP_MEM2 */ - rdmsr(MSR_K8_TOP_MEM2, low, high); - mtrr_tom2 = high; - mtrr_tom2 <<= 32; - mtrr_tom2 |= low; + rdmsrq(MSR_K8_TOP_MEM2, mtrr_tom2); mtrr_tom2 &= 0xffffff800000ULL; } @@ -750,7 +751,9 @@ void __init mtrr_state_warn(void) */ void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) { - if (wrmsr_safe(msr, a, b) < 0) { + struct msr val = { .l = a, .h = b }; + + if (wrmsrq_safe(msr, val.q) < 0) { pr_err("MTRR: CPU %u: Writing MSR %x to %x:%x failed\n", smp_processor_id(), msr, a, b); } @@ -765,11 +768,11 @@ void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b) */ static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords) { - unsigned lo, hi; + struct msr val; - rdmsr(msr, lo, hi); + rdmsrq(msr, val.q); - if (lo != msrwords[0] || hi != msrwords[1]) { + if (val.l != msrwords[0] || val.h != msrwords[1]) { mtrr_wrmsr(msr, msrwords[0], msrwords[1]); *changed = true; } @@ -806,9 +809,8 @@ generic_get_free_region(unsigned long base, unsigned long size, int replace_reg) static void generic_get_mtrr(unsigned int reg, unsigned long *base, unsigned long *size, mtrr_type *type) { - u32 mask_lo, mask_hi, base_lo, base_hi; + u64 tmp, mask, base_msr; unsigned int hi; - u64 tmp, mask; /* * get_mtrr doesn't need to update mtrr_state, also it could be called @@ -816,9 +818,9 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, */ get_cpu(); - rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi); + rdmsrq(MTRRphysMask_MSR(reg), mask); - if (!(mask_lo & MTRR_PHYSMASK_V)) { + if (!(mask & MTRR_PHYSMASK_V)) { /* Invalid (i.e. free) range */ *base = 0; *size = 0; @@ -826,10 +828,10 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, goto out_put_cpu; } - rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi); + rdmsrq(MTRRphysBase_MSR(reg), base_msr); /* Work out the shifted address mask: */ - tmp = (u64)mask_hi << 32 | (mask_lo & PAGE_MASK); + tmp = mask & PAGE_MASK; mask = (u64)phys_hi_rsvd << 32 | tmp; /* Expand tmp with high bits to all 1s: */ @@ -849,8 +851,8 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base, * contiguous range: */ *size = -mask >> PAGE_SHIFT; - *base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT; - *type = base_lo & MTRR_PHYSBASE_TYPE; + *base = base_msr >> PAGE_SHIFT; + *type = base_msr & MTRR_PHYSBASE_TYPE; out_put_cpu: put_cpu(); @@ -884,21 +886,21 @@ static int set_fixed_ranges(mtrr_type *frs) */ static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr) { - unsigned int lo, hi; bool changed = false; + struct msr val; - rdmsr(MTRRphysBase_MSR(index), lo, hi); - if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (lo & ~MTRR_PHYSBASE_RSVD) - || (vr->base_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { + rdmsrq(MTRRphysBase_MSR(index), val.q); + if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (val.l & ~MTRR_PHYSBASE_RSVD) + || (vr->base_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) { mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi); changed = true; } - rdmsr(MTRRphysMask_MSR(index), lo, hi); + rdmsrq(MTRRphysMask_MSR(index), val.q); - if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (lo & ~MTRR_PHYSMASK_RSVD) - || (vr->mask_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) { + if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (val.l & ~MTRR_PHYSMASK_RSVD) + || (vr->mask_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) { mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi); changed = true; } @@ -947,8 +949,12 @@ static unsigned long set_mtrr_state(void) void mtrr_disable(void) { + struct msr val; + /* Save MTRR state */ - rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi); + rdmsrq(MSR_MTRRdefType, val.q); + deftype_lo = val.l; + deftype_hi = val.h; /* Disable MTRRs, and set the default type to uncached */ mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & MTRR_DEF_TYPE_DISABLE, deftype_hi); @@ -1057,8 +1063,9 @@ int generic_validate_add_page(unsigned long base, unsigned long size, static int generic_have_wrcomb(void) { - unsigned long config, dummy; - rdmsr(MSR_MTRRcap, config, dummy); + u64 config; + + rdmsrq(MSR_MTRRcap, config); return config & MTRR_CAP_WC; } diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 4b3d492afe17..468c53b20acf 100644 --- a/arch/x86/kernel/cpu/mtrr/mtrr.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c @@ -547,7 +547,7 @@ void __init mtrr_bp_init(void) { bool generic_mtrrs = cpu_feature_enabled(X86_FEATURE_MTRR); const char *why = "(not available)"; - unsigned long config, dummy; + unsigned long config; phys_hi_rsvd = GENMASK(31, boot_cpu_data.x86_phys_bits - 32); @@ -571,7 +571,7 @@ void __init mtrr_bp_init(void) if (mtrr_enabled()) { /* Get the number of variable MTRR ranges. */ if (mtrr_if == &generic_mtrr_ops) - rdmsr(MSR_MTRRcap, config, dummy); + rdmsrq(MSR_MTRRcap, config); else config = mtrr_if->var_regs; num_var_ranges = config & MTRR_CAP_VCNT; diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 9c01d2562b7a..55214d6fdc49 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -247,7 +247,11 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); hw_res->num_closid = edx + 1; - r->membw.max_bw = 1 << eax; + if (BITS_PER_TYPE(r->membw.max_bw) <= eax) { + pr_warn("Unable to support hardware's maximum bandwidth\n"); + return false; + } + r->membw.max_bw = BIT(eax); /* AMD does not use delay */ r->membw.delay_linear = false; @@ -515,14 +519,12 @@ static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r) return; } - list_add_tail_rcu(&d->hdr.list, add_pos); - err = resctrl_online_ctrl_domain(r, d); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); ctrl_domain_free(hw_dom); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct list_head *add_pos) @@ -556,14 +558,12 @@ static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct return; } - list_add_tail_rcu(&d->hdr.list, add_pos); - err = resctrl_online_mon_domain(r, &d->hdr); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); l3_mon_domain_free(hw_dom); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) @@ -642,9 +642,9 @@ static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) d = container_of(hdr, struct rdt_ctrl_domain, hdr); hw_dom = resctrl_to_arch_ctrl_dom(d); - resctrl_offline_ctrl_domain(r, d); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_ctrl_domain(r, d); /* * rdt_ctrl_domain "d" is going to be freed below, so clear @@ -689,9 +689,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) d = container_of(hdr, struct rdt_l3_mon_domain, hdr); hw_dom = resctrl_to_arch_mon_dom(d); - resctrl_offline_mon_domain(r, hdr); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_mon_domain(r, hdr); l3_mon_domain_free(hw_dom); break; } @@ -702,9 +702,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) return; pkgd = container_of(hdr, struct rdt_perf_pkg_mon_domain, hdr); - resctrl_offline_mon_domain(r, hdr); list_del_rcu(&hdr->list); synchronize_rcu(); + resctrl_offline_mon_domain(r, hdr); kfree(pkgd); break; } @@ -725,13 +725,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) static void clear_closid_rmid(int cpu) { struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state); + struct msr val = { + .l = RESCTRL_RESERVED_RMID, + .h = RESCTRL_RESERVED_CLOSID + }; state->default_closid = RESCTRL_RESERVED_CLOSID; state->default_rmid = RESCTRL_RESERVED_RMID; state->cur_closid = RESCTRL_RESERVED_CLOSID; state->cur_rmid = RESCTRL_RESERVED_RMID; - wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID, - RESCTRL_RESERVED_CLOSID); + wrmsrq(MSR_IA32_PQR_ASSOC, val.q); } static int resctrl_arch_online_cpu(unsigned int cpu) diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index b20e705606b8..e74f1ed54b86 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -53,7 +53,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { hw_dom = resctrl_to_arch_ctrl_dom(d); msr_param.res = NULL; for (t = 0; t < CDP_NUM_TYPES; t++) { @@ -115,7 +115,7 @@ static void _resctrl_sdciae_enable(struct rdt_resource *r, bool enable) lockdep_assert_cpus_held(); /* Update MSR_IA32_L3_QOS_EXT_CFG MSR on all the CPUs in all domains */ - list_for_each_entry(d, &r->ctrl_domains, hdr.list) + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_sdciae_set_one_amd, &enable, 1); } diff --git a/arch/x86/kernel/cpu/resctrl/intel_aet.c b/arch/x86/kernel/cpu/resctrl/intel_aet.c index 89b8b619d5d5..c22c3cf5167d 100644 --- a/arch/x86/kernel/cpu/resctrl/intel_aet.c +++ b/arch/x86/kernel/cpu/resctrl/intel_aet.c @@ -398,12 +398,11 @@ void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r, d->hdr.type = RESCTRL_MON_DOMAIN; d->hdr.rid = RDT_RESOURCE_PERF_PKG; cpumask_set_cpu(cpu, &d->hdr.cpu_mask); - list_add_tail_rcu(&d->hdr.list, add_pos); err = resctrl_online_mon_domain(r, &d->hdr); if (err) { - list_del_rcu(&d->hdr.list); - synchronize_rcu(); kfree(d); + return; } + list_add_tail_rcu(&d->hdr.list, add_pos); } diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 569894d6e5c8..3838e0a13d36 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -136,7 +136,7 @@ static int logical_rmid_to_physical_rmid(int cpu, int lrmid) static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) { - u64 msr_val; + struct msr msr_val = { .l = eventid, .h = prmid }; /* * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured @@ -146,15 +146,15 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62) * are error bits. */ - wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } @@ -283,7 +283,10 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, static int __cntr_id_read(u32 cntr_id, u64 *val) { - u64 msr_val; + struct msr msr_val = { + .l = ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, + .h = cntr_id + }; /* * QM_EVTSEL Register definition: @@ -306,15 +309,15 @@ static int __cntr_id_read(u32 cntr_id, u64 *val) * ID is set in the QM_EVTSEL.RMID field. The RMID_VAL_UNAVAIL bit * is set if the counter data is unavailable. */ - wrmsr(MSR_IA32_QM_EVTSEL, ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, cntr_id); - rdmsrq(MSR_IA32_QM_CTR, msr_val); + wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q); + rdmsrq(MSR_IA32_QM_CTR, msr_val.q); - if (msr_val & RMID_VAL_ERROR) + if (msr_val.q & RMID_VAL_ERROR) return -EIO; - if (msr_val & RMID_VAL_UNAVAIL) + if (msr_val.q & RMID_VAL_UNAVAIL) return -EINVAL; - *val = msr_val; + *val = msr_val.q; return 0; } @@ -510,7 +513,7 @@ static void _resctrl_abmc_enable(struct rdt_resource *r, bool enable) lockdep_assert_cpus_held(); - list_for_each_entry(d, &r->mon_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_abmc_set_one_amd, &enable, 1); resctrl_arch_reset_rmid_all(r, d); diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index de580eca3363..d7caab0409b6 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -241,16 +241,16 @@ int resctrl_arch_pseudo_lock_fn(void *_plr) int resctrl_arch_measure_cycles_lat_fn(void *_plr) { struct pseudo_lock_region *plr = _plr; - u32 saved_low, saved_high; unsigned long i; u64 start, end; void *mem_r; + u64 saved; local_irq_disable(); /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); mem_r = READ_ONCE(plr->kmem); /* @@ -267,7 +267,7 @@ int resctrl_arch_measure_cycles_lat_fn(void *_plr) end = rdtsc_ordered(); trace_pseudo_lock_mem_latency((u32)(end - start)); } - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); @@ -312,11 +312,11 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; struct perf_event *miss_event, *hit_event; int hit_pmcnum, miss_pmcnum; - u32 saved_low, saved_high; unsigned int line_size; unsigned int size; unsigned long i; void *mem_r; + u64 saved; u64 tmp; miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, @@ -346,7 +346,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, /* * Disable hardware prefetchers. */ - rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + rdmsrq(MSR_MISC_FEATURE_CONTROL, saved); wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits); /* Initialize rest of local variables */ @@ -405,7 +405,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, */ rmb(); /* Re-enable hardware prefetchers */ - wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); + wrmsrq(MSR_MISC_FEATURE_CONTROL, saved); local_irq_enable(); out_hit: perf_event_release_kernel(hit_event); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 885026468440..5ffa39fa86fa 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -151,7 +151,7 @@ static int set_cache_qos_cfg(int level, bool enable) return -ENOMEM; r_l = &rdt_resources_all[level].r_resctrl; - list_for_each_entry(d, &r_l->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r_l->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { if (r_l->cache.arch_has_per_cpu_cfg) /* Pick all the CPUs in the domain instance */ for_each_cpu(cpu, &d->hdr.cpu_mask) @@ -249,7 +249,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) * CBMs in all ctrl_domains to the maximum mask value. Pick one CPU * from each domain to update the MSRs below. */ - list_for_each_entry(d, &r->ctrl_domains, hdr.list) { + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { hw_dom = resctrl_to_arch_ctrl_dom(d); for (i = 0; i < hw_res->num_closid; i++) diff --git a/arch/x86/kernel/cpu/topology.h b/arch/x86/kernel/cpu/topology.h index 37326297f80c..74e02bacd854 100644 --- a/arch/x86/kernel/cpu/topology.h +++ b/arch/x86/kernel/cpu/topology.h @@ -22,6 +22,7 @@ void topology_set_dom(struct topo_scan *tscan, enum x86_topology_domains dom, bool cpu_parse_topology_ext(struct topo_scan *tscan); void cpu_parse_topology_amd(struct topo_scan *tscan); void cpu_topology_fixup_amd(struct topo_scan *tscan); +enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c); static inline u32 topo_shift_apicid(u32 apicid, enum x86_topology_domains dom) { diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c index da080d732e10..c5a6944df86a 100644 --- a/arch/x86/kernel/cpu/topology_amd.c +++ b/arch/x86/kernel/cpu/topology_amd.c @@ -177,8 +177,10 @@ static void topoext_fixup(struct topo_scan *tscan) static void parse_topology_amd(struct topo_scan *tscan) { - if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) - tscan->c->topo.cpu_type = cpuid_ebx(0x80000026); + if (cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES)) { + tscan->c->topo.hw_cpu_type = cpuid_ebx(0x80000026); + tscan->c->topo.cpu_type = get_topology_cpu_type(tscan->c); + } /* * Try to get SMT, CORE, TILE, and DIE shifts from extended diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c index cf7513416b70..6845e3c63fbb 100644 --- a/arch/x86/kernel/cpu/topology_common.c +++ b/arch/x86/kernel/cpu/topology_common.c @@ -42,8 +42,9 @@ enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c) } if (c->x86_vendor == X86_VENDOR_AMD) { switch (c->topo.amd_type) { - case 0: return TOPO_CPU_TYPE_PERFORMANCE; - case 1: return TOPO_CPU_TYPE_EFFICIENCY; + case AMD_CPU_TYPE_PERFORMANCE: return TOPO_CPU_TYPE_PERFORMANCE; + case AMD_CPU_TYPE_EFFICIENCY: return TOPO_CPU_TYPE_EFFICIENCY; + case AMD_CPU_TYPE_LOW_POWER: return TOPO_CPU_TYPE_LOW_POWER; } } @@ -57,12 +58,14 @@ const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c) return "performance"; case TOPO_CPU_TYPE_EFFICIENCY: return "efficiency"; + case TOPO_CPU_TYPE_LOW_POWER: + return "low_power"; default: return "unknown"; } } -static unsigned int __maybe_unused parse_num_cores_legacy(struct cpuinfo_x86 *c) +static unsigned int parse_num_cores_legacy(struct cpuinfo_x86 *c) { struct { u32 cache_type : 5, @@ -168,8 +171,12 @@ static void parse_topology(struct topo_scan *tscan, bool early) case X86_VENDOR_INTEL: if (!IS_ENABLED(CONFIG_CPU_SUP_INTEL) || !cpu_parse_topology_ext(tscan)) parse_legacy(tscan); - if (c->cpuid_level >= 0x1a) - c->topo.cpu_type = cpuid_eax(0x1a); + + if (c->cpuid_level >= 0x1a) { + c->topo.hw_cpu_type = cpuid_eax(0x1a); + c->topo.cpu_type = get_topology_cpu_type(c); + } + break; } } diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c index 1fdcd69c625c..3e04b5d01bf8 100644 --- a/arch/x86/kernel/cpu/transmeta.c +++ b/arch/x86/kernel/cpu/transmeta.c @@ -24,7 +24,8 @@ static void early_init_transmeta(struct cpuinfo_x86 *c) static void init_transmeta(struct cpuinfo_x86 *c) { - unsigned int cap_mask, uk, max, dummy; + u64 msr; + unsigned int max, dummy; unsigned int cms_rev1, cms_rev2; unsigned int cpu_rev, cpu_freq = 0, cpu_flags, new_cpu_rev; char cpu_info[65]; @@ -86,10 +87,11 @@ static void init_transmeta(struct cpuinfo_x86 *c) } /* Unhide possibly hidden capability flags */ - rdmsr(0x80860004, cap_mask, uk); - wrmsr(0x80860004, ~0, uk); + rdmsrq(0x80860004, msr); + wrmsrq(0x80860004, msr | ~0U); + cpuid_refresh_leaf(c, 0x1); c->x86_capability[CPUID_1_EDX] = cpuid_edx(0x00000001); - wrmsr(0x80860004, cap_mask, uk); + wrmsrq(0x80860004, msr); /* All Transmeta CPUs have a constant TSC */ set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c index 761aef5590ac..fe504fd43c77 100644 --- a/arch/x86/kernel/cpu/zhaoxin.c +++ b/arch/x86/kernel/cpu/zhaoxin.c @@ -21,7 +21,7 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) { - u32 lo, hi; + u64 msr; /* Test for Extended Feature Flags presence */ if (cpuid_eax(0xC0000000) >= 0xC0000001) { @@ -29,19 +29,17 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c) /* Enable ACE unit, if present and disabled */ if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable ACE unit */ - lo |= ACE_FCR; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | ACE_FCR); pr_info("CPU: Enabled ACE h/w crypto\n"); } /* Enable RNG unit, if present and disabled */ if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) { - rdmsr(MSR_ZHAOXIN_FCR57, lo, hi); + rdmsrq(MSR_ZHAOXIN_FCR57, msr); /* Enable RNG unit */ - lo |= RNG_ENABLE; - wrmsr(MSR_ZHAOXIN_FCR57, lo, hi); + wrmsrq(MSR_ZHAOXIN_FCR57, msr | RNG_ENABLE); pr_info("CPU: Enabled h/w RNG\n"); } diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 584fb9913be4..d1aeecd57f5e 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -294,15 +294,15 @@ void fpu_free_guest_fpstate(struct fpu_guest *gfpu) } EXPORT_SYMBOL_FOR_KVM(fpu_free_guest_fpstate); -/* - * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable - * @guest_fpu: Pointer to the guest FPU container - * @xfeatures: Features requested by guest CPUID - * - * Enable all dynamic xfeatures according to guest perm and requested CPUID. - * - * Return: 0 on success, error code otherwise - */ +/** + * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable + * @guest_fpu: Pointer to the guest FPU container + * @xfeatures: Features requested by guest CPUID + * + * Enable all dynamic xfeatures according to guest perm and requested CPUID. + * + * Return: 0 on success, error code otherwise + */ int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) { lockdep_assert_preemption_enabled(); @@ -874,7 +874,7 @@ void fpu_flush_thread(void) */ void switch_fpu_return(void) { - if (!static_cpu_has(X86_FEATURE_FPU)) + if (!cpu_feature_enabled(X86_FEATURE_FPU)) return; fpregs_restore_userregs(); diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index 20b638c507ca..33e1284bf3e4 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -187,7 +187,7 @@ bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) || IS_ENABLED(CONFIG_IA32_EMULATION)); - if (!static_cpu_has(X86_FEATURE_FPU)) { + if (!cpu_feature_enabled(X86_FEATURE_FPU)) { struct user_i387_ia32_struct fp; fpregs_soft_get(current, NULL, (struct membuf){.p = &fp, diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index c1fac3a9fecc..4e5f8c1736ec 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -762,9 +762,6 @@ int arch_prepare_kprobe(struct kprobe *p) { int ret; - if (alternatives_text_reserved(p->addr, p->addr)) - return -EINVAL; - if (!can_probe((unsigned long)p->addr)) return -EILSEQ; diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 6f826a00eca2..3f8fea52619f 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -77,7 +77,7 @@ found: static void synthesize_clac(kprobe_opcode_t *addr) { /* - * Can't be static_cpu_has() due to how objtool treats this feature bit. + * Can't be cpu_feature_enabled() due to how objtool treats this feature bit. * This isn't a fast path anyway. */ if (!boot_cpu_has(X86_FEATURE_SMAP)) @@ -213,7 +213,6 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) } /* Check whether the address range is reserved */ if (ftrace_text_reserved(src, src + len - 1) || - alternatives_text_reserved(src, src + len - 1) || jump_label_text_reserved(src, src + len - 1) || static_call_text_reserved(src, src + len - 1)) return -EBUSY; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index dcef84da304b..5b554e480c0e 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -663,8 +663,10 @@ static void kvm_flush_tlb_multi(const struct cpumask *cpumask, u8 state; int cpu; struct kvm_steal_time *src; - struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); + struct cpumask *flushmask; + guard(preempt)(); + flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); cpumask_copy(flushmask, cpumask); /* * We have to call flush only on online vCPUs. And diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index 40c5bf97dd5c..4fa7a86adece 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -61,7 +61,7 @@ void load_mm_ldt(struct mm_struct *mm) */ if (unlikely(ldt)) { - if (static_cpu_has(X86_FEATURE_PTI)) { + if (cpu_feature_enabled(X86_FEATURE_PTI)) { if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) { /* * Whoops -- either the new LDT isn't mapped diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index b5b4de4f08e6..33080ad889b7 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -227,7 +227,7 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) { - const Elf_Shdr *s, *alt = NULL, *locks = NULL, + const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL, *calls = NULL, *cfi = NULL; @@ -236,8 +236,6 @@ int module_finalize(const Elf_Ehdr *hdr, for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { if (!strcmp(".altinstructions", secstrings + s->sh_name)) alt = s; - if (!strcmp(".smp_locks", secstrings + s->sh_name)) - locks = s; if (!strcmp(".orc_unwind", secstrings + s->sh_name)) orc = s; if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) @@ -300,14 +298,6 @@ int module_finalize(const Elf_Ehdr *hdr, void *iseg = (void *)ibt_endbr->sh_addr; apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size); } - if (locks) { - void *lseg = (void *)locks->sh_addr; - void *text = me->mem[MOD_TEXT].base; - void *text_end = text + me->mem[MOD_TEXT].size; - alternatives_smp_module_add(me, me->name, - lseg, lseg + locks->sh_size, - text, text_end); - } if (orc && orc_ip) unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size, @@ -318,6 +308,5 @@ int module_finalize(const Elf_Ehdr *hdr, void module_arch_cleanup(struct module *mod) { - alternatives_smp_module_del(mod); its_free_mod(mod); } diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 34bdb752f892..eb988df22521 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -13,6 +13,9 @@ * and then read/write in chunks of 8 bytes. A larger size means multiple * reads or writes of the same register. * + * Writing the same register multiple times can be useful for MSRs with + * I/O-like semantics, e.g. a virtual MSR that accepts logging information. + * * This driver uses /dev/cpu/%d/msr where %d is the minor number, and on * an SMP box will direct the access to CPU %d. */ diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 85435044e33c..346c438ac880 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -579,7 +579,7 @@ static __always_inline void amd_set_core_ssb_state(unsigned long tifn) struct ssb_state *st = this_cpu_ptr(&ssb_state); u64 msr = x86_amd_ls_cfg_base; - if (!static_cpu_has(X86_FEATURE_ZEN)) { + if (!cpu_feature_enabled(X86_FEATURE_ZEN)) { msr |= ssbd_tif_to_amd_ls_cfg(tifn); wrmsrq(MSR_AMD64_LS_CFG, msr); return; @@ -646,14 +646,14 @@ static __always_inline void __speculation_ctrl_update(unsigned long tifp, lockdep_assert_irqs_disabled(); /* Handle change of TIF_SSBD depending on the mitigation method. */ - if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) { + if (cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) { if (tif_diff & _TIF_SSBD) amd_set_ssb_virt_state(tifn); - } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) { + } else if (cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD)) { if (tif_diff & _TIF_SSBD) amd_set_core_ssb_state(tifn); - } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || - static_cpu_has(X86_FEATURE_AMD_SSBD)) { + } else if (cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) || + cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) { updmsr |= !!(tif_diff & _TIF_SSBD); msr |= ssbd_tif_to_spec_ctrl(tifn); } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index d44afbe005bb..2bce7b3f97ed 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -277,7 +277,7 @@ static __always_inline void save_fsgs(struct task_struct *task) { savesegment(fs, task->thread.fsindex); savesegment(gs, task->thread.gsindex); - if (static_cpu_has(X86_FEATURE_FSGSBASE)) { + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { /* * If FSGSBASE is enabled, we can't make any useful guesses * about the base, and user code expects us to save the current @@ -391,7 +391,7 @@ static __always_inline void x86_pkru_load(struct thread_struct *prev, static __always_inline void x86_fsgsbase_load(struct thread_struct *prev, struct thread_struct *next) { - if (static_cpu_has(X86_FEATURE_FSGSBASE)) { + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { /* Update the FS and GS selectors if they could have changed. */ if (unlikely(prev->fsindex || next->fsindex)) loadseg(FS, next->fsindex); @@ -533,7 +533,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip, { WARN_ON_ONCE(regs != current_pt_regs()); - if (static_cpu_has(X86_BUG_NULL_SEG)) { + if (cpu_feature_enabled(X86_BUG_NULL_SEG)) { /* Loading zero below won't clear the base. */ loadsegment(fs, __USER_DS); load_gs_index(__USER_DS); diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal_32.c index e55cf19e68fe..8e87ab586437 100644 --- a/arch/x86/kernel/signal_32.c +++ b/arch/x86/kernel/signal_32.c @@ -357,7 +357,7 @@ int ia32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) unsafe_put_user(ptr_to_compat(&frame->uc), &frame->puc, Efault); /* Create the ucontext. */ - if (static_cpu_has(X86_FEATURE_XSAVE)) + if (cpu_feature_enabled(X86_FEATURE_XSAVE)) unsafe_put_user(UC_FP_XSTATE, &frame->uc.uc_flags, Efault); else unsafe_put_user(0, &frame->uc.uc_flags, Efault); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index cb999feb66b0..ba01a9e919b7 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -991,9 +991,6 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; - /* Just in case we booted with a single CPU. */ - alternatives_enable_smp(); - per_cpu(current_task, cpu) = idle; cpu_init_stack_canary(cpu, idle); diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 52e1f3f0b361..4061430ac74c 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -82,7 +82,7 @@ static __init void x86_late_time_init(void) x86_init.irqs.intr_mode_init(); tsc_init(); - if (static_cpu_has(X86_FEATURE_WAITPKG)) + if (cpu_feature_enabled(X86_FEATURE_WAITPKG)) use_tpause_delay(); } diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index ce10ae4b298b..723347e2cf7f 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1221,11 +1221,11 @@ static void __init check_system_tsc_reliable(void) if (is_geode_lx()) { /* RTSC counts during suspend */ #define RTSC_SUSP 0x100 - unsigned long res_low, res_high; + u64 res; - rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high); + rdmsrq_safe(MSR_GEODE_BUSCONT_CONF0, &res); /* Geode_LX - the OLPC CPU has a very reliable TSC */ - if (res_low & RTSC_SUSP) + if (res & RTSC_SUSP) tsc_clocksource_reliable = 1; } #endif diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 48e6cc1cb017..d74743c8d2a4 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -165,7 +165,8 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = { */ unsigned long cpu_khz_from_msr(void) { - u32 lo, hi, ratio, freq, tscref; + u32 ratio, freq, tscref; + struct msr val; const struct freq_desc *freq_desc; const struct x86_cpu_id *id; const struct muldiv *md; @@ -178,16 +179,16 @@ unsigned long cpu_khz_from_msr(void) freq_desc = (struct freq_desc *)id->driver_data; if (freq_desc->use_msr_plat) { - rdmsr(MSR_PLATFORM_INFO, lo, hi); - ratio = (lo >> 8) & 0xff; + rdmsrq(MSR_PLATFORM_INFO, val.q); + ratio = (val.l >> 8) & 0xff; } else { - rdmsr(MSR_IA32_PERF_STATUS, lo, hi); - ratio = (hi >> 8) & 0x1f; + rdmsrq(MSR_IA32_PERF_STATUS, val.q); + ratio = (val.h >> 8) & 0x1f; } /* Get FSB FREQ ID */ - rdmsr(MSR_FSB_FREQ, lo, hi); - index = lo & freq_desc->mask; + rdmsrq(MSR_FSB_FREQ, val.q); + index = val.l & freq_desc->mask; md = &freq_desc->muldiv[index]; /* diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d3..65a2de82ecd2 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call <uprobe-trampoline> + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ unlock: mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 74e336d7f9dd..2438b89a4620 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -65,9 +65,15 @@ const_cpu_current_top_of_stack = cpu_current_top_of_stack; __end_rodata_hpage_align = .; \ __end_rodata_aligned = .; +#if defined(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION) || defined(CONFIG_MITIGATION_SRSO) #define ALIGN_ENTRY_TEXT_BEGIN . = ALIGN(PMD_SIZE); #define ALIGN_ENTRY_TEXT_END . = ALIGN(PMD_SIZE); #else +#define ALIGN_ENTRY_TEXT_BEGIN +#define ALIGN_ENTRY_TEXT_END +#endif + +#else #define X86_ALIGN_RODATA_BEGIN #define X86_ALIGN_RODATA_END \ @@ -221,7 +227,7 @@ SECTIONS * references to such code must be patched out by alternatives, normally * by using X86_FEATURE_ALWAYS CPU feature bit. * - * See static_cpu_has() for an example. + * See cpu_feature_enabled() for an example. */ .altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) { *(.altinstr_aux) @@ -346,18 +352,6 @@ SECTIONS __init_end = .; } - /* - * smp_locks might be freed after init - * start/end must be page aligned - */ - . = ALIGN(PAGE_SIZE); - .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) { - __smp_locks = .; - *(.smp_locks) - . = ALIGN(PAGE_SIZE); - __smp_locks_end = .; - } - #ifdef CONFIG_X86_64 .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { NOSAVE_DATA |