diff options
Diffstat (limited to 'arch')
92 files changed, 672 insertions, 494 deletions
diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig index f930396d9dae..870e5291b7db 100644 --- a/arch/arc/configs/axs101_defconfig +++ b/arch/arc/configs/axs101_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_DRM=m CONFIG_DRM_I2C_ADV7511=m diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig index 6b779dee5ea0..d45e4d335998 100644 --- a/arch/arc/configs/axs103_defconfig +++ b/arch/arc/configs/axs103_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index a89b50d5369d..f986c0205f13 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_DRM=m CONFIG_DRM_I2C_ADV7511=m diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index 865fbc19ef03..6e396a9ddb8b 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -61,7 +61,6 @@ CONFIG_SERIAL_8250_DW=y CONFIG_I2C=y # CONFIG_I2C_COMPAT is not set CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_GPIO_SYSFS=y # CONFIG_HWMON is not set # CONFIG_USB_SUPPORT is not set diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index b2f2c59279a6..2d99dffed0ce 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -22,6 +22,7 @@ #include <linux/irqdomain.h> #include <linux/export.h> #include <linux/of_fdt.h> +#include <linux/string.h> #include <asm/mach_desc.h> #include <asm/setup.h> @@ -43,9 +44,10 @@ static int __init arc_get_cpu_map(const char *name, struct cpumask *cpumask) { unsigned long dt_root = of_get_flat_dt_root(); const char *buf; + int len; - buf = of_get_flat_dt_prop(dt_root, name, NULL); - if (!buf) + buf = of_get_flat_dt_prop(dt_root, name, &len); + if (!buf || !memchr(buf, '\0', len)) return -EINVAL; if (cpulist_parse(buf, cpumask)) diff --git a/arch/arm/mach-ixp4xx/Kconfig b/arch/arm/mach-ixp4xx/Kconfig index cb46802f5ce5..7f812020e082 100644 --- a/arch/arm/mach-ixp4xx/Kconfig +++ b/arch/arm/mach-ixp4xx/Kconfig @@ -14,6 +14,5 @@ menuconfig ARCH_IXP4XX select IXP4XX_TIMER select USB_EHCI_BIG_ENDIAN_DESC select USB_EHCI_BIG_ENDIAN_MMIO - select USE_OF help Support for Intel's IXP4XX (XScale) family of processors. diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig index 66e26990e2c8..c478fb8a6f78 100644 --- a/arch/arm/mach-pxa/Kconfig +++ b/arch/arm/mach-pxa/Kconfig @@ -22,7 +22,6 @@ config MACH_PXA25X_DT select PINCTRL select POWER_SUPPLY select PXA25x - select USE_OF help Include support for Marvell PXA25x based platforms using the device tree. Needn't select any other machine while @@ -33,7 +32,6 @@ config MACH_PXA27X_DT select PINCTRL select POWER_SUPPLY select PXA27x - select USE_OF help Include support for Marvell PXA27x based platforms using the device tree. Needn't select any other machine while @@ -47,7 +45,6 @@ config MACH_PXA3XX_DT select PINCTRL select POWER_SUPPLY select PXA3xx - select USE_OF help Include support for Marvell PXA3xx based platforms using the device tree. Needn't select any other machine while diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index 8e0c51e496e2..820670dd6042 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -5355,7 +5355,7 @@ #size-cells = <0>; cpu0_0: cpu@0 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00000>; @@ -5374,7 +5374,7 @@ }; cpu0_1: cpu@100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00100>; @@ -5393,7 +5393,7 @@ }; cpu0_2: cpu@200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00200>; @@ -5412,7 +5412,7 @@ }; cpu0_3: cpu@300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00300>; @@ -5431,7 +5431,7 @@ }; cpu1_0: cpu@10000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10000>; @@ -5450,7 +5450,7 @@ }; cpu1_1: cpu@10100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10100>; @@ -5469,7 +5469,7 @@ }; cpu1_2: cpu@10200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10200>; @@ -5488,7 +5488,7 @@ }; cpu1_3: cpu@10300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10300>; @@ -5507,7 +5507,7 @@ }; cpu2_0: cpu@20000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20000>; @@ -5526,7 +5526,7 @@ }; cpu2_1: cpu@20100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20100>; @@ -5545,7 +5545,7 @@ }; cpu2_2: cpu@20200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20200>; @@ -5564,7 +5564,7 @@ }; cpu2_3: cpu@20300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20300>; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 2d8e7e37830f..2d2cb1a3d95c 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -3208,7 +3208,7 @@ }; gpcdma: dma-controller@8400000 { - compatible = "nvidia,tegra264-gpcdma", "nvidia,tegra186-gpcdma"; + compatible = "nvidia,tegra264-gpcdma"; reg = <0x0 0x08400000 0x0 0x210000>; interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>, @@ -4070,7 +4070,7 @@ d-cache-sets = <256>; }; - cpu1: cpu@1 { + cpu1: cpu@10000 { compatible = "arm,neoverse-v3ae"; device_type = "cpu"; reg = <0x10000>; diff --git a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts index d2b3fc08954a..0ab303863155 100644 --- a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts +++ b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts @@ -107,6 +107,47 @@ reg = <0x0 0x8c400000 0x0 0x02000000>; no-map; }; + + /* DRAM controller inline ECC areas */ + ecc@10cccc0000 { + reg = <0x10 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@12cccc0000 { + reg = <0x12 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@14cccc0000 { + reg = <0x14 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@16cccc0000 { + reg = <0x16 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@18cccc0000 { + reg = <0x18 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@1a66660000 { + reg = <0x1a 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1c66660000 { + reg = <0x1c 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1e66660000 { + reg = <0x1e 0x66660000 0x0 0x999a0000>; + no-map; + }; }; }; diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index cdf3e8422ea1..012d711034d1 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,14 @@ struct s1_walk_result { bool failed; }; +static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) +{ + wr->fst = fst; + wr->ptw = s1ptw; + wr->s2 = s1ptw; + wr->failed = true; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index 6297631532e5..767f35ea62b3 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,17 +2,11 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H -#include <linux/cpumask.h> - struct arch_tlbflush_unmap_batch { -#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * Track CPUs that need SME DVMSync on completion of this batch. - * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to - * record cpumask for sending IPI + * For arm64, HW can do TLB shootdown, so we don't need to record a + * cpumask for sending IPIs. */ - cpumask_var_t cpumask; -#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index d52ac8c17190..e0e84332f51b 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -82,6 +82,8 @@ static inline unsigned long get_trans_granule(void) #ifdef CONFIG_ARM64_ERRATUM_4193714 +extern cpumask_t sme_active_cpus; + void sme_do_dvmsync(const struct cpumask *mask); static inline void sme_dvmsync(struct mm_struct *mm) @@ -92,42 +94,12 @@ static inline void sme_dvmsync(struct mm_struct *mm) sme_do_dvmsync(mm_cpumask(mm)); } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) +static inline void sme_dvmsync_batch(void) { if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) return; - /* - * Order the mm_cpumask() read after the hardware DVMSync. - */ - dsb(ish); - if (cpumask_empty(mm_cpumask(mm))) - return; - - /* - * Allocate the batch cpumask on first use. Fall back to an immediate - * IPI for this mm in case of failure. - */ - if (!cpumask_available(batch->cpumask) && - !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { - sme_do_dvmsync(mm_cpumask(mm)); - return; - } - - cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); -} - -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) -{ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - return; - - if (!cpumask_available(batch->cpumask)) - return; - - sme_do_dvmsync(batch->cpumask); - cpumask_clear(batch->cpumask); + sme_do_dvmsync(&sme_active_cpus); } #else @@ -135,11 +107,7 @@ static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) static inline void sme_dvmsync(struct mm_struct *mm) { } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) -{ -} -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void sme_dvmsync_batch(void) { } @@ -285,11 +253,11 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm) sme_dvmsync(mm); } -static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void __tlbi_sync_s1ish_batch(void) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); - sme_dvmsync_batch(batch); + sme_dvmsync_batch(); } static inline void __tlbi_sync_s1ish_kernel(void) @@ -434,7 +402,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish_batch(batch); + __tlbi_sync_s1ish_batch(); } /* @@ -722,7 +690,6 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); - sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 5891f92c2035..681aa2bbc399 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -448,12 +448,14 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 apci_id, return *pcpu; } + set_cpu_present(*pcpu, true); return 0; } EXPORT_SYMBOL(acpi_map_cpu); int acpi_unmap_cpu(int cpu) { + set_cpu_present(cpu, false); return 0; } EXPORT_SYMBOL(acpi_unmap_cpu); diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 25dc5afe9ba0..e7f1682a3059 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1355,6 +1355,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) * SME/CME erratum handling. */ static cpumask_t sme_dvmsync_cpus; +cpumask_t sme_active_cpus; /* * These helpers are only called from non-preemptible contexts, so @@ -1368,13 +1369,15 @@ void sme_set_active(void) return; cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + cpumask_set_cpu(cpu, &sme_active_cpus); /* * A subsequent (post ERET) SME access may use a stale address * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for - * the completion of cpumask_set_cpu() above as it appears in program - * order before the SME access. The post-TLBI+DSB read of mm_cpumask() - * will lead to the IPI being issued. + * the completion of the cpumask_set_cpu() operations above as they + * appear in program order before the SME access. The post-TLBI+DSB + * read of mm_cpumask() or sme_active_cpus will lead to the IPI being + * issued. * * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 */ @@ -1392,6 +1395,7 @@ void sme_clear_active(void) * completed on entering EL1. */ cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); + cpumask_clear_cpu(cpu, &sme_active_cpus); } static void sme_dvmsync_ipi(void *unused) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 033643cd4e5e..581f80e9b9b7 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -341,41 +341,8 @@ void flush_thread(void) flush_gcs(); } -#ifdef CONFIG_ARM64_ERRATUM_4193714 - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ - /* - * Clear the inherited cpumask with memset() to cover both cases where - * cpumask_var_t is a pointer or an array. It will be allocated lazily - * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. - */ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - memset(&dst->tlb_ubc.arch.cpumask, 0, - sizeof(dst->tlb_ubc.arch.cpumask)); -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - free_cpumask_var(tsk->tlb_ubc.arch.cpumask); -} - -#else - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ -} - -#endif /* CONFIG_ARM64_ERRATUM_4193714 */ - void arch_release_task_struct(struct task_struct *tsk) { - arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -391,8 +358,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; - arch_dup_tlbbatch_mask(dst); - /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d46022f72075..cdcdd160e5b6 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -535,23 +535,13 @@ void arch_unregister_cpu(int cpu) { acpi_handle acpi_handle = acpi_get_processor_handle(cpu); struct cpu *c = &per_cpu(cpu_devices, cpu); - acpi_status status; unsigned long long sta; - - if (!acpi_handle) { - pr_err_once("Removing a CPU without associated ACPI handle\n"); - return; - } + acpi_status status; status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return; - - /* For now do not allow anything that looks like physical CPU HP */ - if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) { + if (!ACPI_FAILURE(status) && + cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) pr_err_once("Changing CPU present bit is not supported\n"); - return; - } unregister_cpu(c); } @@ -566,6 +556,11 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) } EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc); +static bool acpi_cpu_is_present(int cpu) +{ + return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED; +} + /* * acpi_map_gic_cpu_interface - parse processor MADT entry * @@ -670,6 +665,10 @@ static void __init acpi_parse_and_init_cpus(void) early_map_cpu_to_node(i, acpi_numa_get_nid(i)); } #else +static bool acpi_cpu_is_present(int cpu) +{ + return false; +} #define acpi_parse_and_init_cpus(...) do { } while (0) #endif @@ -814,7 +813,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (err) continue; - set_cpu_present(cpu, true); + if (acpi_disabled || acpi_cpu_is_present(cpu)) + set_cpu_present(cpu, true); numa_store_cpu_info(cpu); } } diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index b8ded434c63f..640f2dc00a8b 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,14 +11,6 @@ #include <asm/kvm_mmu.h> #include <asm/lsui.h> -static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) -{ - wr->fst = fst; - wr->ptw = s1ptw; - wr->s2 = s1ptw; - wr->failed = true; -} - #define S1_MMU_DISABLED (-127) static int get_ia_size(struct s1_walk_info *wi) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index e688bc5139c1..3c82f392845d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2746,17 +2746,33 @@ static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr) (spsr & PSR_MODE32_BIT) || (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t || mode == PSR_MODE_EL1h))) { + u64 mask; + /* - * The guest is playing with our nerves. Preserve EL, SP, - * masks, flags from the existing PSTATE, and set IL. - * The HW will then generate an Illegal State Exception - * immediately after ERET. + * On an illegal exception return, the flags and masks are + * taken from the SPSR while PSTATE.{EL,SP,nRW} and, if + * FEAT_GCS, PSTATE.EXLOCK are unchanged (R_VWJHB). Set IL + * so the HW generates an Illegal State Exception right + * after ERET. */ - spsr = *vcpu_cpsr(vcpu); + mask = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | + PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + + if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, IMP)) + mask |= PSR_PAN_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, NMI, IMP)) + mask |= ALLINT_ALLINT; + /* FEAT_SPE_EXC and FEAT_TRBE_EXC also gate PSTATE.PM one day... */ + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR1_EL1, EBEP, IMP)) + mask |= BIT_ULL(32); /* SPSR_ELx.PM */ + + spsr &= mask; - spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | - PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | - PSR_MODE_MASK | PSR_MODE32_BIT); + mask = PSR_MODE_MASK | PSR_MODE32_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP)) + mask |= BIT_ULL(34); /* PSTATE.EXLOCK */ + + spsr |= *vcpu_cpsr(vcpu) & mask; spsr |= PSR_IL_BIT; } @@ -2784,7 +2800,7 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) * ERET handling, and the guest will have a little surprise. */ if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) { - esr &= ESR_ELx_ERET_ISS_ERETA; + esr &= (ESR_ELx_ERET_ISS_ERETA | ESR_ELx_IL); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC); kvm_inject_nested_sync(vcpu, esr); return; @@ -2826,6 +2842,7 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2, break; case except_type_serror: kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR); + vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2); break; default: WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type); @@ -2950,6 +2967,6 @@ int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr) * vSError injection. Manually populate EC for an emulated SError * exception. */ - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; return kvm_inject_nested(vcpu, esr, except_type_serror); } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 18131e395e24..4bf624a49591 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -448,16 +448,19 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) { + u64 spsr; + *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* * Finish potential single step before executing the prologue - * instruction. + * instruction. Modify the hardware SPSR_EL2 directly, as vcpu_cpsr() + * may hold a synthetic (vEL2) value for a guest hypervisor. */ - *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; - write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); + spsr = read_sysreg_el2(SYS_SPSR); + write_sysreg_el2(spsr & ~DBG_SPSR_SS, SYS_SPSR); return true; } @@ -602,8 +605,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) return false; break; case ESR_ELx_EC_SYS64: - if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) - return false; fallthrough; case ESR_ELx_EC_SVE: if (!sve_guest) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 1af722771178..a327c2bbb6b6 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -352,7 +352,7 @@ static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE)) @@ -372,7 +372,7 @@ static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE)) @@ -476,11 +476,12 @@ static void __do_ffa_mem_xfer(const u64 func_id, DECLARE_REG(u32, fraglen, ctxt, 2); DECLARE_REG(u64, addr_mbz, ctxt, 3); DECLARE_REG(u32, npages_mbz, ctxt, 4); + u32 offset, nr_ranges, checked_offset, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; struct ffa_mem_region *buf; - u32 offset, nr_ranges, checked_offset; int ret = 0; + size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { @@ -488,8 +489,7 @@ static void __do_ffa_mem_xfer(const u64 func_id, goto out; } - if (fraglen < sizeof(struct ffa_mem_region) + - sizeof(struct ffa_mem_region_attributes)) { + if (fraglen < mem_region_len + ffa_emad_size_get(hyp_ffa_version)) { ret = FFA_RET_INVALID_PARAMETERS; goto out; } @@ -508,8 +508,13 @@ static void __do_ffa_mem_xfer(const u64 func_id, buf = hyp_buffers.tx; memcpy(buf, host_buffers.tx, fraglen); - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; @@ -574,9 +579,9 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, handle_lo, ctxt, 1); DECLARE_REG(u32, handle_hi, ctxt, 2); DECLARE_REG(u32, flags, ctxt, 3); + u32 offset, len, fraglen, fragoff, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; - u32 offset, len, fraglen, fragoff; struct ffa_mem_region *buf; int ret = 0; u64 handle; @@ -599,16 +604,22 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, len = res->a1; fraglen = res->a2; - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + ffa_rx_release(res); + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; /* * We can trust the SPMD to get this right, but let's at least * check that we end up with something that doesn't look _completely_ * bogus. */ - if (WARN_ON(offset > len || - fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) { + if (offset + CONSTITUENTS_OFFSET(0) > len || + fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_ABORTED; ffa_rx_release(res); goto out_unlock; @@ -636,11 +647,16 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, ffa_rx_release(res); } + reg = (void *)buf + offset; + if (offset + CONSTITUENTS_OFFSET(reg->addr_range_cnt) > len) { + ret = FFA_RET_ABORTED; + goto out_unlock; + } + ffa_mem_reclaim(res, handle_lo, handle_hi, flags); if (res->a0 != FFA_SUCCESS) goto out_unlock; - reg = (void *)buf + offset; /* If the SPMD was happy, then we should be too. */ WARN_ON(ffa_host_unshare_ranges(reg->constituents, reg->addr_range_cnt)); @@ -864,7 +880,7 @@ out_unlock: bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { - struct arm_smccc_1_2_regs res; + struct arm_smccc_1_2_regs res = {0}; /* * There's no way we can tell what a non-standard SMC call might diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3b2c4fbc34d8..24d6f164129a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -1056,7 +1056,8 @@ static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa) /* Fake up a data abort (level 3 translation fault on write) */ vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | - ESR_ELx_WNR | ESR_ELx_FSC_FAULT | + ESR_ELx_IL | ESR_ELx_WNR | + ESR_ELx_FSC_FAULT | FIELD_PREP(ESR_ELx_FSC_LEVEL, 3); /* Shuffle the IPA around into the HPFAR */ diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8c3fbb413a06..b1411fb54139 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -268,6 +268,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) write_sysreg_el1(esr, SYS_ESR); write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); + write_sysreg_el1(read_sysreg_el2(SYS_SPSR), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); } @@ -278,7 +279,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) */ static void inject_undef64(struct kvm_vcpu *vcpu) { - inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT)); + inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL); } static u64 read_id_reg(const struct kvm_vcpu *vcpu, diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 91a7dfad6686..b74dd5ce1efd 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1370,16 +1370,19 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, if (prot & KVM_PGTABLE_PROT_W) set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; - ret = stage2_set_xn_attr(prot, &xn); - if (ret) - return ret; + if (prot & KVM_PGTABLE_PROT_X) { + ret = stage2_set_xn_attr(prot, &xn); + if (ret) + return ret; - set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; - clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + } ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags); if (!ret || ret == -EAGAIN) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, + (ret == -EAGAIN) ? TLBI_TTL_UNKNOWN : level); return ret; } diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 89982bd3345f..d6c4fc16f879 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -138,11 +138,10 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr pend_sync_exception(vcpu); /* - * Build an {i,d}abort, depending on the level and the - * instruction set. Report an external synchronous abort. + * Build an {i,d}abort, depending on the level. + * Report an external synchronous abort. */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + esr |= ESR_ELx_IL; /* * Here, the guest runs in AArch64 mode when in EL1. If we get @@ -170,14 +169,7 @@ void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr) static void inject_undef64(struct kvm_vcpu *vcpu) { - u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); - - /* - * Build an unknown exception, depending on the instruction - * set. - */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; kvm_inject_sync(vcpu, esr); } @@ -389,7 +381,7 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) */ if (!serror_is_masked(vcpu)) { pend_serror_exception(vcpu); - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); return 1; } diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index e2285ed8c91d..d1c3a352d5a2 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -126,6 +126,10 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) len = kvm_vcpu_dabt_get_as(vcpu); data = kvm_mmio_read_buf(run->mmio.data, len); + trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, + &data); + data = vcpu_data_host_to_guest(vcpu, data, len); + if (kvm_vcpu_dabt_issext(vcpu) && len < sizeof(unsigned long)) { mask = 1U << ((len * 8) - 1); @@ -135,9 +139,6 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) if (!kvm_vcpu_dabt_issf(vcpu)) data = data & 0xffffffff; - trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, - &data); - data = vcpu_data_host_to_guest(vcpu, data, len); vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..dfb96edbdc43 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -24,6 +24,7 @@ struct vncr_tlb { struct s1_walk_result wr; u64 hpa; + bool hpa_writable; /* -1 when not mapped on a CPU */ int cpu; @@ -1401,15 +1402,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); - if (!memslot) + if (!memslot) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } *is_gmem = kvm_slot_has_gmem(memslot); if (!*is_gmem) { pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, &writable, &page); - if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) + if (is_error_noslot_pfn(pfn)) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } } else { ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); if (ret) { @@ -1417,6 +1422,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) write_fault, false, false); return ret; } + + writable = !(memslot->flags & KVM_MEM_READONLY); + } + + /* + * FIXME: This check is too restrictive as KVM allows cacheable memory + * attributes for PFNMAP VMAs that have cacheable attributes in host + * stage-1. + */ + if (!pfn_is_map_memory(pfn)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); + return -EINVAL; } scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { @@ -1427,116 +1445,100 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->gva = va; vt->hpa = pfn << PAGE_SHIFT; + vt->hpa_writable = writable; vt->valid = true; vt->cpu = -1; kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); - kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); + kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); } - if (vt->wr.pw) + if (vt->wr.pw && vt->hpa_writable) mark_page_dirty(vcpu->kvm, gfn); return 0; } -static void inject_vncr_perm(struct kvm_vcpu *vcpu) +static void handle_vncr_perm(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + u64 fsc; + + /* + * Promote to an external abort if the stage-1 permits writes but the + * HPA is read-only (e.g. RO memslot). + */ + if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) + fsc = ESR_ELx_FSC_EXTABT; + /* + * Otherwise, inject a permission fault using the guest's translation + * level rather than the host's. + */ + else + fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); - /* Adjust the fault level to reflect that of the guest's */ esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, - ESR_ELx_FSC_PERM_L(vt->wr.level)); + esr |= FIELD_PREP(ESR_ELx_FSC, fsc); kvm_inject_nested_sync(vcpu, esr); } -static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) -{ - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; - - lockdep_assert_held_read(&vcpu->kvm->mmu_lock); - - if (!vt->valid) - return false; - - if (read_vncr_el2(vcpu) != vt->gva) - return false; - - if (vt->wr.nG) - return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid; - - return true; -} - int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + bool is_gmem = false; + bool perm; + int ret; WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); if (kvm_vcpu_abt_issea(vcpu)) return kvm_handle_guest_sea(vcpu); - if (esr_fsc_is_permission_fault(esr)) { - inject_vncr_perm(vcpu); - } else if (esr_fsc_is_translation_fault(esr)) { - bool valid, is_gmem = false; - int ret; - - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - valid = kvm_vncr_tlb_lookup(vcpu); - - if (!valid) - ret = kvm_translate_vncr(vcpu, &is_gmem); - else - ret = -EPERM; - - switch (ret) { - case -EAGAIN: - /* Let's try again... */ - break; - case -ENOMEM: - /* - * For guest_memfd, this indicates that it failed to - * create a folio to back the memory. Inform userspace. - */ - if (is_gmem) - return 0; - /* Otherwise, let's try again... */ - break; - case -EFAULT: - case -EIO: - case -EHWPOISON: - if (is_gmem) - return 0; - fallthrough; - case -EINVAL: - case -ENOENT: - case -EACCES: - /* - * Translation failed, inject the corresponding - * exception back to EL2. - */ - BUG_ON(!vt->wr.failed); + if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { + KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); + return -EIO; + } - esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); + ret = kvm_translate_vncr(vcpu, &is_gmem); + switch (ret) { + case -EAGAIN: + /* Let's try again... */ + return 1; + case -ENOMEM: + /* + * For guest_memfd, this indicates that it failed to + * create a folio to back the memory. Inform userspace. + */ + if (is_gmem) + return 0; + /* Otherwise, let's try again... */ + break; + case -EFAULT: + case -EIO: + case -EHWPOISON: + if (is_gmem) + return 0; + fallthrough; + case -EINVAL: + case -ENOENT: + case -EACCES: + /* + * Translation failed, inject the corresponding + * exception back to EL2. + */ + esr &= ~ESR_ELx_FSC; + esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); - kvm_inject_nested_sync(vcpu, esr); - break; - case -EPERM: - /* Hack to deal with POE until we get kernel support */ - inject_vncr_perm(vcpu); - break; - case 0: - break; - } - } else { - WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); + kvm_inject_nested_sync(vcpu, esr); + break; + case 0: + perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; + if (!perm) + handle_vncr_perm(vcpu); + break; } return 1; @@ -1574,7 +1576,7 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) vt->cpu = smp_processor_id(); - if (vt->wr.pw && vt->wr.pr) + if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) prot = PAGE_KERNEL_RO; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..428723b1b0f5 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -352,7 +352,7 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64 page = pfn_to_page(mapping->pfn); WARN_ON_ONCE(mapping->nr_pages != 1); unpin_user_pages_dirty_lock(&page, 1, true); - account_locked_vm(current->mm, 1, false); + account_locked_vm(kvm->mm, 1, false); pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); kfree(mapping); } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 4477f870c7b3..740b39875728 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -508,6 +508,8 @@ static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db) struct kvm_io_device *kvm_io_dev; struct vgic_io_device *iodev; + guard(srcu)(&kvm->srcu); + kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db); if (!kvm_io_dev) return ERR_PTR(-EINVAL); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5a4768d8cd4f..ccb7e3a90cd0 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -203,6 +203,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu) list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { if (irq_is_lpi(vcpu->kvm, irq->intid)) { raw_spin_lock(&irq->irq_lock); + irq->pending_latch = false; list_del(&irq->ap_list); irq->vcpu = NULL; raw_spin_unlock(&irq->irq_lock); @@ -792,7 +793,11 @@ retry: continue; } - /* This interrupt looks like it has to be migrated. */ + /* + * This interrupt looks like it has to be migrated, + * make sure it is kept alive while locks are dropped. + */ + vgic_get_irq_ref(irq); raw_spin_unlock(&irq->irq_lock); raw_spin_unlock(&vgic_cpu->ap_list_lock); @@ -815,15 +820,16 @@ retry: raw_spin_lock(&irq->irq_lock); /* - * If the affinity has been preserved, move the - * interrupt around. Otherwise, it means things have - * changed while the interrupt was unlocked, and we - * need to replay this. + * If the interrupt is still ours and its affinity has + * been preserved, move it around. Otherwise, it means + * things have changed while the interrupt was unlocked + * (it may even have been taken off the list with its + * affinity left untouched), and we need to replay this. * * In all cases, we cannot trust the list not to have * changed, so we restart from the beginning. */ - if (target_vcpu == vgic_target_oracle(irq)) { + if (irq->vcpu == vcpu && target_vcpu == vgic_target_oracle(irq)) { struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; list_del(&irq->ap_list); @@ -836,6 +842,8 @@ retry: raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + if (target_vcpu_needs_kick) { kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu); kvm_vcpu_kick(target_vcpu); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f2be501468ce..a25d8beacc83 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1515,7 +1515,13 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, if (free_mapped) { /* CONT blocks are not supported in the vmemmap */ WARN_ON(pmd_cont(pmd)); - flush_tlb_kernel_range(addr, addr + PMD_SIZE); + /* + * Invalidating a block entry requires just + * a single overlapping TLB invalidation, + * so limit the range of the flush to a single + * page. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pmd_page(pmd), PMD_SIZE, altmap); } @@ -1545,7 +1551,8 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, if (pud_leaf(pud)) { pud_clear(pudp); if (free_mapped) { - flush_tlb_kernel_range(addr, addr + PUD_SIZE); + /* See comment in unmap_hotplug_pmd_range(). */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pud_page(pud), PUD_SIZE, altmap); } diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index bc1788b1662b..7cb61aca3797 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1806,7 +1806,7 @@ Res0 15:8 UnsignedEnum 7:4 BWE 0b0000 NI 0b0001 FEAT_BWE - 0b0002 FEAT_BWE2 + 0b0010 FEAT_BWE2 EndEnum UnsignedEnum 3:0 STEP 0b0000 NI diff --git a/arch/m68k/coldfire/m523x.c b/arch/m68k/coldfire/m523x.c index 11d7423ef646..ec04c32bb03c 100644 --- a/arch/m68k/coldfire/m523x.c +++ b/arch/m68k/coldfire/m523x.c @@ -79,7 +79,7 @@ static void __init m523x_i2c_init(void) static void __init m523x_fec_init(void) { /* Set multi-function pins to ethernet use */ - mcf_write8(read8(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C); + mcf_write8(mcf_read8(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C); } /***************************************************************************/ diff --git a/arch/m68k/coldfire/m528x.c b/arch/m68k/coldfire/m528x.c index b244c9ba40a7..3383b1ba106a 100644 --- a/arch/m68k/coldfire/m528x.c +++ b/arch/m68k/coldfire/m528x.c @@ -113,11 +113,11 @@ void wildfiremod_halt(void) mcf_write16(read16(MCFGPIO_PEPAR) & ~(1 << (5 * 2)), MCFGPIO_PEPAR); /* Make portE.5 an output */ - mcf_write8(read8(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E); + mcf_write8(mcf_read8(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E); /* Now toggle portE.5 from low to high */ - mcf_write8(read8(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E); - mcf_write8(read8(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E); + mcf_write8(mcf_read8(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E); + mcf_write8(mcf_read8(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E); printk(KERN_EMERG "Failed to hibernate. Halting!\n"); } diff --git a/arch/m68k/include/asm/page_mm.h b/arch/m68k/include/asm/page_mm.h index ed782609ca41..0971a0651d49 100644 --- a/arch/m68k/include/asm/page_mm.h +++ b/arch/m68k/include/asm/page_mm.h @@ -55,10 +55,12 @@ static inline void clear_page(void *page) #define clear_user_page(addr, vaddr, page) \ do { clear_page(addr); \ flush_dcache_page(page); \ + (void)(vaddr); \ } while (0) #define copy_user_page(to, from, vaddr, page) \ do { copy_page(to, from); \ flush_dcache_page(page); \ + (void)(vaddr); \ } while (0) extern unsigned long m68k_memoffset; diff --git a/arch/powerpc/include/asm/preempt.h b/arch/powerpc/include/asm/preempt.h deleted file mode 100644 index 000e2b9681f3..000000000000 --- a/arch/powerpc/include/asm/preempt.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_POWERPC_PREEMPT_H -#define __ASM_POWERPC_PREEMPT_H - -#include <asm-generic/preempt.h> - -#if defined(CONFIG_PREEMPT_DYNAMIC) -#include <linux/jump_label.h> -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -#define need_irq_preemption() \ - (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) -#else -#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION)) -#endif - -#endif /* __ASM_POWERPC_PREEMPT_H */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 7b8c56962c31..49039074b33f 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -537,7 +537,7 @@ static inline void __user *mask_user_address(const void __user *ptr) if (IS_ENABLED(CONFIG_PPC64)) return mask_user_address_simple(ptr); - if (IS_ENABLED(CONFIG_E500)) + if (IS_ENABLED(CONFIG_PPC_E500)) return mask_user_address_isel(ptr); if (TASK_SIZE <= UL(SZ_2G) && border >= UL(SZ_2G)) return mask_user_address_simple(ptr); diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 3af6c06af02f..e5853daa6a48 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -704,6 +704,15 @@ static void __init cpufeatures_setup_start(u32 isa) if (isa >= ISA_V3_1) { cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; + + /* + * CPU_FTR_P11_PVR is a kernel-internal flag to identify + * Power11 and later processors. While ISA v3.1 is supported + * by Power10+, this flag specifically indicates Power11+ + * for code that needs to distinguish between P10 and P11. + */ + if (PVR_VER(mfspr(SPRN_PVR)) >= PVR_POWER11) + cur_cpu_spec->cpu_features |= CPU_FTR_P11_PVR; } } diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 3460d1a5a97c..11145c40183d 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -377,7 +377,6 @@ void vtime_task_switch(struct task_struct *prev) } } -#ifdef CONFIG_NO_HZ_COMMON /** * vtime_reset - Fast forward vtime entry clocks * @@ -394,6 +393,7 @@ void vtime_reset(void) #endif } +#ifdef CONFIG_NO_HZ_COMMON /** * vtime_dyntick_start - Inform vtime about entry to idle-dynticks * @@ -933,6 +933,7 @@ static void __init set_decrementer_max(void) static void __init init_decrementer_clockevent(void) { register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } void secondary_cpu_time_init(void) @@ -948,6 +949,7 @@ void secondary_cpu_time_init(void) /* FIME: Should make unrelated change to move snapshot_timebase * call here ! */ register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } /* diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c index 57e897b60db8..cc9fb72cb4eb 100644 --- a/arch/powerpc/lib/vmx-helper.c +++ b/arch/powerpc/lib/vmx-helper.c @@ -46,7 +46,7 @@ int exit_vmx_usercopy(void) * set and we are preemptible. The hack here is to schedule a * decrementer to fire here and reschedule for us if necessary. */ - if (need_irq_preemption() && need_resched()) + if (need_resched()) set_dec(1); return 0; } diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c index 757811155587..c11deb2f50ed 100644 --- a/arch/powerpc/platforms/85xx/common.c +++ b/arch/powerpc/platforms/85xx/common.c @@ -42,6 +42,8 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = { { .compatible = "fsl,qoriq-pcie-v2.3", }, { .compatible = "fsl,qoriq-pcie-v2.2", }, { .compatible = "fsl,fman", }, + /* IFC NAND and NOR controllers */ + { .compatible = "fsl,ifc", }, {}, }; diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index f6de8c1169d5..de7494748fec 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -268,10 +268,12 @@ static int spufs_mem_mmap_access(struct vm_area_struct *vma, if (write && !(vma->vm_flags & VM_WRITE)) return -EACCES; + if (offset >= LS_SIZE) + return -EFAULT; if (spu_acquire(ctx)) return -EINTR; - if ((offset + len) > vma->vm_end) - len = vma->vm_end - offset; + if ((offset + len) > LS_SIZE) + len = LS_SIZE - offset; local_store = ctx->ops->get_ls(ctx); if (write) memcpy_toio(local_store + offset, buf, len); diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index f7052b131a4c..74910ce3a541 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -154,6 +154,7 @@ config HV_PERF_CTRS config VPA_PMU tristate "VPA PMU events" depends on KVM_BOOK3S_64_HV && HV_PERF_CTRS + default m help Enable access to the VPA PMU counters via perf. This enables code that support measurement for KVM on PowerVM(KoP) feature. diff --git a/arch/powerpc/platforms/pseries/papr_platform_attributes.c b/arch/powerpc/platforms/pseries/papr_platform_attributes.c index c6159870de0e..9c3758aa54c6 100644 --- a/arch/powerpc/platforms/pseries/papr_platform_attributes.c +++ b/arch/powerpc/platforms/pseries/papr_platform_attributes.c @@ -271,11 +271,9 @@ retry: esi_buf_size = ESI_HDR_SIZE + (CURR_MAX_ESI_ATTRS * max_esi_attrs); temp_esi_buf = krealloc(esi_buf, esi_buf_size, GFP_KERNEL); - if (temp_esi_buf) - esi_buf = temp_esi_buf; - else - return -ENOMEM; - + if (!temp_esi_buf) + goto out_free_esi_buf; + esi_buf = temp_esi_buf; goto retry; } diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c0a6992933e4..f7028caaeae0 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -157,7 +157,7 @@ config RISCV select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) - select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C + select FUNCTION_ALIGNMENT_4B if DYNAMIC_FTRACE && RISCV_ISA_C select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index c174ac0ec46b..429e07589306 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -84,6 +84,12 @@ config ARCH_THEAD help This enables support for the RISC-V based T-HEAD SoCs. +config ARCH_ULTRARISC + bool "UltraRISC RISC-V SoCs" + help + This enables support for UltraRISC SoC platform hardware, + including boards based on the UR-DP1000. + config ARCH_VIRT bool "QEMU Virt Machine" select POWER_RESET diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 74ba5acc12a4..ed605b5e3162 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -33,6 +33,7 @@ CONFIG_SOC_STARFIVE=y CONFIG_ARCH_SUNXI=y CONFIG_ARCH_TENSTORRENT=y CONFIG_ARCH_THEAD=y +CONFIG_ARCH_ULTRARISC=y CONFIG_ARCH_VIRT=y CONFIG_ARCH_CANAAN=y CONFIG_SMP=y diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 8cfe59483a8f..c2b0a2928f06 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -40,7 +40,7 @@ do { \ flush_icache_mm(vma->vm_mm, 0); \ } while (0) -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* This is accessed in assembly code. cpumask_var_t would be too complex. */ extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); extern char _end[]; @@ -56,7 +56,8 @@ static inline void mark_new_valid_map(void) #define flush_cache_vmap flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (is_vmalloc_or_module_addr((void *)start)) + if (is_vmalloc_or_module_addr((void *)start) || + (start >= VMEMMAP_START && end <= VMEMMAP_END)) mark_new_valid_map(); } #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 09bb5f57a9d3..92d5f831f349 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -102,12 +102,14 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) #define readsw(addr, buffer, count) __readsw(addr, buffer, count) #define readsl(addr, buffer, count) __readsl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) #define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) #define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) #define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) __io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) @@ -116,26 +118,32 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) #define writesw(addr, buffer, count) __writesw(addr, buffer, count) #define writesl(addr, buffer, count) __writesl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) #define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) #define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) #define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) +#endif #ifdef CONFIG_64BIT __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) #define readsq(addr, buffer, count) __readsq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) #define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) #define writesq(addr, buffer, count) __writesq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) #endif +#endif #include <asm-generic/io.h> diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 60017ceec9d2..e2d5808169e4 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -209,13 +209,13 @@ struct kvm_vcpu_arch { /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 08df724e13b9..d799c4e56f80 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -137,7 +137,7 @@ SYM_CODE_START(handle_exception) .Lrestore_kernel_tpsp: csrr tp, CSR_SCRATCH -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* * The RISC-V kernel does not flush TLBs on all CPUS after each new * vmalloc mapping or kfence_unprotect(), which may result in diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 2306ce3e5f22..738df176ff6f 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -41,6 +41,9 @@ machine_kexec_prepare(struct kimage *image) if (image->segment[i].memsz <= sizeof(fdt)) continue; + if (!image->segment[i].buf) + continue; + if (image->file_mode) memcpy(&fdt, image->segment[i].buf, sizeof(fdt)); else if (copy_from_user(&fdt, image->segment[i].buf, sizeof(fdt))) diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 1659d31fd288..caf6762427c8 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -450,6 +450,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, if (cpusetsize > cpumask_size()) cpusetsize = cpumask_size(); + cpumask_clear(&cpus); ret = copy_from_user(&cpus, cpus_user, cpusetsize); if (ret) return -EFAULT; diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index a842dc034571..43ee881f6c6f 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -69,9 +69,9 @@ CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif # Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) # Force dependency $(obj)/$(vdso_o): $(obj)/$(vdso_so) diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index e82987dc3739..d6f96b1abe40 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -7,11 +7,19 @@ #include <asm/unistd.h> #include <asm/assembler.h> +/* + * WARNING: Do NOT add a CFI landing pad at the start of this function. + * Unwinders such as libgcc identify the sigreturn trampoline by matching the + * instruction sequence. Adding a landing pad here would break unwinding from + * signal handlers. + * + * This trampoline is used only for signal return and not via an indirect + * call/jump from userspace, so adding CFI landing pad is unnecessary. + */ .text SYM_FUNC_START(__vdso_rt_sigreturn) .cfi_startproc .cfi_signal_frame - vdso_lpad li a7, __NR_rt_sigreturn ecall .cfi_endproc diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index bafb009c5ce5..9a653b4ad40a 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -53,12 +53,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long mask, val; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (!kvm_riscv_aia_available()) return; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; + mask = vcpu->arch.irqs_pending_mask[1]; + if (mask) { + vcpu->arch.irqs_pending_mask[1] = 0; + val = vcpu->arch.irqs_pending[1] & mask; csr->hviph &= ~mask; csr->hviph |= val; @@ -69,6 +72,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (kvm_riscv_aia_available()) csr->vsieh = ncsr_read(CSR_VSIEH); } @@ -77,13 +82,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { unsigned long seip; +#ifdef CONFIG_32BIT + unsigned long flags; + bool pending; +#endif if (!kvm_riscv_aia_available()) return false; #ifdef CONFIG_32BIT - if (READ_ONCE(vcpu->arch.irqs_pending[1]) & - (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + pending = vcpu->arch.irqs_pending[1] & + (vcpu->arch.aia_context.guest_csr.vsieh & + upper_32_bits(mask)); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + if (pending) return true; #endif @@ -207,6 +221,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); +#ifdef CONFIG_32BIT + unsigned long flags; +#endif if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; @@ -219,8 +236,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = val; #ifdef CONFIG_32BIT - if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); + if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[1] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, + flags); + } #endif } diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index c4c3b79567f1..b0474fcf065a 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -5,11 +5,13 @@ */ #include <linux/bitops.h> +#include <linux/cpufeature.h> #include <linux/errno.h> #include <linux/kvm_host.h> #include <linux/module.h> #include <linux/pgtable.h> #include <asm/kvm_gstage.h> +#include <asm/hwcap.h> #ifdef CONFIG_64BIT unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3; @@ -171,8 +173,10 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, } if (pte_val(*ptep) != pte_val(map->pte)) { + bool was_invalid = !pte_val(*ptep); set_pte(ptep, map->pte); - if (gstage_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep) && + !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))) gstage_tlb_flush(gstage, current_level, map->addr); } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 082f9b261733..8a0aa5e0e216 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -41,6 +41,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, pgprot_t prot; unsigned long pfn; phys_addr_t addr, end; + unsigned long pgd_levels = kvm->arch.pgd_levels; struct kvm_mmu_memory_cache pcache = { .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, .gfp_zero = __GFP_ZERO, @@ -63,7 +64,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, if (!writable) map.pte = pte_wrprotect(map.pte); - ret = kvm_mmu_topup_memory_cache(&pcache, kvm->arch.pgd_levels); + ret = __kvm_mmu_topup_memory_cache(&pcache, pgd_levels, pgd_levels); if (ret) goto out; diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index cf6e231e76e2..977e36ab83d3 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -80,6 +80,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) { + unsigned long flags; bool loaded; /** @@ -104,8 +105,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) kvm_riscv_vcpu_aia_reset(vcpu); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_riscv_vcpu_pmu_reset(vcpu); @@ -151,6 +154,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); + raw_spin_lock_init(&vcpu->arch.irqs_pending_lock); spin_lock_init(&vcpu->arch.reset_state.lock); @@ -352,10 +356,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long mask, val; + unsigned long flags; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + + mask = vcpu->arch.irqs_pending_mask[0]; + if (mask) { + vcpu->arch.irqs_pending_mask[0] = 0; + val = vcpu->arch.irqs_pending[0] & mask; csr->hvip &= ~mask; csr->hvip |= val; @@ -363,11 +371,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) /* Flush AIA high interrupts */ kvm_riscv_vcpu_aia_flush_interrupts(vcpu); + + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); } void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) { unsigned long hvip; + unsigned long flags; struct kvm_vcpu_arch *v = &vcpu->arch; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; @@ -376,34 +387,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) /* Sync-up HVIP.VSSIP bit changes does by Guest */ hvip = ncsr_read(CSR_HVIP); + + raw_spin_lock_irqsave(&v->irqs_pending_lock, flags); + if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { if (hvip & (1UL << IRQ_VS_SOFT)) { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - set_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __set_bit(IRQ_VS_SOFT, v->irqs_pending); } else { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - clear_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __clear_bit(IRQ_VS_SOFT, v->irqs_pending); } } /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { if (!(hvip & (1UL << IRQ_PMU_OVF)) && - !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) - clear_bit(IRQ_PMU_OVF, v->irqs_pending); + !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) + __clear_bit(IRQ_PMU_OVF, v->irqs_pending); } /* Sync-up AIA high interrupts */ kvm_riscv_vcpu_aia_sync_interrupts(vcpu); + raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags); + /* Sync-up timer CSRs */ kvm_riscv_vcpu_timer_sync(vcpu); } int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, and external * interrupts when irq is one of the local interrupts @@ -416,9 +434,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - set_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __set_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_vcpu_kick(vcpu); @@ -427,6 +446,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, counter overflow and external * interrupts when irq is one of the local interrupts @@ -439,26 +460,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - clear_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __clear_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); return 0; } bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + unsigned long flags; unsigned long ie; + bool ret; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; - if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) - return true; + ret = vcpu->arch.irqs_pending[0] & ie; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); /* Check AIA high interrupts */ - return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + if (!ret) + ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + + return ret; } void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 0bb0c51e3c89..6c8530b9f29e 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -38,6 +38,25 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, return kvm_riscv_vcpu_mmio_store(vcpu, run, fault_addr, trap->htinst); + case EXC_INST_GUEST_PAGE_FAULT: { + /* + * No memslot backs this GPA and an instruction fetch + * cannot be emulated as MMIO. On bare metal a fetch + * from an unbacked physical address raises an + * instruction access fault, so reflect that back to + * the guest. + */ + struct kvm_cpu_trap inst_trap = { + .sepc = trap->sepc, + .scause = EXC_INST_ACCESS, + .stval = trap->stval, + .htval = 0, + .htinst = 0, + }; + + kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); + return 1; + } default: return -EOPNOTSUPP; }; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index bb920e8923c9..99b9107b1ac1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -50,19 +50,13 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - reg_val = riscv_cbom_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - reg_val = riscv_cboz_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - reg_val = riscv_cbop_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): reg_val = vcpu->arch.mvendorid; @@ -144,21 +138,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, } break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - if (reg_val != riscv_cbom_block_size) + if (reg_val && reg_val != riscv_cbom_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - if (reg_val != riscv_cboz_block_size) + if (reg_val && reg_val != riscv_cboz_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - if (reg_val != riscv_cbop_block_size) + if (reg_val && reg_val != riscv_cbop_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): @@ -298,6 +286,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); + unsigned long flags; if (reg_num >= regs_max) return -ENOENT; @@ -311,8 +300,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = reg_val; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[0] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + } return 0; } @@ -614,20 +606,6 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, u64 size; u64 reg; - /* - * Avoid reporting config reg if the corresponding extension - * was not available. - */ - if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && - kvm_riscv_isa_check_host(ZICBOM)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && - kvm_riscv_isa_check_host(ZICBOZ)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && - kvm_riscv_isa_check_host(ZICBOP)) - continue; - size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index bb46dcbfb24d..2025b664961c 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -586,7 +586,7 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } /* Start the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) @@ -658,7 +658,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } /* Stop the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c index ab39ac464ffd..1342adb3180c 100644 --- a/arch/riscv/kvm/vcpu_sbi_fwft.c +++ b/arch/riscv/kvm/vcpu_sbi_fwft.c @@ -327,9 +327,11 @@ static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) return SBI_ERR_DENIED_LOCKED; - conf->flags = flags; + ret = conf->feature->set(vcpu, conf, false, value); + if (ret == SBI_SUCCESS) + conf->flags = flags; - return conf->feature->set(vcpu, conf, false, value); + return ret; } static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 62d2fb77bb9b..3708616e2c32 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -10,6 +10,7 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> #include <asm/kvm_isa.h> @@ -129,11 +130,20 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, return -ENOENT; } } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { + unsigned long reg_offset; + if (reg_size != vlenb) return -EINVAL; WARN_ON(!cntx->vector.datap); - *reg_addr = cntx->vector.datap + - (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; + /* + * The reg_num is derived from the userspace-provided ONE_REG + * id. Sanitize it with array_index_nospec() to prevent + * speculative out-of-bounds access to the vector register + * buffer (32 vector registers: v0..v31). + */ + reg_offset = array_index_nospec( + reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); + *reg_addr = cntx->vector.datap + reg_offset * vlenb; } else { return -ENOENT; } diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 5b1b3c88b4d1..3e450890be07 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -37,7 +37,9 @@ #include "../kernel/head.h" +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); +#endif struct kernel_mapping kernel_map __ro_after_init; EXPORT_SYMBOL(kernel_map); diff --git a/arch/s390/kernel/diag/diag310.c b/arch/s390/kernel/diag/diag310.c index f411562aa7f6..c68ec9d28513 100644 --- a/arch/s390/kernel/diag/diag310.c +++ b/arch/s390/kernel/diag/diag310.c @@ -190,17 +190,18 @@ static int memtop_get_stride_len(unsigned long *res) static int memtop_get_page_count(unsigned long *res, unsigned long level) { static unsigned long memtop_pages[DIAG310_LEVELMAX]; - unsigned long pages; + unsigned long pages, idx; int rc; if (level > DIAG310_LEVELMAX || level < DIAG310_LEVELMIN) return -EINVAL; - pages = READ_ONCE(memtop_pages[level - 1]); + idx = array_index_nospec(level - 1, ARRAY_SIZE(memtop_pages)); + pages = READ_ONCE(memtop_pages[idx]); if (!pages) { rc = diag310_get_memtop_size(&pages, level); if (rc) return rc; - WRITE_ONCE(memtop_pages[level - 1], pages); + WRITE_ONCE(memtop_pages[idx], pages); } *res = pages; return 0; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 7aa655664ecc..2076ac22e2c4 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -15,6 +15,7 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/perf_event.h> +#include <linux/nospec.h> #include <asm/cpu_mf.h> #include <asm/hwctrset.h> @@ -768,6 +769,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user)); ev = cpumf_generic_events_user[ev]; } } else if (!attr->exclude_kernel && attr->exclude_user) { @@ -778,6 +780,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic)); ev = cpumf_generic_events_basic[ev]; } } diff --git a/arch/s390/kernel/vdso/note.S b/arch/s390/kernel/vdso/note.S index db19d0680a0a..d215781f26b5 100644 --- a/arch/s390/kernel/vdso/note.S +++ b/arch/s390/kernel/vdso/note.S @@ -4,6 +4,7 @@ * Here we can supply some information useful to userland. */ +#include <linux/build-salt.h> #include <linux/uts.h> #include <linux/version.h> #include <linux/elfnote.h> @@ -11,3 +12,5 @@ ELFNOTE_START(Linux, 0, "a") .long LINUX_VERSION_CODE ELFNOTE_END + +BUILD_SALT diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 5f1960ec982d..ed4259d17629 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -570,6 +570,8 @@ static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *tabl else if (walk->ops->pte_entry) rc = dat_pte_walk_range(max(start, cur), min(end, next), dereference_pmd(crste.pmd), walk); + if (rc) + break; } } return rc; diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 298fbaecec28..8abb4f55b306 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -1374,8 +1374,13 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare /* Only allow one real-space gmap shadow. */ list_for_each_entry(sg, &parent->children, list) { if (sg->guest_asce.r) { - scoped_guard(write_lock, &parent->kvm->mmu_lock) + if (write_trylock(&parent->kvm->mmu_lock)) { gmap_unshadow(sg); + write_unlock(&parent->kvm->mmu_lock); + } else { + gmap_put(new); + return ERR_PTR(-EAGAIN); + } break; } } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 23c817595e28..150b5dd2170e 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1280,8 +1280,10 @@ static int kvm_s390_vm_stop_migration(struct kvm *kvm) * PGSTEs might have cmma_d set. */ WRITE_ONCE(kvm->arch.migration_mode, 0); - if (kvm->arch.use_cmma) - kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); + if (!kvm->arch.use_cmma) + return 0; + + kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */ gmap_set_cmma_all_clean(kvm->arch.gmap); atomic64_set(&kvm->arch.cmma_dirty_pages, 0); diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 5b075c38998e..720bb58cabe2 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -300,9 +300,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, gaite->gisc = fib->fmt0.isc; gaite->count++; - gaite->aisbo = fib->fmt0.aisbo; - gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & - ~PAGE_MASK)); + if (fib->fmt0.sum == 1) { + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + + (fib->fmt0.aisb & ~PAGE_MASK)); + } else { + gaite->aisbo = 0; + gaite->aisb = 0; + } aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); @@ -328,6 +333,7 @@ unpin2: unpin1: unpin_user_page(aibv_page); out: + kvm_s390_gisc_unregister(kvm, fib->fmt0.isc); return rc; } diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c index 458abd9bac70..9d74ceff136c 100644 --- a/arch/s390/lib/csum-partial.c +++ b/arch/s390/lib/csum-partial.c @@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w if (!cpu_has_vx()) { if (copy) memcpy(dst, src, len); - return cksm(dst, len, sum); + return cksm(src, len, sum); } kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); fpu_vlvgf(16, (__force u32)sum, 1); diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 2a222a7e14f4..ef7bfc87758c 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -64,7 +64,7 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } -static int get_align_mask(struct file *filp, unsigned long flags) +static unsigned long get_align_mask(struct file *filp, unsigned long flags) { if (filp && is_file_hugepages(filp)) return huge_page_mask_align(filp); diff --git a/arch/x86/boot/compressed/acpi.c b/arch/x86/boot/compressed/acpi.c index f196b1d1ddf8..aed27604c11f 100644 --- a/arch/x86/boot/compressed/acpi.c +++ b/arch/x86/boot/compressed/acpi.c @@ -184,10 +184,15 @@ static unsigned long get_cmdline_acpi_rsdp(void) char val[MAX_ADDR_LEN] = { }; int ret; - ret = cmdline_find_option("acpi_rsdp", val, MAX_ADDR_LEN); + ret = cmdline_find_option("acpi_rsdp", val, sizeof(val)); if (ret < 0) return 0; + if (ret >= sizeof(val)) { + warn("acpi_rsdp= value too long; ignoring"); + return 0; + } + if (boot_kstrtoul(val, 16, &addr)) return 0; #endif diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c index 023bf1c3de8b..5b83beab89e1 100644 --- a/arch/x86/boot/early_serial_console.c +++ b/arch/x86/boot/early_serial_console.c @@ -117,7 +117,7 @@ static unsigned int probe_baud(int port) static void parse_console_uart8250(void) { char optstr[64], *options; - int baud = DEFAULT_BAUD; + int baud; int port = 0; /* @@ -136,10 +136,13 @@ static void parse_console_uart8250(void) else return; - if (options && (options[0] == ',')) - baud = simple_strtoull(options + 1, &options, 0); - else + if (options && (options[0] == ',')) { + baud = simple_strtoull(options + 1, NULL, 0); + if (!baud) + baud = DEFAULT_BAUD; + } else { baud = probe_baud(port); + } if (port) early_serial_init(port, baud); diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index 06f35a6b58a5..dc564688f3d7 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -259,13 +259,13 @@ void amd_brs_disable_all(void) amd_brs_disable(); } -static bool amd_brs_match_plm(struct perf_event *event, u64 to) +static bool amd_brs_match_plm(struct perf_event *event, u64 from, u64 to) { int type = event->attr.branch_sample_type; int plm_k = PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_HV; int plm_u = PERF_SAMPLE_BRANCH_USER; - if (!(type & plm_k) && kernel_ip(to)) + if (!(type & plm_k) && (kernel_ip(to) || kernel_ip(from))) return 0; if (!(type & plm_u) && !kernel_ip(to)) @@ -338,11 +338,11 @@ void amd_brs_drain(void) */ to = (u64)(((s64)to << shift) >> shift); - if (!amd_brs_match_plm(event, to)) - continue; - rdmsrq(brs_from(brs_idx), from); + if (!amd_brs_match_plm(event, from, to)) + continue; + perf_clear_branch_entry_bitfields(br+nr); br[nr].from = from; diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 6569048a8c1c..a787409f5a62 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -754,13 +754,11 @@ static void amd_pmu_enable_event(struct perf_event *event) x86_pmu_enable_event(event); } -static void amd_pmu_enable_all(int added) +static void __amd_pmu_enable_all(void) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx; - amd_brs_enable_all(); - for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { /* only activate events which are marked as active */ if (!test_bit(idx, cpuc->active_mask)) @@ -775,6 +773,12 @@ static void amd_pmu_enable_all(int added) } } +static void amd_pmu_enable_all(int added) +{ + amd_brs_enable_all(); + __amd_pmu_enable_all(); +} + static void amd_pmu_v2_enable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -1561,7 +1565,7 @@ static inline void amd_pmu_reload_virt(void) * set global enable bits once again */ amd_pmu_v2_disable_all(); - amd_pmu_enable_all(0); + __amd_pmu_enable_all(); amd_pmu_v2_enable_all(0); return; } diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 5b437dc8e4ce..9d9c961989d5 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -127,7 +127,8 @@ static void amd_pmu_lbr_filter(void) } /* If type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && kernel_ip(cpuc->lbr_entries[i].from))) { cpuc->lbr_entries[i].from = 0; /* mark invalid */ compress = true; } diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index d2e8a849f180..5b1070ec85d9 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -46,7 +46,6 @@ obj-$(CONFIG_CPU_SUP_HYGON) += hygon.o obj-$(CONFIG_CPU_SUP_CYRIX_32) += cyrix.o obj-$(CONFIG_CPU_SUP_CENTAUR) += centaur.o obj-$(CONFIG_CPU_SUP_TRANSMETA_32) += transmeta.o -obj-$(CONFIG_CPU_SUP_UMC_32) += umc.o obj-$(CONFIG_CPU_SUP_ZHAOXIN) += zhaoxin.o obj-$(CONFIG_CPU_SUP_VORTEX_32) += vortex.o diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 8c62c6d4d5c1..cb8ac4b9b0d7 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -488,8 +488,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) { ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry); - if (ret) + if (ret) { kvm->arch.nr_possible_bypass_irqs--; + irqfd->producer = NULL; + } } spin_unlock_irq(&kvm->irqfds.lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f30bbdddb5a..48b019114c19 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2052,7 +2052,7 @@ static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) ktimer->expired_tscdeadline = ktimer->tscdeadline; - if (!from_timer_fn && apic->apicv_active) { + if (!from_timer_fn && apic->apicv_active && vcpu->wants_to_run) { WARN_ON(kvm_get_running_vcpu() != vcpu); kvm_apic_inject_pending_timer_irqs(apic); return; @@ -3371,6 +3371,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { int vector; + + if (unlikely(!pv_eoi_enabled(vcpu))) { + __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + return; + } + /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host * and KVM_PV_EOI_ENABLED in guest memory as follows: @@ -3382,8 +3388,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: * -> host enabled PV EOI, guest executed EOI. */ - BUG_ON(!pv_eoi_enabled(vcpu)); - if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 234d0a95abf5..22cf222d3033 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2642,6 +2642,7 @@ static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, */ if (tdp_enabled && invalid_list && child->role.guest_mode && + !child->root_count && !atomic_long_read(&child->parent_ptes.val)) return kvm_mmu_prepare_zap_page(kvm, child, invalid_list); @@ -4852,16 +4853,17 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault if (r != RET_PF_CONTINUE) return r; - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = direct_map(vcpu, fault); out_unlock: @@ -7574,7 +7576,9 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) static void mmu_destroy_caches(void) { kmem_cache_destroy(pte_list_desc_cache); + pte_list_desc_cache = NULL; kmem_cache_destroy(mmu_page_header_cache); + mmu_page_header_cache = NULL; } static void kvm_wake_nx_recovery_thread(struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index df3ae0c7ec2c..1ba840a73b7a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -864,15 +864,17 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault } #endif - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = FNAME(fetch)(vcpu, fault, &walker); out_unlock: diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 427229347876..944aaea6501f 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2129,8 +2129,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (ret) return ret; + /* Do not allow SNP VM migration until additional state transfer is implemented */ if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { + sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { ret = -EINVAL; goto out_unlock; } @@ -2851,8 +2852,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disallow out-of-band SEV/SEV-ES init if the target is already an * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being * created after SEV/SEV-ES initialization, e.g. to init intercepts. + * Also do not allow SNP VM mirroring until additional state transfer is implemented. */ - if (sev_guest(kvm) || !sev_guest(source_kvm) || + if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; goto e_unlock; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4d2bacd00ec4..d0971685034b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -571,7 +571,12 @@ static int svm_enable_virtualization_cpu(void) return r; sd = per_cpu_ptr(&svm_data, me); - sd->asid_generation = 1; + /* + * Bump the current asid_generation value to ensure any vCPU that + * previously ran on this CPU sees a stale generation and is forced + * to acquire a new ASID, preventing a latent ASID collision. + */ + sd->asid_generation++; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 0db25bba17f6..93de876c318c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -490,7 +490,7 @@ TRACE_EVENT(kvm_inj_exception, TP_printk("%s%s%s%s%s", __print_symbolic(__entry->exception, kvm_trace_sym_exc), !__entry->has_error ? "" : " (", - !__entry->has_error ? "" : __print_symbolic(__entry->error_code, { }), + !__entry->has_error ? "" : __print_symbolic(__entry->error_code), !__entry->has_error ? "" : ")", __entry->reinjected ? " [reinjected]" : "") ); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6957bb6f5cf7..ddf6df7bee93 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -336,6 +336,7 @@ static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) static void free_nested(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct vmcs *shadow_vmcs; if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) vmx_switch_vmcs(vcpu, &vmx->vmcs01); @@ -353,9 +354,15 @@ static void free_nested(struct kvm_vcpu *vcpu) vmx->nested.current_vmptr = INVALID_GPA; if (enable_shadow_vmcs) { vmx_disable_shadow_vmcs(vmx); - vmcs_clear(vmx->vmcs01.shadow_vmcs); - free_vmcs(vmx->vmcs01.shadow_vmcs); + + /* + * Keep the pointer visible until after VMCLEAR, so migration + * can clear an active shadow VMCS on the old CPU. + */ + shadow_vmcs = vmx->vmcs01.shadow_vmcs; + vmcs_clear(shadow_vmcs); vmx->vmcs01.shadow_vmcs = NULL; + free_vmcs(shadow_vmcs); } kfree(vmx->nested.cached_vmcs12); vmx->nested.cached_vmcs12 = NULL; @@ -582,6 +589,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; + u32 vtpr; + if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) return 0; @@ -591,6 +601,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) return -EINVAL; + /* + * Do the illegal vTPR vs. TPR Threshold consistency check if and only + * if KVM is configured to WARN on missed consistency checks, otherwise + * it's a waste of time. KVM needs to rely on hardware to fully detect + * an illegal combination due to the vTPR being writable by L1 at all + * times (it's an in-memory value, not a VMCS field). I.e. even if the + * check passes now, it might fail at the actual VM-Enter. + * + * If reading guest memory fails, skip the check as KVM's de facto ABI + * for VMX instruction accesses to non-existent memory is to provide + * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR + * is guaranteed to greater than or equal to the threshold. + * + * Note! Deliberately use the VM-scoped API when reading guest memory, + * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, + * and only perform the check when in KVM_RUN, to avoid a false failure + * if userspace hasn't yet configured memslots during state restore. + */ + if (warn_on_missed_cc && vcpu->wants_to_run && + nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && + !nested_cpu_has_vid(vmcs12) && + !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && + !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && + CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) + return -EINVAL; + return 0; } @@ -3104,38 +3140,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, return 0; } -static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) -{ - void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; - u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; - - /* - * Don't bother with the consistency checks if KVM isn't configured to - * WARN on missed consistency checks, as KVM needs to rely on hardware - * to fully detect an illegal vTPR vs. TRP Threshold combination due to - * the vTPR being writable by L1 at all times (it's an in-memory value, - * not a VMCS field). I.e. even if the check passes now, it might fail - * at the actual VM-Enter. - * - * Keying off the module param also allows treating an invalid vAPIC - * mapping as a consistency check failure without increasing the risk - * of breaking a "real" VM. - */ - if (!warn_on_missed_cc) - return 0; - - if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && - !nested_cpu_has_vid(vmcs12) && - !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && - (CC(!vapic) || - CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) - return -EINVAL; - - return 0; -} - static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { @@ -3661,19 +3665,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, &vmx->nested.pre_vmenter_ssp_tbl); /* - * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the - * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but - * not KVM, KVM must unwind its software model to the pre-VM-Entry host - * state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not - * L1's "real" CR3, which causes nested_vmx_restore_host_state() to - * corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the - * unwind naturally setting arch.cr3 to the correct value. Smashing - * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind, - * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be - * overwritten with a shadow CR3 prior to re-entering L1. + * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a + * VM-Fail detected by hardware but not KVM, KVM can unwind its + * software model to the pre-VM-Entry host state. When EPT is + * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, + * and so simply restoring from vmcs01.GUEST_CR3 would corrupt + * vcpu->arch.cr3. */ - if (!enable_ept) - vmcs_writel(GUEST_CR3, vcpu->arch.cr3); + vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu); vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); @@ -3685,11 +3684,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_KVM_INTERNAL_ERROR; } - if (nested_vmx_check_controls_late(vcpu, vmcs12)) { - vmx_switch_vmcs(vcpu, &vmx->vmcs01); - return NVMX_VMENTRY_VMFAIL; - } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; @@ -3774,6 +3768,8 @@ vmentry_fail_vmexit: if (!from_vmentry) return NVMX_VMENTRY_VMEXIT; + nested_put_vmcs12_pages(vcpu); + load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) @@ -4990,7 +4986,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); nested_ept_uninit_mmu_context(vcpu); - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3; kvm_register_mark_available(vcpu, VCPU_REG_CR3); /* diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 989ab29b8c6f..545b03d9d10b 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2797,7 +2797,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) goto out; } - if (init_vm->cpuid.padding) { + /* + * Reject the request if userspace changes cpuid.nent between the + * initial read and the subsequent copy. + */ + if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) { ret = -EINVAL; goto out; } diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index de9de0d2016c..dc8517f15bc4 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -160,6 +160,13 @@ struct nested_vmx { bool preemption_timer_expired; /* + * Used to restore L1's CR3 if hardware detects a VM-Fail Consistency + * Check that KVM does not, in which case KVM needs to unwind CR3 back + * to its pre-VM-Enter state, NOT to vmcs01.HOST_CR3. + */ + unsigned long pre_vmenter_cr3; + + /* * Used to snapshot MSRs that are conditionally loaded on VM-Enter in * order to propagate the guest's pre-VM-Enter value into vmcs02. For * emulation of VMLAUNCH/VMRESUME, the snapshot will be of L1's value. diff --git a/arch/x86/video/video-common.c b/arch/x86/video/video-common.c index 152789f00fcd..8ed82fff7638 100644 --- a/arch/x86/video/video-common.c +++ b/arch/x86/video/video-common.c @@ -43,21 +43,26 @@ bool video_is_primary_device(struct device *dev) if (!pci_is_display(pdev)) return false; - if (pdev == vga_default_device()) - return true; - #ifdef CONFIG_SCREEN_INFO numres = screen_info_resources(si, res, ARRAY_SIZE(res)); - for (i = 0; i < numres; ++i) { - if (!(res[i].flags & IORESOURCE_MEM)) - continue; + if (numres > 0) { + for (i = 0; i < numres; ++i) { + if (!(res[i].flags & IORESOURCE_MEM)) + continue; + + if (pci_find_resource(pdev, &res[i])) + return true; + } - if (pci_find_resource(pdev, &res[i])) - return true; + return false; } #endif - return false; + /* + * No framebuffer was set up by the firmware/bootloader, so fall back + * to the default VGA device. + */ + return pdev == vga_default_device(); } EXPORT_SYMBOL(video_is_primary_device); diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 8bcdce98f6dc..cff285d8ad8e 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -536,6 +536,8 @@ int snp_prepare(void) goto unlock; } + wbinvd_on_all_cpus(); + /* * MtrrFixDramModEn is not shared between threads on a core, * therefore it must be set on all CPUs prior to enabling SNP. |