From 217c0a5c177a3d4f7c8497950cbf5c36756e8bbb Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Fri, 20 Mar 2026 15:59:48 +0200 Subject: x86/efi: efi_unmap_boot_services: fix calculation of ranges_to_free size ranges_to_free array should have enough room to store the entire EFI memmap plus an extra element for NULL entry. The calculation of this array size wrongly adds 1 to the overall size instead of adding 1 to the number of elements. Add parentheses to properly size the array. Reported-by: Guenter Roeck Fixes: a4b0bf6a40f3 ("x86/efi: defer freeing of boot services memory") Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Ard Biesheuvel --- arch/x86/platform/efi/quirks.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 35caa5746115..79f0818131e8 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -424,7 +424,7 @@ void __init efi_unmap_boot_services(void) if (efi_enabled(EFI_DBG)) return; - sz = sizeof(*ranges_to_free) * efi.memmap.nr_map + 1; + sz = sizeof(*ranges_to_free) * (efi.memmap.nr_map + 1); ranges_to_free = kzalloc(sz, GFP_KERNEL); if (!ranges_to_free) { pr_err("Failed to allocate storage for freeable EFI regions\n"); -- cgit v1.2.3 From 05243d490bb7852a8acca7b5b5658019c7797a52 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 18 Mar 2026 07:56:52 +0000 Subject: x86/cpu: Enable FSGSBASE early in cpu_init_exception_handling() Move FSGSBASE enablement from identify_cpu() to cpu_init_exception_handling() to ensure it is enabled before any exceptions can occur on both boot and secondary CPUs. == Background == Exception entry code (paranoid_entry()) uses ALTERNATIVE patching based on X86_FEATURE_FSGSBASE to decide whether to use RDGSBASE/WRGSBASE instructions or the slower RDMSR/SWAPGS sequence for saving/restoring GSBASE. On boot CPU, ALTERNATIVE patching happens after enabling FSGSBASE in CR4. When the feature is available, the code is permanently patched to use RDGSBASE/WRGSBASE, which require CR4.FSGSBASE=1 to execute without triggering == Boot Sequence == Boot CPU (with CR pinning enabled): trap_init() cpu_init() <- Uses unpatched code (RDMSR/SWAPGS) x2apic_setup() ... arch_cpu_finalize_init() identify_boot_cpu() identify_cpu() cr4_set_bits(X86_CR4_FSGSBASE) # Enables the feature # This becomes part of cr4_pinned_bits ... alternative_instructions() <- Patches code to use RDGSBASE/WRGSBASE Secondary CPUs (with CR pinning enabled): start_secondary() cr4_init() <- Code already patched, CR4.FSGSBASE=1 set implicitly via cr4_pinned_bits cpu_init() <- exceptions work because FSGSBASE is already enabled Secondary CPU (with CR pinning disabled): start_secondary() cr4_init() <- Code already patched, CR4.FSGSBASE=0 cpu_init() x2apic_setup() rdmsrq(MSR_IA32_APICBASE) <- Triggers #VC in SNP guests exc_vmm_communication() paranoid_entry() <- Uses RDGSBASE with CR4.FSGSBASE=0 (patched code) ... ap_starting() identify_secondary_cpu() identify_cpu() cr4_set_bits(X86_CR4_FSGSBASE) <- Enables the feature, which is too late == CR Pinning == Currently, for secondary CPUs, CR4.FSGSBASE is set implicitly through CR-pinning: the boot CPU sets it during identify_cpu(), it becomes part of cr4_pinned_bits, and cr4_init() applies those pinned bits to secondary CPUs. This works but creates an undocumented dependency between cr4_init() and the pinning mechanism. == Problem == Secondary CPUs boot after alternatives have been applied globally. They execute already-patched paranoid_entry() code that uses RDGSBASE/WRGSBASE instructions, which require CR4.FSGSBASE=1. Upcoming changes to CR pinning behavior will break the implicit dependency, causing secondary CPUs to generate #UD. This issue manifests itself on AMD SEV-SNP guests, where the rdmsrq() in x2apic_setup() triggers a #VC exception early during cpu_init(). The #VC handler (exc_vmm_communication()) executes the patched paranoid_entry() path. Without CR4.FSGSBASE enabled, RDGSBASE instructions trigger #UD. == Fix == Enable FSGSBASE explicitly in cpu_init_exception_handling() before loading exception handlers. This makes the dependency explicit and ensures both boot and secondary CPUs have FSGSBASE enabled before paranoid_entry() executes. Fixes: c82965f9e530 ("x86/entry/64: Handle FSGSBASE enabled paranoid entry/exit") Reported-by: Borislav Petkov Suggested-by: Sohil Mehta Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Sohil Mehta Cc: Link: https://patch.msgid.link/20260318075654.1792916-2-nikunj@amd.com --- arch/x86/kernel/cpu/common.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a8ff4376c286..7840b224d6a7 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2050,12 +2050,6 @@ static void identify_cpu(struct cpuinfo_x86 *c) setup_umip(c); setup_lass(c); - /* Enable FSGSBASE instructions if available. */ - if (cpu_has(c, X86_FEATURE_FSGSBASE)) { - cr4_set_bits(X86_CR4_FSGSBASE); - elf_hwcap2 |= HWCAP2_FSGSBASE; - } - /* * The vendor-specific functions might have changed features. * Now we do "generic changes." @@ -2416,6 +2410,18 @@ void cpu_init_exception_handling(bool boot_cpu) /* GHCB needs to be setup to handle #VC. */ setup_ghcb(); + /* + * On CPUs with FSGSBASE support, paranoid_entry() uses + * ALTERNATIVE-patched RDGSBASE/WRGSBASE instructions. Secondary CPUs + * boot after alternatives are patched globally, so early exceptions + * execute patched code that depends on FSGSBASE. Enable the feature + * before any exceptions occur. + */ + if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) { + cr4_set_bits(X86_CR4_FSGSBASE); + elf_hwcap2 |= HWCAP2_FSGSBASE; + } + if (cpu_feature_enabled(X86_FEATURE_FRED)) { /* The boot CPU has enabled FRED during early boot */ if (!boot_cpu) -- cgit v1.2.3 From 411df123c017169922cc767affce76282b8e6c85 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Thu, 19 Mar 2026 12:07:59 +0100 Subject: x86/cpu: Remove X86_CR4_FRED from the CR4 pinned bits mask Commit in Fixes added the FRED CR4 bit to the CR4 pinned bits mask so that whenever something else modifies CR4, that bit remains set. Which in itself is a perfectly fine idea. However, there's an issue when during boot FRED is initialized: first on the BSP and later on the APs. Thus, there's a window in time when exceptions cannot be handled. This becomes particularly nasty when running as SEV-{ES,SNP} or TDX guests which, when they manage to trigger exceptions during that short window described above, triple fault due to FRED MSRs not being set up yet. See Link tag below for a much more detailed explanation of the situation. So, as a result, the commit in that Link URL tried to address this shortcoming by temporarily disabling CR4 pinning when an AP is not online yet. However, that is a problem in itself because in this case, an attack on the kernel needs to only modify the online bit - a single bit in RW memory - and then disable CR4 pinning and then disable SM*P, leading to more and worse things to happen to the system. So, instead, remove the FRED bit from the CR4 pinning mask, thus obviating the need to temporarily disable CR4 pinning. If someone manages to disable FRED when poking at CR4, then idt_invalidate() would make sure the system would crash'n'burn on the first exception triggered, which is a much better outcome security-wise. Fixes: ff45746fbf00 ("x86/cpu: Add X86_CR4_FRED macro") Suggested-by: Dave Hansen Suggested-by: Peter Zijlstra Signed-off-by: Borislav Petkov (AMD) Cc: # 6.12+ Link: https://lore.kernel.org/r/177385987098.1647592.3381141860481415647.tip-bot2@tip-bot2 --- arch/x86/kernel/cpu/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 7840b224d6a7..c57e8972d30f 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -433,7 +433,7 @@ static __always_inline void setup_lass(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | - X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED; + X86_CR4_FSGSBASE | X86_CR4_CET; static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); static unsigned long cr4_pinned_bits __ro_after_init; -- cgit v1.2.3 From 3645eb7e3915990a149460c151a00894cb586253 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Wed, 18 Mar 2026 07:56:54 +0000 Subject: x86/fred: Fix early boot failures on SEV-ES/SNP guests FRED-enabled SEV-(ES,SNP) guests fail to boot due to the following issues in the early boot sequence: * FRED does not have a #VC exception handler in the dispatch logic * Early FRED #VC exceptions attempt to use uninitialized per-CPU GHCBs instead of boot_ghcb Add X86_TRAP_VC case to fred_hwexc() with a new exc_vmm_communication() function that provides the unified entry point FRED requires, dispatching to existing user/kernel handlers based on privilege level. The function is already declared via DECLARE_IDTENTRY_VC(). Fix early GHCB access by falling back to boot_ghcb in __sev_{get,put}_ghcb() when per-CPU GHCBs are not yet initialized. Fixes: 14619d912b65 ("x86/fred: FRED entry/exit and dispatch code") Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Cc: # 6.12+ Link: https://patch.msgid.link/20260318075654.1792916-4-nikunj@amd.com --- arch/x86/coco/sev/noinstr.c | 6 ++++++ arch/x86/entry/entry_fred.c | 14 ++++++++++++++ 2 files changed, 20 insertions(+) (limited to 'arch') diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c index 9d94aca4a698..5afd663a1c21 100644 --- a/arch/x86/coco/sev/noinstr.c +++ b/arch/x86/coco/sev/noinstr.c @@ -121,6 +121,9 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return boot_ghcb; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; @@ -164,6 +167,9 @@ noinstr void __sev_put_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); + if (!sev_cfg.ghcbs_initialized) + return; + data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index 88c757ac8ccd..fbe2d10dd737 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -177,6 +177,16 @@ static noinstr void fred_extint(struct pt_regs *regs) } } +#ifdef CONFIG_AMD_MEM_ENCRYPT +noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) +{ + if (user_mode(regs)) + return user_exc_vmm_communication(regs, error_code); + else + return kernel_exc_vmm_communication(regs, error_code); +} +#endif + static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) { /* Optimize for #PF. That's the only exception which matters performance wise */ @@ -207,6 +217,10 @@ static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) #ifdef CONFIG_X86_CET case X86_TRAP_CP: return exc_control_protection(regs, error_code); #endif +#ifdef CONFIG_AMD_MEM_ENCRYPT + case X86_TRAP_VC: return exc_vmm_communication(regs, error_code); +#endif + default: return fred_bad_type(regs, error_code); } -- cgit v1.2.3 From a3e93cac25316aad03bf561e3c205f4ca0b8f452 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 20 Mar 2026 10:25:21 +0100 Subject: x86/cpu: Add comment clarifying CRn pinning To avoid future confusion on the purpose and design of the CRn pinning code. Also note that if the attacker controls page-tables, the CRn bits lose much of the attraction anyway. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov (AMD) Link: https://patch.msgid.link/20260320092521.GG3739106@noisy.programming.kicks-ass.net --- arch/x86/kernel/cpu/common.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'arch') diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index c57e8972d30f..ec0670114efa 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -434,6 +434,19 @@ static __always_inline void setup_lass(struct cpuinfo_x86 *c) /* These bits should not change their value after CPU init is finished. */ static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE | X86_CR4_CET; + +/* + * The CR pinning protects against ROP on the 'mov %reg, %CRn' instruction(s). + * Since you can ROP directly to these instructions (barring shadow stack), + * any protection must follow immediately and unconditionally after that. + * + * Specifically, the CR[04] write functions below will have the value + * validation controlled by the @cr_pinning static_branch which is + * __ro_after_init, just like the cr4_pinned_bits value. + * + * Once set, an attacker will have to defeat page-tables to get around these + * restrictions. Which is a much bigger ask than 'simple' ROP. + */ static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning); static unsigned long cr4_pinned_bits __ro_after_init; -- cgit v1.2.3 From 3a28daa9b7d7c2ddf2c722e9e95d7e0928bf0cd1 Mon Sep 17 00:00:00 2001 From: Li Jun Date: Thu, 26 Mar 2026 14:29:08 +0800 Subject: LoongArch: Fix missing NULL checks for kstrdup() 1. Replace "of_find_node_by_path("/")" with "of_root" to avoid multiple calls to "of_node_put()". 2. Fix a potential kernel oops during early boot when memory allocation fails while parsing CPU model from device tree. Cc: stable@vger.kernel.org Signed-off-by: Li Jun Signed-off-by: Huacai Chen --- arch/loongarch/kernel/env.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 841206fde3ab..652456768b55 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -42,16 +42,15 @@ static int __init init_cpu_fullname(void) int cpu, ret; char *cpuname; const char *model; - struct device_node *root; /* Parsing cpuname from DTS model property */ - root = of_find_node_by_path("/"); - ret = of_property_read_string(root, "model", &model); + ret = of_property_read_string(of_root, "model", &model); if (ret == 0) { cpuname = kstrdup(model, GFP_KERNEL); + if (!cpuname) + return -ENOMEM; loongson_sysconf.cpuname = strsep(&cpuname, " "); } - of_node_put(root); if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { for (cpu = 0; cpu < NR_CPUS; cpu++) -- cgit v1.2.3 From 95db0c9f526d583634cddb2e5914718570fbac87 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 26 Mar 2026 14:29:09 +0800 Subject: LoongArch: Workaround LS2K/LS7A GPU DMA hang bug 1. Hardware limitation: GPU, DC and VPU are typically PCI device 06.0, 06.1 and 06.2. They share some hardware resources, so when configure the PCI 06.0 device BAR1, DMA memory access cannot be performed through this BAR, otherwise it will cause hardware abnormalities. 2. In typical scenarios of reboot or S3/S4, DC access to memory through BAR is not prohibited, resulting in GPU DMA hangs. 3. Workaround method: When configuring the 06.0 device BAR1, turn off the memory access of DC, GPU and VPU (via DC's CRTC registers). Cc: stable@vger.kernel.org Signed-off-by: Qianhai Wu Signed-off-by: Huacai Chen --- arch/loongarch/pci/pci.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) (limited to 'arch') diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index d923295ab8c6..d233ea2218fe 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -15,6 +17,9 @@ #define PCI_DEVICE_ID_LOONGSON_DC1 0x7a06 #define PCI_DEVICE_ID_LOONGSON_DC2 0x7a36 #define PCI_DEVICE_ID_LOONGSON_DC3 0x7a46 +#define PCI_DEVICE_ID_LOONGSON_GPU1 0x7a15 +#define PCI_DEVICE_ID_LOONGSON_GPU2 0x7a25 +#define PCI_DEVICE_ID_LOONGSON_GPU3 0x7a35 int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, int reg, int len, u32 *val) @@ -99,3 +104,78 @@ static void pci_fixup_vgadev(struct pci_dev *pdev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC1, pci_fixup_vgadev); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC2, pci_fixup_vgadev); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_DC3, pci_fixup_vgadev); + +#define CRTC_NUM_MAX 2 +#define CRTC_OUTPUT_ENABLE 0x100 + +static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on) +{ + u32 i, val, count, crtc_offset, device; + void __iomem *crtc_reg, *base, *regbase; + static u32 crtc_status[CRTC_NUM_MAX] = { 0 }; + + base = pdev->bus->ops->map_bus(pdev->bus, pdev->devfn + 1, 0); + device = readw(base + PCI_DEVICE_ID); + + regbase = ioremap(readq(base + PCI_BASE_ADDRESS_0) & ~0xffull, SZ_64K); + if (!regbase) { + pci_err(pdev, "Failed to ioremap()\n"); + return; + } + + switch (device) { + case PCI_DEVICE_ID_LOONGSON_DC2: + crtc_reg = regbase + 0x1240; + crtc_offset = 0x10; + break; + case PCI_DEVICE_ID_LOONGSON_DC3: + crtc_reg = regbase; + crtc_offset = 0x400; + break; + } + + for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) { + val = readl(crtc_reg); + + if (!on) + crtc_status[i] = val; + + /* No need to fixup if the status is off at startup. */ + if (!(crtc_status[i] & CRTC_OUTPUT_ENABLE)) + continue; + + if (on) + val |= CRTC_OUTPUT_ENABLE; + else + val &= ~CRTC_OUTPUT_ENABLE; + + mb(); + writel(val, crtc_reg); + + for (count = 0; count < 40; count++) { + val = readl(crtc_reg) & CRTC_OUTPUT_ENABLE; + if ((on && val) || (!on && !val)) + break; + udelay(1000); + } + + pci_info(pdev, "DMA hang fixup at reg[0x%lx]: 0x%x\n", + (unsigned long)crtc_reg & 0xffff, readl(crtc_reg)); + } + + iounmap(regbase); +} + +static void pci_fixup_dma_hang_early(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, false); +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_early); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_early); + +static void pci_fixup_dma_hang_final(struct pci_dev *pdev) +{ + loongson_gpu_fixup_dma_hang(pdev, true); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU2, pci_fixup_dma_hang_final); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, PCI_DEVICE_ID_LOONGSON_GPU3, pci_fixup_dma_hang_final); -- cgit v1.2.3 From e4878c37f6679fdea91b27a0f4e60a871f0b7bad Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Thu, 26 Mar 2026 14:29:09 +0800 Subject: LoongArch: vDSO: Emit GNU_EH_FRAME correctly With -fno-asynchronous-unwind-tables and --no-eh-frame-hdr (the default of the linker), the GNU_EH_FRAME segment (specified by vdso.lds.S) is empty. This is not valid, as the current DWARF specification mandates the first byte of the EH frame to be the version number 1. It causes some unwinders to complain, for example the ClickHouse query profiler spams the log with messages: clickhouse-server[365854]: libunwind: unsupported .eh_frame_hdr version: 127 at 7ffffffb0000 Here "127" is just the byte located at the p_vaddr (0, i.e. the beginning of the vDSO) of the empty GNU_EH_FRAME segment. Cross- checking with /proc/365854/maps has also proven 7ffffffb0000 is the start of vDSO in the process VM image. In LoongArch the -fno-asynchronous-unwind-tables option seems just a MIPS legacy, and MIPS only uses this option to satisfy the MIPS-specific "genvdso" program, per the commit cfd75c2db17e ("MIPS: VDSO: Explicitly use -fno-asynchronous-unwind-tables"). IIRC it indicates some inherent limitation of the MIPS ELF ABI and has nothing to do with LoongArch. So we can simply flip it over to -fasynchronous-unwind-tables and pass --eh-frame-hdr for linking the vDSO, allowing the profilers to unwind the stack for statistics even if the sample point is taken when the PC is in the vDSO. However simply adjusting the options above would exploit an issue: when the libgcc unwinder saw the invalid GNU_EH_FRAME segment, it silently falled back to a machine-specific routine to match the code pattern of rt_sigreturn() and extract the registers saved in the sigframe if the code pattern is matched. As unwinding from signal handlers is vital for libgcc to support pthread cancellation etc., the fall-back routine had been silently keeping the LoongArch Linux systems functioning since Linux 5.19. But when we start to emit GNU_EH_FRAME with the correct format, fall-back routine will no longer be used and libgcc will fail to unwind the sigframe, and unwinding from signal handlers will no longer work, causing dozens of glibc test failures. To make it possible to unwind from signal handlers again, it's necessary to code the unwind info in __vdso_rt_sigreturn via .cfi_* directives. The offsets in the .cfi_* directives depend on the layout of struct sigframe, notably the offset of sigcontext in the sigframe. To use the offset in the assembly file, factor out struct sigframe into a header to allow asm-offsets.c to output the offset for assembly. To work around a long-term issue in the libgcc unwinder (the pc is unconditionally substracted by 1: doing so is technically incorrect for a signal frame), a nop instruction is included with the two real instructions in __vdso_rt_sigreturn in the same FDE PC range. The same hack has been used on x86 for a long time. Cc: stable@vger.kernel.org Fixes: c6b99bed6b8f ("LoongArch: Add VDSO and VSYSCALL support") Signed-off-by: Xi Ruoyao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/linkage.h | 36 +++++++++++++++++++++++++++++++++++ arch/loongarch/include/asm/sigframe.h | 9 +++++++++ arch/loongarch/kernel/asm-offsets.c | 2 ++ arch/loongarch/kernel/signal.c | 6 +----- arch/loongarch/vdso/Makefile | 4 ++-- arch/loongarch/vdso/sigreturn.S | 6 +++--- 6 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 arch/loongarch/include/asm/sigframe.h (limited to 'arch') diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h index e2eca1a25b4e..a1bd6a3ee03a 100644 --- a/arch/loongarch/include/asm/linkage.h +++ b/arch/loongarch/include/asm/linkage.h @@ -41,4 +41,40 @@ .cfi_endproc; \ SYM_END(name, SYM_T_NONE) +/* + * This is for the signal handler trampoline, which is used as the return + * address of the signal handlers in userspace instead of called normally. + * The long standing libgcc bug https://gcc.gnu.org/PR124050 requires a + * nop between .cfi_startproc and the actual address of the trampoline, so + * we cannot simply use SYM_FUNC_START. + * + * This wrapper also contains all the .cfi_* directives for recovering + * the content of the GPRs and the "return address" (where the rt_sigreturn + * syscall will jump to), assuming there is a struct rt_sigframe (where + * a struct sigcontext containing those information we need to recover) at + * $sp. The "DWARF for the LoongArch(TM) Architecture" manual states + * column 0 is for $zero, but it does not make too much sense to + * save/restore the hardware zero register. Repurpose this column here + * for the return address (here it's not the content of $ra we cannot use + * the default column 3). + */ +#define SYM_SIGFUNC_START(name) \ + .cfi_startproc; \ + .cfi_signal_frame; \ + .cfi_def_cfa 3, RT_SIGFRAME_SC; \ + .cfi_return_column 0; \ + .cfi_offset 0, SC_PC; \ + \ + .irp num, 1, 2, 3, 4, 5, 6, 7, 8, \ + 9, 10, 11, 12, 13, 14, 15, 16, \ + 17, 18, 19, 20, 21, 22, 23, 24, \ + 25, 26, 27, 28, 29, 30, 31; \ + .cfi_offset \num, SC_REGS + \num * SZREG; \ + .endr; \ + \ + nop; \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) + +#define SYM_SIGFUNC_END(name) SYM_FUNC_END(name) + #endif diff --git a/arch/loongarch/include/asm/sigframe.h b/arch/loongarch/include/asm/sigframe.h new file mode 100644 index 000000000000..109298b8d7e0 --- /dev/null +++ b/arch/loongarch/include/asm/sigframe.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#include +#include + +struct rt_sigframe { + struct siginfo rs_info; + struct ucontext rs_uctx; +}; diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 3017c7157600..2cc953f113ac 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static void __used output_ptreg_defines(void) @@ -220,6 +221,7 @@ static void __used output_sc_defines(void) COMMENT("Linux sigcontext offsets."); OFFSET(SC_REGS, sigcontext, sc_regs); OFFSET(SC_PC, sigcontext, sc_pc); + OFFSET(RT_SIGFRAME_SC, rt_sigframe, rs_uctx.uc_mcontext); BLANK(); } diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index c9f7ca778364..d4151d2fb82e 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -51,11 +52,6 @@ #define lock_lbt_owner() ({ preempt_disable(); pagefault_disable(); }) #define unlock_lbt_owner() ({ pagefault_enable(); preempt_enable(); }) -struct rt_sigframe { - struct siginfo rs_info; - struct ucontext rs_uctx; -}; - struct _ctx_layout { struct sctx_info *addr; unsigned int size; diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index 520f1513f07d..294c16b9517f 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -26,7 +26,7 @@ cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ -std=gnu11 -fms-extensions -O2 -g -fno-strict-aliasing -fno-common -fno-builtin \ -fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \ - $(call cc-option, -fno-asynchronous-unwind-tables) \ + $(call cc-option, -fasynchronous-unwind-tables) \ $(call cc-option, -fno-stack-protector) aflags-vdso := $(ccflags-vdso) \ -D__ASSEMBLY__ -Wa,-gdwarf-2 @@ -41,7 +41,7 @@ endif # VDSO linker flags. ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T + $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id --eh-frame-hdr -T # # Shared build commands. diff --git a/arch/loongarch/vdso/sigreturn.S b/arch/loongarch/vdso/sigreturn.S index 9cb3c58fad03..59f940d928de 100644 --- a/arch/loongarch/vdso/sigreturn.S +++ b/arch/loongarch/vdso/sigreturn.S @@ -12,13 +12,13 @@ #include #include +#include .section .text - .cfi_sections .debug_frame -SYM_FUNC_START(__vdso_rt_sigreturn) +SYM_SIGFUNC_START(__vdso_rt_sigreturn) li.w a7, __NR_rt_sigreturn syscall 0 -SYM_FUNC_END(__vdso_rt_sigreturn) +SYM_SIGFUNC_END(__vdso_rt_sigreturn) -- cgit v1.2.3 From 2db06c15d8c7a0ccb6108524e16cd9163753f354 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 26 Mar 2026 14:29:09 +0800 Subject: LoongArch: KVM: Make kvm_get_vcpu_by_cpuid() more robust kvm_get_vcpu_by_cpuid() takes a cpuid parameter whose type is int, so cpuid can be negative. Let kvm_get_vcpu_by_cpuid() return NULL for this case so as to make it more robust. This fix an out-of-bounds access to kvm_arch::phyid_map::phys_map[]. Cc: Fixes: 73516e9da512adc ("LoongArch: KVM: Add vcpu mapping from physical cpuid") Reported-by: Aurelien Jarno Link: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1131431 Signed-off-by: Huacai Chen --- arch/loongarch/kvm/vcpu.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch') diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 8ffd50a470e6..831f381a8fd1 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -588,6 +588,9 @@ struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) { struct kvm_phyid_map *map; + if (cpuid < 0) + return NULL; + if (cpuid >= KVM_MAX_PHYID) return NULL; -- cgit v1.2.3 From b97bd69eb0f67b5f961b304d28e9ba45e202d841 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 26 Mar 2026 14:29:09 +0800 Subject: LoongArch: KVM: Handle the case that EIOINTC's coremap is empty EIOINTC's coremap in eiointc_update_sw_coremap() can be empty, currently we get a cpuid with -1 in this case, but we actually need 0 because it's similar as the case that cpuid >= 4. This fix an out-of-bounds access to kvm_arch::phyid_map::phys_map[]. Cc: Fixes: 3956a52bc05bd81 ("LoongArch: KVM: Add EIOINTC read and write functions") Reported-by: Aurelien Jarno Link: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1131431 Signed-off-by: Huacai Chen --- arch/loongarch/kvm/intc/eiointc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index d2acb4d09e73..c7badc813923 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -83,7 +83,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, if (!(s->status & BIT(EIOINTC_ENABLE_CPU_ENCODE))) { cpuid = ffs(cpuid) - 1; - cpuid = (cpuid >= 4) ? 0 : cpuid; + cpuid = ((cpuid < 0) || (cpuid >= 4)) ? 0 : cpuid; } vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid); -- cgit v1.2.3 From 6bcfb7f46d667b04bd1a1169ccedf5fb699c60df Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Thu, 26 Mar 2026 14:29:09 +0800 Subject: LoongArch: KVM: Fix base address calculation in kvm_eiointc_regs_access() In function kvm_eiointc_regs_access(), the register base address is caculated from array base address plus offset, the offset is absolute value from the base address. The data type of array base address is u64, it should be converted into the "void *" type and then plus the offset. Cc: Fixes: d3e43a1f34ac ("LoongArch: KVM: Use 64-bit register definition for EIOINTC"). Reported-by: Aurelien Jarno Link: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1131431 Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kvm/intc/eiointc.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'arch') diff --git a/arch/loongarch/kvm/intc/eiointc.c b/arch/loongarch/kvm/intc/eiointc.c index c7badc813923..003bd773e11c 100644 --- a/arch/loongarch/kvm/intc/eiointc.c +++ b/arch/loongarch/kvm/intc/eiointc.c @@ -472,34 +472,34 @@ static int kvm_eiointc_regs_access(struct kvm_device *dev, switch (addr) { case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END: offset = (addr - EIOINTC_NODETYPE_START) / 4; - p = s->nodetype + offset * 4; + p = (void *)s->nodetype + offset * 4; break; case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END: offset = (addr - EIOINTC_IPMAP_START) / 4; - p = &s->ipmap + offset * 4; + p = (void *)&s->ipmap + offset * 4; break; case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END: offset = (addr - EIOINTC_ENABLE_START) / 4; - p = s->enable + offset * 4; + p = (void *)s->enable + offset * 4; break; case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END: offset = (addr - EIOINTC_BOUNCE_START) / 4; - p = s->bounce + offset * 4; + p = (void *)s->bounce + offset * 4; break; case EIOINTC_ISR_START ... EIOINTC_ISR_END: offset = (addr - EIOINTC_ISR_START) / 4; - p = s->isr + offset * 4; + p = (void *)s->isr + offset * 4; break; case EIOINTC_COREISR_START ... EIOINTC_COREISR_END: if (cpu >= s->num_cpu) return -EINVAL; offset = (addr - EIOINTC_COREISR_START) / 4; - p = s->coreisr[cpu] + offset * 4; + p = (void *)s->coreisr[cpu] + offset * 4; break; case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END: offset = (addr - EIOINTC_COREMAP_START) / 4; - p = s->coremap + offset * 4; + p = (void *)s->coremap + offset * 4; break; default: kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr); -- cgit v1.2.3 From 0f54755343f56ac108cfd55173bc1b5c5376384d Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:10 +0100 Subject: KVM: s390: vsie: Fix dat_split_ste() If the guest misbehaves and puts the page tables for its nested guest inside the memory of the nested guest itself, and the guest and nested guest are being mapped with large pages, the shadow mapping will lose synchronization with the actual mapping, since this will cause the large page with the vsie notification bit to be split, but the vsie notification bit will not be propagated to the resulting small pages. Fix this by propagating the vsie_notif bit from large pages to normal pages when splitting a large page. Fixes: 2db149a0a6c5 ("KVM: s390: KVM page table management functions: walks") Reviewed-by: Christoph Schlameuss Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/dat.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch') diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 670404d4fa44..48b5f2bcf172 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -292,6 +292,7 @@ static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t g pt->ptes[i].val = init.val | i * PAGE_SIZE; /* No need to take locks as the page table is not installed yet. */ pgste_init.prefix_notif = old.s.fc1.prefix_notif; + pgste_init.vsie_notif = old.s.fc1.vsie_notif; pgste_init.pcl = uses_skeys && init.h.i; dat_init_pgstes(pt, pgste_init.val); } else { -- cgit v1.2.3 From b827ef02f409bd42c7e7fb82663b84753c0e5d14 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:11 +0100 Subject: KVM: s390: Remove non-atomic dat_crstep_xchg() In practice dat_crstep_xchg() is racy and hard to use correctly. Simply remove it and replace its uses with dat_crstep_xchg_atomic(). This solves some actual races that lead to system hangs / crashes. Opportunistically fix an alignment issue in _gmap_crstep_xchg_atomic(). Fixes: 589071eaaa8f ("KVM: s390: KVM page table management functions: clear and replace") Fixes: 94fd9b16cc67 ("KVM: s390: KVM page table management functions: lifecycle management") Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/dat.c | 51 +++++++++--------------------- arch/s390/kvm/dat.h | 9 ++++-- arch/s390/kvm/gaccess.c | 26 +++++++++------- arch/s390/kvm/gmap.c | 82 +++++++++++++++++++++++++++++-------------------- arch/s390/kvm/gmap.h | 29 +++++++++-------- 5 files changed, 100 insertions(+), 97 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 48b5f2bcf172..4d44c0f9ad45 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -134,32 +134,6 @@ int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newt return 0; } -/** - * dat_crstep_xchg() - Exchange a gmap CRSTE with another. - * @crstep: Pointer to the CRST entry - * @new: Replacement entry. - * @gfn: The affected guest address. - * @asce: The ASCE of the address space. - * - * Context: This function is assumed to be called with kvm->mmu_lock held. - */ -void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce) -{ - if (crstep->h.i) { - WRITE_ONCE(*crstep, new); - return; - } else if (cpu_has_edat2()) { - crdte_crste(crstep, *crstep, new, gfn, asce); - return; - } - - if (machine_has_tlb_guest()) - idte_crste(crstep, gfn, IDTE_GUEST_ASCE, asce, IDTE_GLOBAL); - else - idte_crste(crstep, gfn, 0, NULL_ASCE, IDTE_GLOBAL); - WRITE_ONCE(*crstep, new); -} - /** * dat_crstep_xchg_atomic() - Atomically exchange a gmap CRSTE with another. * @crstep: Pointer to the CRST entry. @@ -175,8 +149,8 @@ void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce * * Return: %true if the exchange was successful. */ -bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, - union asce asce) +bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, + gfn_t gfn, union asce asce) { if (old.h.i) return arch_try_cmpxchg((long *)crstep, &old.val, new.val); @@ -894,7 +868,8 @@ static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct d /* This table entry needs to be updated. */ if (walk->start <= gfn && walk->end >= next) { - dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce); + if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce)) + return -EINVAL; /* A lower level table was present, needs to be freed. */ if (!crste.h.fc && !crste.h.i) { if (is_pmd(crste)) @@ -1072,17 +1047,19 @@ int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { - union crste crste = READ_ONCE(*crstep); + union crste newcrste, oldcrste; int *n = walk->priv; - if (!crste.h.fc || crste.h.i || crste.h.p) - return 0; - + do { + oldcrste = READ_ONCE(*crstep); + if (!oldcrste.h.fc || oldcrste.h.i || oldcrste.h.p) + return 0; + if (oldcrste.s.fc1.prefix_notif) + break; + newcrste = oldcrste; + newcrste.s.fc1.prefix_notif = 1; + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, walk->asce)); *n = 2; - if (crste.s.fc1.prefix_notif) - return 0; - crste.s.fc1.prefix_notif = 1; - dat_crstep_xchg(crstep, crste, gfn, walk->asce); return 0; } diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 123e11dcd70d..22dafc775335 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -938,11 +938,14 @@ static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pu return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); } -static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce) +static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce) { - union crste newcrste = _CRSTE_EMPTY(crstep->h.tt); + union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt); - dat_crstep_xchg(crstep, newcrste, gfn, asce); + do { + oldcrste = READ_ONCE(*crstep); + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce)); + return oldcrste; } static inline int get_level(union crste *crstep, union pte *ptep) diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index a9da9390867d..4ee862424ca0 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1456,7 +1456,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table, struct guest_fault *f, bool p) { - union crste newcrste; + union crste newcrste, oldcrste; gfn_t gfn; int rc; @@ -1469,16 +1469,20 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni if (rc) return rc; - newcrste = _crste_fc1(f->pfn, host->h.tt, f->writable, !p); - newcrste.s.fc1.d |= host->s.fc1.d; - newcrste.s.fc1.sd |= host->s.fc1.sd; - newcrste.h.p &= host->h.p; - newcrste.s.fc1.vsie_notif = 1; - newcrste.s.fc1.prefix_notif = host->s.fc1.prefix_notif; - _gmap_crstep_xchg(sg->parent, host, newcrste, f->gfn, false); - - newcrste = _crste_fc1(f->pfn, host->h.tt, 0, !p); - dat_crstep_xchg(table, newcrste, gpa_to_gfn(raddr), sg->asce); + do { + oldcrste = READ_ONCE(*host); + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, f->writable, !p); + newcrste.s.fc1.d |= oldcrste.s.fc1.d; + newcrste.s.fc1.sd |= oldcrste.s.fc1.sd; + newcrste.h.p &= oldcrste.h.p; + newcrste.s.fc1.vsie_notif = 1; + newcrste.s.fc1.prefix_notif = oldcrste.s.fc1.prefix_notif; + } while (!_gmap_crstep_xchg_atomic(sg->parent, host, oldcrste, newcrste, f->gfn, false)); + + newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, 0, !p); + gfn = gpa_to_gfn(raddr); + while (!dat_crstep_xchg_atomic(table, READ_ONCE(*table), newcrste, gfn, sg->asce)) + ; return 0; } diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index ef0c6ebfdde2..956be4c01797 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -313,13 +313,16 @@ static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, st struct clear_young_pte_priv *priv = walk->priv; union crste crste, new; - crste = READ_ONCE(*crstep); + do { + crste = READ_ONCE(*crstep); + + if (!crste.h.fc) + return 0; + if (!crste.s.fc1.y && crste.h.i) + return 0; + if (crste_prefix(crste) && !gmap_mkold_prefix(priv->gmap, gfn, end)) + break; - if (!crste.h.fc) - return 0; - if (!crste.s.fc1.y && crste.h.i) - return 0; - if (!crste_prefix(crste) || gmap_mkold_prefix(priv->gmap, gfn, end)) { new = crste; new.h.i = 1; new.s.fc1.y = 0; @@ -328,8 +331,8 @@ static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, st folio_set_dirty(phys_to_folio(crste_origin_large(crste))); new.s.fc1.d = 0; new.h.p = 1; - dat_crstep_xchg(crstep, new, gfn, walk->asce); - } + } while (!dat_crstep_xchg_atomic(crstep, crste, new, gfn, walk->asce)); + priv->young = 1; return 0; } @@ -391,14 +394,18 @@ static long _gmap_unmap_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct { struct gmap_unmap_priv *priv = walk->priv; struct folio *folio = NULL; + union crste old = *crstep; - if (crstep->h.fc) { - if (crstep->s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) - folio = phys_to_folio(crste_origin_large(*crstep)); - gmap_crstep_xchg(priv->gmap, crstep, _CRSTE_EMPTY(crstep->h.tt), gfn); - if (folio) - uv_convert_from_secure_folio(folio); - } + if (!old.h.fc) + return 0; + + if (old.s.fc1.pr && test_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &priv->gmap->flags)) + folio = phys_to_folio(crste_origin_large(old)); + /* No races should happen because kvm->mmu_lock is held in write mode */ + KVM_BUG_ON(!gmap_crstep_xchg_atomic(priv->gmap, crstep, old, _CRSTE_EMPTY(old.h.tt), gfn), + priv->gmap->kvm); + if (folio) + uv_convert_from_secure_folio(folio); return 0; } @@ -474,23 +481,24 @@ static long _crste_test_and_clear_softdirty(union crste *table, gfn_t gfn, gfn_t if (fatal_signal_pending(current)) return 1; - crste = READ_ONCE(*table); - if (!crste.h.fc) - return 0; - if (crste.h.p && !crste.s.fc1.sd) - return 0; + do { + crste = READ_ONCE(*table); + if (!crste.h.fc) + return 0; + if (crste.h.p && !crste.s.fc1.sd) + return 0; - /* - * If this large page contains one or more prefixes of vCPUs that are - * currently running, do not reset the protection, leave it marked as - * dirty. - */ - if (!crste.s.fc1.prefix_notif || gmap_mkold_prefix(gmap, gfn, end)) { + /* + * If this large page contains one or more prefixes of vCPUs that are + * currently running, do not reset the protection, leave it marked as + * dirty. + */ + if (crste.s.fc1.prefix_notif && !gmap_mkold_prefix(gmap, gfn, end)) + break; new = crste; new.h.p = 1; new.s.fc1.sd = 0; - gmap_crstep_xchg(gmap, table, new, gfn); - } + } while (!gmap_crstep_xchg_atomic(gmap, table, crste, new, gfn)); for ( ; gfn < end; gfn++) mark_page_dirty(gmap->kvm, gfn); @@ -646,8 +654,8 @@ int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fau static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, bool force_alloc) { + union crste newcrste, oldcrste; struct page_table *pt; - union crste newcrste; union crste *crstep; union pte *ptep; int rc; @@ -673,7 +681,11 @@ static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, &crstep, &ptep); if (rc) return rc; - dat_crstep_xchg(crstep, newcrste, c_gfn, gmap->asce); + do { + oldcrste = READ_ONCE(*crstep); + if (oldcrste.val == newcrste.val) + break; + } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, c_gfn, gmap->asce)); return 0; } @@ -777,8 +789,10 @@ static void gmap_ucas_unmap_one(struct gmap *gmap, gfn_t c_gfn) int rc; rc = dat_entry_walk(NULL, c_gfn, gmap->asce, 0, TABLE_TYPE_SEGMENT, &crstep, &ptep); - if (!rc) - dat_crstep_xchg(crstep, _PMD_EMPTY, c_gfn, gmap->asce); + if (rc) + return; + while (!dat_crstep_xchg_atomic(crstep, READ_ONCE(*crstep), _PMD_EMPTY, c_gfn, gmap->asce)) + ; } void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count) @@ -1017,8 +1031,8 @@ static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level) dat_ptep_xchg(ptep, _PTE_EMPTY, r_gfn, sg->asce, uses_skeys(sg)); return; } - crste = READ_ONCE(*crstep); - dat_crstep_clear(crstep, r_gfn, sg->asce); + + crste = dat_crstep_clear_atomic(crstep, r_gfn, sg->asce); if (crste_leaf(crste) || crste.h.i) return; if (is_pmd(crste)) diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h index ccb5cd751e31..150e91e15ee0 100644 --- a/arch/s390/kvm/gmap.h +++ b/arch/s390/kvm/gmap.h @@ -194,35 +194,40 @@ static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, uni return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true); } -static inline void _gmap_crstep_xchg(struct gmap *gmap, union crste *crstep, union crste ne, - gfn_t gfn, bool needs_lock) +static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, + union crste oldcrste, union crste newcrste, + gfn_t gfn, bool needs_lock) { - unsigned long align = 8 + (is_pmd(*crstep) ? 0 : 11); + unsigned long align = is_pmd(newcrste) ? _PAGE_ENTRIES : _PAGE_ENTRIES * _CRST_ENTRIES; + + if (KVM_BUG_ON(crstep->h.tt != oldcrste.h.tt || newcrste.h.tt != oldcrste.h.tt, gmap->kvm)) + return true; lockdep_assert_held(&gmap->kvm->mmu_lock); if (!needs_lock) lockdep_assert_held(&gmap->children_lock); gfn = ALIGN_DOWN(gfn, align); - if (crste_prefix(*crstep) && (ne.h.p || ne.h.i || !crste_prefix(ne))) { - ne.s.fc1.prefix_notif = 0; + if (crste_prefix(oldcrste) && (newcrste.h.p || newcrste.h.i || !crste_prefix(newcrste))) { + newcrste.s.fc1.prefix_notif = 0; gmap_unmap_prefix(gmap, gfn, gfn + align); } - if (crste_leaf(*crstep) && crstep->s.fc1.vsie_notif && - (ne.h.p || ne.h.i || !ne.s.fc1.vsie_notif)) { - ne.s.fc1.vsie_notif = 0; + if (crste_leaf(oldcrste) && oldcrste.s.fc1.vsie_notif && + (newcrste.h.p || newcrste.h.i || !newcrste.s.fc1.vsie_notif)) { + newcrste.s.fc1.vsie_notif = 0; if (needs_lock) gmap_handle_vsie_unshadow_event(gmap, gfn); else _gmap_handle_vsie_unshadow_event(gmap, gfn); } - dat_crstep_xchg(crstep, ne, gfn, gmap->asce); + return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); } -static inline void gmap_crstep_xchg(struct gmap *gmap, union crste *crstep, union crste ne, - gfn_t gfn) +static inline bool __must_check gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep, + union crste oldcrste, union crste newcrste, + gfn_t gfn) { - return _gmap_crstep_xchg(gmap, crstep, ne, gfn, true); + return _gmap_crstep_xchg_atomic(gmap, crstep, oldcrste, newcrste, gfn, true); } /** -- cgit v1.2.3 From 6f93d1ed6f46b7b0be288cc45250d67bceb28982 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:12 +0100 Subject: KVM: s390: vsie: Fix check for pre-existing shadow mapping When shadowing a nested guest, a check is performed and no shadowing is attempted if the nested guest is already shadowed. The existing check was incomplete; fix it by also checking whether the leaf DAT table entry in the existing shadow gmap has the same protection as the one specified in the guest DAT entry. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/gaccess.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 4ee862424ca0..8fd690255e1b 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1506,8 +1506,9 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, if (rc) return rc; - /* A race occourred. The shadow mapping is already valid, nothing to do */ - if ((ptep && !ptep->h.i) || (!ptep && crste_leaf(*table))) + /* A race occurred. The shadow mapping is already valid, nothing to do */ + if ((ptep && !ptep->h.i && ptep->h.p == w->p) || + (!ptep && crste_leaf(*table) && !table->h.i && table->h.p == w->p)) return 0; gl = get_level(table, ptep); -- cgit v1.2.3 From 45921d0212d4a335680854c89a14efd01eae911a Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:13 +0100 Subject: KVM: s390: Fix gmap_link() The slow path of the fault handler ultimately called gmap_link(), which assumed the fault was a major fault, and blindly called dat_link(). In case of minor faults, things were not always handled properly; in particular the prefix and vsie marker bits were ignored. Move dat_link() into gmap.c, renaming it accordingly. Once moved, the new _gmap_link() function will be able to correctly honour the prefix and vsie markers. This will cause spurious unshadows in some uncommon cases. Fixes: 94fd9b16cc67 ("KVM: s390: KVM page table management functions: lifecycle management") Fixes: a2c17f9270cc ("KVM: s390: New gmap code") Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/dat.c | 48 -------------------------------------------- arch/s390/kvm/dat.h | 2 -- arch/s390/kvm/gmap.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 52 insertions(+), 54 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 4d44c0f9ad45..7b8d70fe406d 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -997,54 +997,6 @@ bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; } -int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, - bool uses_skeys, struct guest_fault *f) -{ - union crste oldval, newval; - union pte newpte, oldpte; - union pgste pgste; - int rc = 0; - - rc = dat_entry_walk(mc, f->gfn, asce, DAT_WALK_ALLOC_CONTINUE, level, &f->crstep, &f->ptep); - if (rc == -EINVAL || rc == -ENOMEM) - return rc; - if (rc) - return -EAGAIN; - - if (WARN_ON_ONCE(unlikely(get_level(f->crstep, f->ptep) > level))) - return -EINVAL; - - if (f->ptep) { - pgste = pgste_get_lock(f->ptep); - oldpte = *f->ptep; - newpte = _pte(f->pfn, f->writable, f->write_attempt | oldpte.s.d, !f->page); - newpte.s.sd = oldpte.s.sd; - oldpte.s.sd = 0; - if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == f->pfn) { - pgste = __dat_ptep_xchg(f->ptep, pgste, newpte, f->gfn, asce, uses_skeys); - if (f->callback) - f->callback(f); - } else { - rc = -EAGAIN; - } - pgste_set_unlock(f->ptep, pgste); - } else { - oldval = READ_ONCE(*f->crstep); - newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, - f->write_attempt | oldval.s.fc1.d); - newval.s.fc1.sd = oldval.s.fc1.sd; - if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && - crste_origin_large(oldval) != crste_origin_large(newval)) - return -EAGAIN; - if (!dat_crstep_xchg_atomic(f->crstep, oldval, newval, f->gfn, asce)) - return -EAGAIN; - if (f->callback) - f->callback(f); - } - - return rc; -} - static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union crste newcrste, oldcrste; diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 22dafc775335..efedcf96110c 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -540,8 +540,6 @@ int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gf u16 type, u16 param); int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); -int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, - bool uses_skeys, struct guest_fault *f); int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); long dat_reset_cmma(union asce asce, gfn_t start_gfn); diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 956be4c01797..03e15b5e0b9a 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -631,10 +631,60 @@ static inline bool gmap_1m_allowed(struct gmap *gmap, gfn_t gfn) return test_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &gmap->flags); } +static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int level, + struct guest_fault *f) +{ + union crste oldval, newval; + union pte newpte, oldpte; + union pgste pgste; + int rc = 0; + + rc = dat_entry_walk(mc, f->gfn, gmap->asce, DAT_WALK_ALLOC_CONTINUE, level, + &f->crstep, &f->ptep); + if (rc == -ENOMEM) + return rc; + if (KVM_BUG_ON(rc == -EINVAL, gmap->kvm)) + return rc; + if (rc) + return -EAGAIN; + if (KVM_BUG_ON(get_level(f->crstep, f->ptep) > level, gmap->kvm)) + return -EINVAL; + + if (f->ptep) { + pgste = pgste_get_lock(f->ptep); + oldpte = *f->ptep; + newpte = _pte(f->pfn, f->writable, f->write_attempt | oldpte.s.d, !f->page); + newpte.s.sd = oldpte.s.sd; + oldpte.s.sd = 0; + if (oldpte.val == _PTE_EMPTY.val || oldpte.h.pfra == f->pfn) { + pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, pgste, f->gfn); + if (f->callback) + f->callback(f); + } else { + rc = -EAGAIN; + } + pgste_set_unlock(f->ptep, pgste); + } else { + do { + oldval = READ_ONCE(*f->crstep); + newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, + f->write_attempt | oldval.s.fc1.d); + newval.s.fc1.sd = oldval.s.fc1.sd; + if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && + crste_origin_large(oldval) != crste_origin_large(newval)) + return -EAGAIN; + } while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn)); + if (f->callback) + f->callback(f); + } + + return rc; +} + int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fault *f) { unsigned int order; - int rc, level; + int level; lockdep_assert_held(&gmap->kvm->mmu_lock); @@ -646,9 +696,7 @@ int gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, struct guest_fau else if (order >= get_order(_SEGMENT_SIZE) && gmap_1m_allowed(gmap, f->gfn)) level = TABLE_TYPE_SEGMENT; } - rc = dat_link(mc, gmap->asce, level, uses_skeys(gmap), f); - KVM_BUG_ON(rc == -EINVAL, gmap->kvm); - return rc; + return _gmap_link(mc, gmap, level, f); } static int gmap_ucas_map_one(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, -- cgit v1.2.3 From 0f2b760a17126cb7940d410c99edfa14e928554c Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:14 +0100 Subject: KVM: s390: Correctly handle guest mappings without struct page Introduce a new special softbit for large pages, like already presend for normal pages, and use it to mark guest mappings that do not have struct pages. Whenever a leaf DAT entry becomes dirty, check the special softbit and only call SetPageDirty() if there is an actual struct page. Move the logic to mark pages dirty inside _gmap_ptep_xchg() and _gmap_crstep_xchg_atomic(), to avoid needlessly duplicating the code. Fixes: 5a74e3d93417 ("KVM: s390: KVM-specific bitfields and helper functions") Fixes: a2c17f9270cc ("KVM: s390: New gmap code") Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/dat.h | 12 ++++++------ arch/s390/kvm/gmap.c | 11 ++++------- arch/s390/kvm/gmap.h | 4 ++++ 3 files changed, 14 insertions(+), 13 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index efedcf96110c..874cc962e196 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -160,14 +160,14 @@ union pmd { unsigned long :44; /* HW */ unsigned long : 3; /* Unused */ unsigned long : 1; /* HW */ + unsigned long s : 1; /* Special */ unsigned long w : 1; /* Writable soft-bit */ unsigned long r : 1; /* Readable soft-bit */ unsigned long d : 1; /* Dirty */ unsigned long y : 1; /* Young */ - unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long vsie_notif : 1; /* Referenced in a shadow table */ - unsigned long : 1; /* Unused */ unsigned long : 4; /* HW */ unsigned long sd : 1; /* Soft-Dirty */ unsigned long pr : 1; /* Present */ @@ -183,14 +183,14 @@ union pud { unsigned long :33; /* HW */ unsigned long :14; /* Unused */ unsigned long : 1; /* HW */ + unsigned long s : 1; /* Special */ unsigned long w : 1; /* Writable soft-bit */ unsigned long r : 1; /* Readable soft-bit */ unsigned long d : 1; /* Dirty */ unsigned long y : 1; /* Young */ - unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long vsie_notif : 1; /* Referenced in a shadow table */ - unsigned long : 1; /* Unused */ unsigned long : 4; /* HW */ unsigned long sd : 1; /* Soft-Dirty */ unsigned long pr : 1; /* Present */ @@ -254,14 +254,14 @@ union crste { struct { unsigned long :47; unsigned long : 1; /* HW (should be 0) */ + unsigned long s : 1; /* Special */ unsigned long w : 1; /* Writable */ unsigned long r : 1; /* Readable */ unsigned long d : 1; /* Dirty */ unsigned long y : 1; /* Young */ - unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long : 3; /* HW */ + unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ unsigned long vsie_notif : 1; /* Referenced in a shadow table */ - unsigned long : 1; unsigned long : 4; /* HW */ unsigned long sd : 1; /* Soft-Dirty */ unsigned long pr : 1; /* Present */ diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 03e15b5e0b9a..c8b79ad04ac9 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -519,7 +519,7 @@ void gmap_sync_dirty_log(struct gmap *gmap, gfn_t start, gfn_t end) _dat_walk_gfn_range(start, end, gmap->asce, &walk_ops, 0, gmap); } -static int gmap_handle_minor_crste_fault(union asce asce, struct guest_fault *f) +static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault *f) { union crste newcrste, oldcrste = READ_ONCE(*f->crstep); @@ -544,10 +544,8 @@ static int gmap_handle_minor_crste_fault(union asce asce, struct guest_fault *f) newcrste.s.fc1.d = 1; newcrste.s.fc1.sd = 1; } - if (!oldcrste.s.fc1.d && newcrste.s.fc1.d) - SetPageDirty(phys_to_page(crste_origin_large(newcrste))); /* In case of races, let the slow path deal with it. */ - return !dat_crstep_xchg_atomic(f->crstep, oldcrste, newcrste, f->gfn, asce); + return !gmap_crstep_xchg_atomic(gmap, f->crstep, oldcrste, newcrste, f->gfn); } /* Trying to write on a read-only page, let the slow path deal with it. */ return 1; @@ -576,8 +574,6 @@ static int _gmap_handle_minor_pte_fault(struct gmap *gmap, union pgste *pgste, newpte.s.d = 1; newpte.s.sd = 1; } - if (!oldpte.s.d && newpte.s.d) - SetPageDirty(pfn_to_page(newpte.h.pfra)); *pgste = gmap_ptep_xchg(gmap, f->ptep, newpte, *pgste, f->gfn); return 0; @@ -614,7 +610,7 @@ int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault) fault->callback(fault); pgste_set_unlock(fault->ptep, pgste); } else { - rc = gmap_handle_minor_crste_fault(gmap->asce, fault); + rc = gmap_handle_minor_crste_fault(gmap, fault); if (!rc && fault->callback) fault->callback(fault); } @@ -669,6 +665,7 @@ static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int leve oldval = READ_ONCE(*f->crstep); newval = _crste_fc1(f->pfn, oldval.h.tt, f->writable, f->write_attempt | oldval.s.fc1.d); + newval.s.fc1.s = !f->page; newval.s.fc1.sd = oldval.s.fc1.sd; if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && crste_origin_large(oldval) != crste_origin_large(newval)) diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h index 150e91e15ee0..579399ef5480 100644 --- a/arch/s390/kvm/gmap.h +++ b/arch/s390/kvm/gmap.h @@ -185,6 +185,8 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un else _gmap_handle_vsie_unshadow_event(gmap, gfn); } + if (!ptep->s.d && newpte.s.d && !newpte.s.s) + SetPageDirty(pfn_to_page(newpte.h.pfra)); return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap)); } @@ -220,6 +222,8 @@ static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, unio else _gmap_handle_vsie_unshadow_event(gmap, gfn); } + if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s) + SetPageDirty(phys_to_page(crste_origin_large(newcrste))); return dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce); } -- cgit v1.2.3 From fd7bc612cf27f7c98764e86e2fba3511610fff20 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:15 +0100 Subject: KVM: s390: vsie: Fix nested guest memory shadowing Fix _do_shadow_pte() to use the correct pointer (guest pte instead of nested guest) to set up the new pte. Add a check to return -EOPNOTSUPP if the mapping for the nested guest is writeable but the same page in the guest is only read-only. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/gaccess.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 8fd690255e1b..6bc30f678921 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1436,13 +1436,19 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union if (!pgste_get_trylock(ptep_h, &pgste)) return -EAGAIN; - newpte = _pte(f->pfn, f->writable, !p, 0); - newpte.s.d |= ptep->s.d; - newpte.s.sd |= ptep->s.sd; - newpte.h.p &= ptep->h.p; - pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false); - pgste.vsie_notif = 1; + newpte = _pte(f->pfn, f->writable, !p, ptep_h->s.s); + newpte.s.d |= ptep_h->s.d; + newpte.s.sd |= ptep_h->s.sd; + newpte.h.p &= ptep_h->h.p; + if (!newpte.h.p && !f->writable) { + rc = -EOPNOTSUPP; + } else { + pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false); + pgste.vsie_notif = 1; + } pgste_set_unlock(ptep_h, pgste); + if (rc) + return rc; newpte = _pte(f->pfn, 0, !p, 0); if (!pgste_get_trylock(ptep, &pgste)) @@ -1477,6 +1483,9 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni newcrste.h.p &= oldcrste.h.p; newcrste.s.fc1.vsie_notif = 1; newcrste.s.fc1.prefix_notif = oldcrste.s.fc1.prefix_notif; + newcrste.s.fc1.s = oldcrste.s.fc1.s; + if (!newcrste.h.p && !f->writable) + return -EOPNOTSUPP; } while (!_gmap_crstep_xchg_atomic(sg->parent, host, oldcrste, newcrste, f->gfn, false)); newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, 0, !p); -- cgit v1.2.3 From 0ec456b8a53d78644af7363a225c182494c1dacf Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:16 +0100 Subject: KVM: s390: vsie: Fix refcount overflow for shadow gmaps In most cases gmap_put() was not called when it should have. Add the missing gmap_put() in vsie_run(). Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/vsie.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 0330829b4046..72895dddc39a 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1328,7 +1328,7 @@ static void unregister_shadow_scb(struct kvm_vcpu *vcpu) static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - struct gmap *sg; + struct gmap *sg = NULL; int rc = 0; while (1) { @@ -1368,6 +1368,8 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) sg = gmap_put(sg); cond_resched(); } + if (sg) + sg = gmap_put(sg); if (rc == -EFAULT) { /* -- cgit v1.2.3 From 19d6c5b8044366c88c1b1f6e831c0661ff1ddd20 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:17 +0100 Subject: KVM: s390: vsie: Fix unshadowing while shadowing If shadowing causes the shadow gmap to get unshadowed, exit early to prevent an attempt to dereference the parent pointer, which at this point is NULL. Opportunistically add some more checks to prevent NULL parents. Fixes: a2c17f9270cc ("KVM: s390: New gmap code") Fixes: e5f98a6899bd ("KVM: s390: Add some helper functions needed for vSIE") Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/gaccess.c | 9 +++++++++ arch/s390/kvm/gmap.c | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 6bc30f678921..8d99667e7d34 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1449,6 +1449,8 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union pgste_set_unlock(ptep_h, pgste); if (rc) return rc; + if (!sg->parent) + return -EAGAIN; newpte = _pte(f->pfn, 0, !p, 0); if (!pgste_get_trylock(ptep, &pgste)) @@ -1476,6 +1478,9 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni return rc; do { + /* _gmap_crstep_xchg_atomic() could have unshadowed this shadow gmap */ + if (!sg->parent) + return -EAGAIN; oldcrste = READ_ONCE(*host); newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, f->writable, !p); newcrste.s.fc1.d |= oldcrste.s.fc1.d; @@ -1487,6 +1492,8 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni if (!newcrste.h.p && !f->writable) return -EOPNOTSUPP; } while (!_gmap_crstep_xchg_atomic(sg->parent, host, oldcrste, newcrste, f->gfn, false)); + if (!sg->parent) + return -EAGAIN; newcrste = _crste_fc1(f->pfn, oldcrste.h.tt, 0, !p); gfn = gpa_to_gfn(raddr); @@ -1531,6 +1538,8 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, entries[i - 1].pfn, i, entries[i - 1].writable); if (rc) return rc; + if (!sg->parent) + return -EAGAIN; } rc = dat_entry_walk(NULL, entries[LEVEL_MEM].gfn, sg->parent->asce, DAT_WALK_LEAF, diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index c8b79ad04ac9..645c32c767d2 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -1160,6 +1160,7 @@ struct gmap_protect_asce_top_level { static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gmap *sg, struct gmap_protect_asce_top_level *context) { + struct gmap *parent; int rc, i; guard(write_lock)(&sg->kvm->mmu_lock); @@ -1167,7 +1168,12 @@ static inline int __gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, s if (kvm_s390_array_needs_retry_safe(sg->kvm, context->seq, context->f)) return -EAGAIN; - scoped_guard(spinlock, &sg->parent->children_lock) { + parent = READ_ONCE(sg->parent); + if (!parent) + return -EAGAIN; + scoped_guard(spinlock, &parent->children_lock) { + if (READ_ONCE(sg->parent) != parent) + return -EAGAIN; for (i = 0; i < CRST_TABLE_PAGES; i++) { if (!context->f[i].valid) continue; @@ -1250,6 +1256,9 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare struct gmap *sg, *new; int rc; + if (WARN_ON(!parent)) + return ERR_PTR(-EINVAL); + scoped_guard(spinlock, &parent->children_lock) { sg = gmap_find_shadow(parent, asce, edat_level); if (sg) { -- cgit v1.2.3 From a12cc7e3d6a62f26262c1940a526f0682fefa3ba Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:18 +0100 Subject: KVM: s390: vsie: Fix guest page tables protection When shadowing, the guest page tables are write-protected, in order to trap changes and properly unshadow the shadow mapping for the nested guest. Already shadowed levels are skipped, so that only the needed levels are write protected. Currently the levels that get write protected are exactly one level too deep: the last level (nested guest memory) gets protected in the wrong way, and will be protected again correctly a few lines afterwards; most importantly, the highest non-shadowed level does *not* get write protected. Moreover, if the nested guest is running in a real address space, there are no DAT tables to shadow. Write protect the correct levels, so that all the levels that need to be protected are protected, and avoid double protecting the last level; skip attempting to shadow the DAT tables when the nested guest is running in a real address space. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Tested-by: Christian Borntraeger Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/gaccess.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 8d99667e7d34..53a8550e7102 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -1529,13 +1529,20 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gl = get_level(table, ptep); + /* In case of a real address space */ + if (w->level <= LEVEL_MEM) { + l = TABLE_TYPE_PAGE_TABLE; + hl = TABLE_TYPE_REGION1; + goto real_address_space; + } + /* * Skip levels that are already protected. For each level, protect * only the page containing the entry, not the whole table. */ for (i = gl ; i >= w->level; i--) { - rc = gmap_protect_rmap(mc, sg, entries[i - 1].gfn, gpa_to_gfn(saddr), - entries[i - 1].pfn, i, entries[i - 1].writable); + rc = gmap_protect_rmap(mc, sg, entries[i].gfn, gpa_to_gfn(saddr), + entries[i].pfn, i + 1, entries[i].writable); if (rc) return rc; if (!sg->parent) @@ -1551,6 +1558,7 @@ static int _gaccess_do_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *sg, /* Get the smallest granularity */ l = min3(gl, hl, w->level); +real_address_space: flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0); /* If necessary, create the shadow mapping */ if (l < gl) { -- cgit v1.2.3 From 0a28e06575b3f3b30c1e99fc08fa0907956f35a4 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Thu, 26 Mar 2026 14:17:19 +0100 Subject: KVM: s390: Fix KVM_S390_VCPU_FAULT ioctl A previous commit changed the behaviour of the KVM_S390_VCPU_FAULT ioctl. The current (wrong) implementation will trigger a guest addressing exception if the requested address lies outside of a memslot, unless the VM is UCONTROL. Restore the previous behaviour by open coding the fault-in logic. Fixes: 3762e905ec2e ("KVM: s390: use __kvm_faultin_pfn()") Acked-by: Christian Borntraeger Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda --- arch/s390/kvm/kvm-s390.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index b2c01fa7b852..d7838334a338 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5520,9 +5520,21 @@ long kvm_arch_vcpu_ioctl(struct file *filp, } #endif case KVM_S390_VCPU_FAULT: { - idx = srcu_read_lock(&vcpu->kvm->srcu); - r = vcpu_dat_fault_handler(vcpu, arg, 0); - srcu_read_unlock(&vcpu->kvm->srcu, idx); + gpa_t gaddr = arg; + + scoped_guard(srcu, &vcpu->kvm->srcu) { + r = vcpu_ucontrol_translate(vcpu, &gaddr); + if (r) + break; + + r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false); + if (r == PGM_ADDRESSING) + r = -EFAULT; + if (r <= 0) + break; + r = -EIO; + KVM_BUG_ON(r, vcpu->kvm); + } break; } case KVM_ENABLE_CAP: -- cgit v1.2.3 From aad885e774966e97b675dfe928da164214a71605 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 5 Mar 2026 17:28:04 -0800 Subject: KVM: x86/mmu: Drop/zap existing present SPTE even when creating an MMIO SPTE When installing an emulated MMIO SPTE, do so *after* dropping/zapping the existing SPTE (if it's shadow-present). While commit a54aa15c6bda3 was right about it being impossible to convert a shadow-present SPTE to an MMIO SPTE due to a _guest_ write, it failed to account for writes to guest memory that are outside the scope of KVM. E.g. if host userspace modifies a shadowed gPTE to switch from a memslot to emulted MMIO and then the guest hits a relevant page fault, KVM will install the MMIO SPTE without first zapping the shadow-present SPTE. ------------[ cut here ]------------ is_shadow_present_pte(*sptep) WARNING: arch/x86/kvm/mmu/mmu.c:484 at mark_mmio_spte+0xb2/0xc0 [kvm], CPU#0: vmx_ept_stale_r/4292 Modules linked in: kvm_intel kvm irqbypass CPU: 0 UID: 1000 PID: 4292 Comm: vmx_ept_stale_r Not tainted 7.0.0-rc2-eafebd2d2ab0-sink-vm #319 PREEMPT Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:mark_mmio_spte+0xb2/0xc0 [kvm] Call Trace: mmu_set_spte+0x237/0x440 [kvm] ept_page_fault+0x535/0x7f0 [kvm] kvm_mmu_do_page_fault+0xee/0x1f0 [kvm] kvm_mmu_page_fault+0x8d/0x620 [kvm] vmx_handle_exit+0x18c/0x5a0 [kvm_intel] kvm_arch_vcpu_ioctl_run+0xc55/0x1c20 [kvm] kvm_vcpu_ioctl+0x2d5/0x980 [kvm] __x64_sys_ioctl+0x8a/0xd0 do_syscall_64+0xb5/0x730 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x47fa3f ---[ end trace 0000000000000000 ]--- Reported-by: Alexander Bulekov Debugged-by: Alexander Bulekov Suggested-by: Fred Griffoul Fixes: a54aa15c6bda3 ("KVM: x86/mmu: Handle MMIO SPTEs directly in mmu_set_spte()") Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b922a8b00057..98406d6aa2d6 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3044,12 +3044,6 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, bool prefetch = !fault || fault->prefetch; bool write_fault = fault && fault->write; - if (unlikely(is_noslot_pfn(pfn))) { - vcpu->stat.pf_mmio_spte_created++; - mark_mmio_spte(vcpu, sptep, gfn, pte_access); - return RET_PF_EMULATE; - } - if (is_shadow_present_pte(*sptep)) { if (prefetch && is_last_spte(*sptep, level) && pfn == spte_to_pfn(*sptep)) @@ -3073,6 +3067,14 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, was_rmapped = 1; } + if (unlikely(is_noslot_pfn(pfn))) { + vcpu->stat.pf_mmio_spte_created++; + mark_mmio_spte(vcpu, sptep, gfn, pte_access); + if (flush) + kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level); + return RET_PF_EMULATE; + } + wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch, false, host_writable, &spte); -- cgit v1.2.3 From df83746075778958954aa0460cca55f4b3fc9c02 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 5 Mar 2026 17:42:14 -0800 Subject: KVM: x86/mmu: Only WARN in direct MMUs when overwriting shadow-present SPTE Adjust KVM's sanity check against overwriting a shadow-present SPTE with a another SPTE with a different target PFN to only apply to direct MMUs, i.e. only to MMUs without shadowed gPTEs. While it's impossible for KVM to overwrite a shadow-present SPTE in response to a guest write, writes from outside the scope of KVM, e.g. from host userspace, aren't detected by KVM's write tracking and so can break KVM's shadow paging rules. ------------[ cut here ]------------ pfn != spte_to_pfn(*sptep) WARNING: arch/x86/kvm/mmu/mmu.c:3069 at mmu_set_spte+0x1e4/0x440 [kvm], CPU#0: vmx_ept_stale_r/872 Modules linked in: kvm_intel kvm irqbypass CPU: 0 UID: 1000 PID: 872 Comm: vmx_ept_stale_r Not tainted 7.0.0-rc2-eafebd2d2ab0-sink-vm #319 PREEMPT Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:mmu_set_spte+0x1e4/0x440 [kvm] Call Trace: ept_page_fault+0x535/0x7f0 [kvm] kvm_mmu_do_page_fault+0xee/0x1f0 [kvm] kvm_mmu_page_fault+0x8d/0x620 [kvm] vmx_handle_exit+0x18c/0x5a0 [kvm_intel] kvm_arch_vcpu_ioctl_run+0xc55/0x1c20 [kvm] kvm_vcpu_ioctl+0x2d5/0x980 [kvm] __x64_sys_ioctl+0x8a/0xd0 do_syscall_64+0xb5/0x730 entry_SYSCALL_64_after_hwframe+0x4b/0x53 ---[ end trace 0000000000000000 ]--- Fixes: 11d45175111d ("KVM: x86/mmu: Warn if PFN changes on shadow-present SPTE in shadow MMU") Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 98406d6aa2d6..dd06453d5b72 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3060,7 +3060,8 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, child = spte_to_child_sp(pte); drop_parent_pte(vcpu->kvm, child, sptep); flush = true; - } else if (WARN_ON_ONCE(pfn != spte_to_pfn(*sptep))) { + } else if (pfn != spte_to_pfn(*sptep)) { + WARN_ON_ONCE(vcpu->arch.mmu->root_role.direct); drop_spte(vcpu->kvm, sptep); flush = true; } else -- cgit v1.2.3 From c5c0a268b38adffbb2e70e6957017537ff54c157 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 26 Mar 2026 14:38:44 +0100 Subject: s390/barrier: Make array_index_mask_nospec() __always_inline Mark array_index_mask_nospec() as __always_inline to guarantee the mitigation is emitted inline regardless of compiler inlining decisions. Fixes: e2dd833389cc ("s390: add optimized array_index_mask_nospec") Cc: stable@kernel.org Reviewed-by: Ilya Leoshkevich Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/barrier.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index f3184073e754..dad02f5b3c8d 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -62,8 +62,8 @@ do { \ * @size: number of elements in array */ #define array_index_mask_nospec array_index_mask_nospec -static inline unsigned long array_index_mask_nospec(unsigned long index, - unsigned long size) +static __always_inline unsigned long array_index_mask_nospec(unsigned long index, + unsigned long size) { unsigned long mask; -- cgit v1.2.3 From 48b8814e25d073dd84daf990a879a820bad2bcbd Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 24 Mar 2026 17:34:05 +0100 Subject: s390/syscalls: Add spectre boundary for syscall dispatch table The s390 syscall number is directly controlled by userspace, but does not have an array_index_nospec() boundary to prevent access past the syscall function pointer tables. Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Arnd Bergmann Fixes: 56e62a737028 ("s390: convert to generic entry") Cc: stable@kernel.org Assisted-by: gkh_clanker_2000 Signed-off-by: Greg Kroah-Hartman Reviewed-by: Vasily Gorbik Link: https://lore.kernel.org/r/2026032404-sterling-swoosh-43e6@gregkh Signed-off-by: Vasily Gorbik --- arch/s390/kernel/syscall.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index 795b6cca74c9..d103c853e120 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -13,6 +13,7 @@ */ #include +#include #include #include #include @@ -131,8 +132,10 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET))) goto out; regs->gprs[2] = -ENOSYS; - if (likely(nr < NR_syscalls)) + if (likely(nr < NR_syscalls)) { + nr = array_index_nospec(nr, NR_syscalls); regs->gprs[2] = sys_call_table[nr](regs); + } out: syscall_exit_to_user_mode(regs); } -- cgit v1.2.3 From 0738d395aab8fae3b5a3ad3fc640630c91693c27 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 26 Mar 2026 19:50:14 +0100 Subject: s390/entry: Scrub r12 register on kernel entry Before commit f33f2d4c7c80 ("s390/bp: remove TIF_ISOLATE_BP"), all entry handlers loaded r12 with the current task pointer (lg %r12,__LC_CURRENT) for use by the BPENTER/BPEXIT macros. That commit removed TIF_ISOLATE_BP, dropping both the branch prediction macros and the r12 load, but did not add r12 to the register clearing sequence. Add the missing xgr %r12,%r12 to make the register scrub consistent across all entry points. Fixes: f33f2d4c7c80 ("s390/bp: remove TIF_ISOLATE_BP") Cc: stable@kernel.org Reviewed-by: Ilya Leoshkevich Signed-off-by: Vasily Gorbik --- arch/s390/kernel/entry.S | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch') diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 4873fe9d891b..689d253e1afc 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -271,6 +271,7 @@ SYM_CODE_START(system_call) xgr %r9,%r9 xgr %r10,%r10 xgr %r11,%r11 + xgr %r12,%r12 la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs mvc __PT_R8(64,%r2),__LC_SAVE_AREA(%r13) MBEAR %r2,%r13 @@ -407,6 +408,7 @@ SYM_CODE_START(\name) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA(%r13) MBEAR %r11,%r13 @@ -496,6 +498,7 @@ SYM_CODE_START(mcck_int_handler) xgr %r6,%r6 xgr %r7,%r7 xgr %r10,%r10 + xgr %r12,%r12 stmg %r8,%r9,__PT_PSW(%r11) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) -- cgit v1.2.3