diff options
| author | Dave Airlie <airlied@redhat.com> | 2026-08-20 10:58:44 +1000 |
|---|---|---|
| committer | Dave Airlie <airlied@redhat.com> | 2026-08-20 10:58:44 +1000 |
| commit | c44e278ce02efd0c4be79a8eda1ea6885c1ce5ec (patch) | |
| tree | b6b7324a56fb1869a094da177d6a41f4d2efe8e0 /arch | |
| parent | 65b92bc671db7c222f8de1146c88e40fe8764d95 (diff) | |
| parent | 8d3ae59288f1e7d58d76558a6ee96d533bc5019f (diff) | |
| download | linux-next-c44e278ce02efd0c4be79a8eda1ea6885c1ce5ec.tar.gz linux-next-c44e278ce02efd0c4be79a8eda1ea6885c1ce5ec.zip | |
BackMerge tag 'v7.2' into drm-next
Linux 7.2
There was a lot of conflicts this round between fixes and next,
and I'd like to get the merge resolutions that we have in drm-tip.
Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'arch')
188 files changed, 1589 insertions, 898 deletions
diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig index f930396d9dae..870e5291b7db 100644 --- a/arch/arc/configs/axs101_defconfig +++ b/arch/arc/configs/axs101_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_DRM=m CONFIG_DRM_I2C_ADV7511=m diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig index 6b779dee5ea0..d45e4d335998 100644 --- a/arch/arc/configs/axs103_defconfig +++ b/arch/arc/configs/axs103_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_FB=y CONFIG_FRAMEBUFFER_CONSOLE=y diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index a89b50d5369d..f986c0205f13 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y # CONFIG_HWMON is not set CONFIG_DRM=m CONFIG_DRM_I2C_ADV7511=m diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index 865fbc19ef03..6e396a9ddb8b 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -61,7 +61,6 @@ CONFIG_SERIAL_8250_DW=y CONFIG_I2C=y # CONFIG_I2C_COMPAT is not set CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_GPIO_SYSFS=y # CONFIG_HWMON is not set # CONFIG_USB_SUPPORT is not set diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index b2f2c59279a6..2d99dffed0ce 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -22,6 +22,7 @@ #include <linux/irqdomain.h> #include <linux/export.h> #include <linux/of_fdt.h> +#include <linux/string.h> #include <asm/mach_desc.h> #include <asm/setup.h> @@ -43,9 +44,10 @@ static int __init arc_get_cpu_map(const char *name, struct cpumask *cpumask) { unsigned long dt_root = of_get_flat_dt_root(); const char *buf; + int len; - buf = of_get_flat_dt_prop(dt_root, name, NULL); - if (!buf) + buf = of_get_flat_dt_prop(dt_root, name, &len); + if (!buf || !memchr(buf, '\0', len)) return -EINVAL; if (cpulist_parse(buf, cpumask)) diff --git a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi index 392a25713669..bd52de0faa3e 100644 --- a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi +++ b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi @@ -141,7 +141,7 @@ /* PCIe Controller 2 */ <0x00014000 0 &gic GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>, - <0x00014000 1 &gic GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>, + <0x00014000 1 &gic GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 2 &gic GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 3 &gic GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 4 &gic GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts index 5bbc2ba0f959..87569408bb69 100644 --- a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts +++ b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts @@ -26,7 +26,7 @@ nvram@1c080000 { compatible = "brcm,nvram"; - reg = <0x1c080000 0x180000>; + reg = <0x1c080000 0x100000>; et2macaddr: et2macaddr { #nvmem-cell-cells = <1>; diff --git a/arch/arm/mach-ixp4xx/Kconfig b/arch/arm/mach-ixp4xx/Kconfig index cb46802f5ce5..7f812020e082 100644 --- a/arch/arm/mach-ixp4xx/Kconfig +++ b/arch/arm/mach-ixp4xx/Kconfig @@ -14,6 +14,5 @@ menuconfig ARCH_IXP4XX select IXP4XX_TIMER select USB_EHCI_BIG_ENDIAN_DESC select USB_EHCI_BIG_ENDIAN_MMIO - select USE_OF help Support for Intel's IXP4XX (XScale) family of processors. diff --git a/arch/arm/mach-npcm/platsmp.c b/arch/arm/mach-npcm/platsmp.c index 41891d3aa124..4c1fc9983746 100644 --- a/arch/arm/mach-npcm/platsmp.c +++ b/arch/arm/mach-npcm/platsmp.c @@ -32,6 +32,7 @@ static int npcm7xx_smp_boot_secondary(unsigned int cpu, goto out; } gcr_base = of_iomap(gcr_np, 0); + of_node_put(gcr_np); if (!gcr_base) { pr_err("could not iomap gcr"); ret = -ENOMEM; @@ -60,6 +61,7 @@ static void __init npcm7xx_smp_prepare_cpus(unsigned int max_cpus) return; } scu_base = of_iomap(scu_np, 0); + of_node_put(scu_np); if (!scu_base) { pr_err("could not iomap scu"); return; diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig index 66e26990e2c8..c478fb8a6f78 100644 --- a/arch/arm/mach-pxa/Kconfig +++ b/arch/arm/mach-pxa/Kconfig @@ -22,7 +22,6 @@ config MACH_PXA25X_DT select PINCTRL select POWER_SUPPLY select PXA25x - select USE_OF help Include support for Marvell PXA25x based platforms using the device tree. Needn't select any other machine while @@ -33,7 +32,6 @@ config MACH_PXA27X_DT select PINCTRL select POWER_SUPPLY select PXA27x - select USE_OF help Include support for Marvell PXA27x based platforms using the device tree. Needn't select any other machine while @@ -47,7 +45,6 @@ config MACH_PXA3XX_DT select PINCTRL select POWER_SUPPLY select PXA3xx - select USE_OF help Include support for Marvell PXA3xx based platforms using the device tree. Needn't select any other machine while diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi index c6196225e96e..1ee61c5b3409 100644 --- a/arch/arm64/boot/dts/apple/t8122.dtsi +++ b/arch/arm64/boot/dts/apple/t8122.dtsi @@ -186,12 +186,12 @@ /* Required to get >32-bit DMA via DARTs */ dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; - i2c0: i2c@235010000 { + i2c0: i2c@2a1010000 { compatible = "apple,t8122-i2c", "apple,t8103-i2c"; - reg = <0x2 0x35010000 0x0 0x4000>; + reg = <0x2 0xa1010000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = <AIC_IRQ 761 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <AIC_IRQ 780 IRQ_TYPE_LEVEL_HIGH>; pinctrl-0 = <&i2c0_pins>; pinctrl-names = "default"; #address-cells = <0x1>; @@ -200,12 +200,12 @@ status = "disabled"; }; - i2c1: i2c@235014000 { + i2c1: i2c@2a1014000 { compatible = "apple,t8122-i2c", "apple,t8103-i2c"; - reg = <0x2 0x35014000 0x0 0x4000>; + reg = <0x2 0xa1014000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = <AIC_IRQ 762 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <AIC_IRQ 781 IRQ_TYPE_LEVEL_HIGH>; pinctrl-0 = <&i2c1_pins>; pinctrl-names = "default"; #address-cells = <0x1>; @@ -214,12 +214,12 @@ status = "disabled"; }; - i2c2: i2c@235018000 { + i2c2: i2c@2a1018000 { compatible = "apple,t8122-i2c", "apple,t8103-i2c"; - reg = <0x2 0x35018000 0x0 0x4000>; + reg = <0x2 0xa1018000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = <AIC_IRQ 763 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <AIC_IRQ 782 IRQ_TYPE_LEVEL_HIGH>; pinctrl-0 = <&i2c2_pins>; pinctrl-names = "default"; #address-cells = <0x1>; @@ -228,12 +228,12 @@ status = "disabled"; }; - i2c3: i2c@23501c000 { + i2c3: i2c@2a101c000 { compatible = "apple,t8122-i2c", "apple,t8103-i2c"; - reg = <0x2 0x3501c000 0x0 0x4000>; + reg = <0x2 0xa101c000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = <AIC_IRQ 764 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <AIC_IRQ 783 IRQ_TYPE_LEVEL_HIGH>; pinctrl-0 = <&i2c3_pins>; pinctrl-names = "default"; #address-cells = <0x1>; @@ -242,12 +242,12 @@ status = "disabled"; }; - i2c4: i2c@235020000 { + i2c4: i2c@2a1020000 { compatible = "apple,t8122-i2c", "apple,t8103-i2c"; - reg = <0x2 0x35020000 0x0 0x4000>; + reg = <0x2 0xa1020000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; - interrupts = <AIC_IRQ 765 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <AIC_IRQ 784 IRQ_TYPE_LEVEL_HIGH>; pinctrl-0 = <&i2c4_pins>; pinctrl-names = "default"; #address-cells = <0x1>; diff --git a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi index 761c59d90ffc..09ff5e9959d3 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi @@ -678,8 +678,6 @@ <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | - IRQ_TYPE_LEVEL_LOW)>, - <GIC_PPI 12 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index eb7e3c666df6..5e510f77b538 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -3176,6 +3176,8 @@ <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, <GIC_PPI 10 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 15 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; interrupt-parent = <&gic>; always-on; diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index 8e0c51e496e2..820670dd6042 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -5355,7 +5355,7 @@ #size-cells = <0>; cpu0_0: cpu@0 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00000>; @@ -5374,7 +5374,7 @@ }; cpu0_1: cpu@100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00100>; @@ -5393,7 +5393,7 @@ }; cpu0_2: cpu@200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00200>; @@ -5412,7 +5412,7 @@ }; cpu0_3: cpu@300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00300>; @@ -5431,7 +5431,7 @@ }; cpu1_0: cpu@10000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10000>; @@ -5450,7 +5450,7 @@ }; cpu1_1: cpu@10100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10100>; @@ -5469,7 +5469,7 @@ }; cpu1_2: cpu@10200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10200>; @@ -5488,7 +5488,7 @@ }; cpu1_3: cpu@10300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10300>; @@ -5507,7 +5507,7 @@ }; cpu2_0: cpu@20000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20000>; @@ -5526,7 +5526,7 @@ }; cpu2_1: cpu@20100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20100>; @@ -5545,7 +5545,7 @@ }; cpu2_2: cpu@20200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20200>; @@ -5564,7 +5564,7 @@ }; cpu2_3: cpu@20300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20300>; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 2d8e7e37830f..2d2cb1a3d95c 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -3208,7 +3208,7 @@ }; gpcdma: dma-controller@8400000 { - compatible = "nvidia,tegra264-gpcdma", "nvidia,tegra186-gpcdma"; + compatible = "nvidia,tegra264-gpcdma"; reg = <0x0 0x08400000 0x0 0x210000>; interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>, @@ -4070,7 +4070,7 @@ d-cache-sets = <256>; }; - cpu1: cpu@1 { + cpu1: cpu@10000 { compatible = "arm,neoverse-v3ae"; device_type = "cpu"; reg = <0x10000>; diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi index 977de44b816e..72b786fec195 100644 --- a/arch/arm64/boot/dts/qcom/eliza.dtsi +++ b/arch/arm64/boot/dts/qcom/eliza.dtsi @@ -2559,7 +2559,7 @@ "dsi_pll"; clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "ref"; diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi index 20b49af7298e..129be417ac42 100644 --- a/arch/arm64/boot/dts/qcom/glymur.dtsi +++ b/arch/arm64/boot/dts/qcom/glymur.dtsi @@ -1876,7 +1876,7 @@ spi0: spi@b80000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b80000 0x0 0x4000>; - interrupts = <GIC_SPI 1052 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 92 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -1903,7 +1903,7 @@ i2c1: i2c@b84000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00b84000 0x0 0x4000>; - interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 93 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -1930,7 +1930,7 @@ spi1: spi@b84000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b84000 0x0 0x4000>; - interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 93 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -1957,7 +1957,7 @@ i2c2: i2c@b88000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00b88000 0x0 0x4000>; - interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -1984,7 +1984,7 @@ spi2: spi@b88000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b88000 0x0 0x4000>; - interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2011,7 +2011,7 @@ uart2: serial@b88000 { compatible = "qcom,geni-uart"; reg = <0x0 0x00b88000 0x0 0x4000>; - interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2056,7 +2056,7 @@ spi3: spi@b8c000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b8c000 0x0 0x4000>; - interrupts = <GIC_SPI 1055 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 95 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2110,7 +2110,7 @@ spi4: spi@b90000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b90000 0x0 0x4000>; - interrupts = <GIC_SPI 1056 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 96 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2164,7 +2164,7 @@ spi5: spi@b94000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b94000 0x0 0x4000>; - interrupts = <GIC_SPI 1057 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 97 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2191,7 +2191,7 @@ i2c6: i2c@b98000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00b98000 0x0 0x4000>; - interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2218,7 +2218,7 @@ spi6: spi@b98000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b98000 0x0 0x4000>; - interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2245,7 +2245,7 @@ i2c7: i2c@b9c000 { compatible = "qcom,geni-i2c"; reg = <0x0 0x00b9c000 0x0 0x4000>; - interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -2272,7 +2272,7 @@ spi7: spi@b9c000 { compatible = "qcom,geni-spi"; reg = <0x0 0x00b9c000 0x0 0x4000>; - interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>; clock-names = "se"; interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS @@ -6874,9 +6874,9 @@ pcie_smmu: iommu@15480000 { compatible = "arm,smmu-v3"; reg = <0x0 0x15480000 0x0 0x20000>; - interrupts = <GIC_SPI 964 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 962 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 960 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_ESPI 4 IRQ_TYPE_LEVEL_HIGH>, + <GIC_ESPI 2 IRQ_TYPE_LEVEL_HIGH>, + <GIC_ESPI 0 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "eventq", "cmdq-sync", "gerror"; dma-coherent; #iommu-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index e4c8466f941b..a350a8ba48fa 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -7384,6 +7384,7 @@ interrupt-controller; #redistributor-regions = <1>; redistributor-stride = <0x0 0x20000>; + #address-cells = <0>; }; watchdog@17c10000 { diff --git a/arch/arm64/boot/dts/qcom/purwa.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi index 9ab4f26b35f2..5b17840fb62f 100644 --- a/arch/arm64/boot/dts/qcom/purwa.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa.dtsi @@ -47,6 +47,8 @@ &gpu { compatible = "qcom,adreno-43030c00", "qcom,adreno"; + iommus = <&adreno_smmu 0 0x0>; + nvmem-cells = <&gpu_speed_bin>; nvmem-cell-names = "speed_bin"; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts index f3c00be67081..cd575e20d90a 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts @@ -637,7 +637,7 @@ compatible = "huawei,gaokun3-ec"; reg = <0x38>; - interrupts-extended = <&tlmm 107 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&tlmm 103 IRQ_TYPE_LEVEL_LOW>; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index a2bd6b10e475..b0de9e262f29 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -5356,7 +5356,7 @@ <211 699 1>, <212 705 1>, <213 450 1>, - <214 643 1>, + <214 643 2>, <216 646 5>, <221 390 5>, <226 700 3>, @@ -5379,7 +5379,7 @@ <252 798 1>, <253 765 1>, <254 763 1>, - <255 454 1>, + <255 454 3>, <258 139 1>, <259 786 2>, <261 370 2>, diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index e41200839dbe..8cdaef5360d1 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -347,6 +347,10 @@ status = "okay"; }; +&cluster_sleep_0 { + arm,psci-suspend-param = <0x41008244>; +}; + &gcc { protected-clocks = <GCC_QSPI_CORE_CLK>, <GCC_QSPI_CORE_CLK_SRC>, diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index 160ead25ecf7..65c4a2b46d21 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -7091,8 +7091,8 @@ ranges = <0 0 0x14680000 0x2c000>; - ipa_modem_tables: modem-tables@8000 { - reg = <0x8000 0x2000>; + ipa_modem_tables: modem-tables@3000 { + reg = <0x3000 0x2000>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts index d2b3fc08954a..0ab303863155 100644 --- a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts +++ b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts @@ -107,6 +107,47 @@ reg = <0x0 0x8c400000 0x0 0x02000000>; no-map; }; + + /* DRAM controller inline ECC areas */ + ecc@10cccc0000 { + reg = <0x10 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@12cccc0000 { + reg = <0x12 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@14cccc0000 { + reg = <0x14 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@16cccc0000 { + reg = <0x16 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@18cccc0000 { + reg = <0x18 0xcccc0000 0x0 0x33340000>; + no-map; + }; + + ecc@1a66660000 { + reg = <0x1a 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1c66660000 { + reg = <0x1c 0x66660000 0x0 0x999a0000>; + no-map; + }; + + ecc@1e66660000 { + reg = <0x1e 0x66660000 0x0 0x999a0000>; + no-map; + }; }; }; diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 81c17320a588..f816f5d77f1a 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -131,7 +131,7 @@ * Annoyingly, the negative levels for Address size faults aren't laid out * contiguously (or in the desired order) */ -#define ESR_ELx_FSC_ADDRSZ_nL(n) ((n) == -1 ? 0x25 : 0x2C) +#define ESR_ELx_FSC_ADDRSZ_nL(n) ((n) == -1 ? 0x29 : 0x2C) #define ESR_ELx_FSC_ADDRSZ_L(n) ((n) < 0 ? ESR_ELx_FSC_ADDRSZ_nL(n) : \ (ESR_ELx_FSC_ADDRSZ + (n))) diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 65555284446e..7075a3bd2c61 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -78,8 +78,12 @@ enum fixed_addresses { /* * Temporary boot-time mappings, used by early_ioremap(), * before ioremap() is functional. + * + * Reserve one extra page so a 256K mapping may start at any + * offset within a page. early_ioremap() maps the page-aligned + * physical range, so the initial offset can consume an extra page. */ -#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) +#define NR_FIX_BTMAPS ((SZ_256K / PAGE_SIZE) + 1) #define FIX_BTMAPS_SLOTS 7 #define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS) diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h index f2782560647b..35ce2c94040e 100644 --- a/arch/arm64/include/asm/kprobes.h +++ b/arch/arm64/include/asm/kprobes.h @@ -26,6 +26,12 @@ struct prev_kprobe { struct kprobe *kp; unsigned int status; + + /* + * The original DAIF state of the outer kprobe, saved here before + * a nested kprobe overwrites kcb->saved_irqflag during reentry. + */ + unsigned long saved_irqflag; }; /* per-cpu kprobe control block */ diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h index 743c49bd878f..5f6e6789d121 100644 --- a/arch/arm64/include/asm/kvm_hypevents.h +++ b/arch/arm64/include/asm/kvm_hypevents.h @@ -12,6 +12,7 @@ enum hyp_enter_exit_reason { HYP_REASON_SMC, HYP_REASON_HVC, + HYP_REASON_SYS, HYP_REASON_PSCI, HYP_REASON_HOST_ABORT, HYP_REASON_GUEST_EXIT, diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index cdf3e8422ea1..012d711034d1 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,14 @@ struct s1_walk_result { bool failed; }; +static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) +{ + wr->fst = fst; + wr->ptw = s1ptw; + wr->s2 = s1ptw; + wr->failed = true; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index 5b374a6ab34a..50a195eda8ed 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -7,8 +7,6 @@ #include <linux/ptdump.h> -DECLARE_STATIC_KEY_FALSE(arm64_ptdump_lock_key); - #ifdef CONFIG_PTDUMP #include <linux/mm_types.h> diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index 6297631532e5..767f35ea62b3 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,17 +2,11 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H -#include <linux/cpumask.h> - struct arch_tlbflush_unmap_batch { -#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * Track CPUs that need SME DVMSync on completion of this batch. - * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to - * record cpumask for sending IPI + * For arm64, HW can do TLB shootdown, so we don't need to record a + * cpumask for sending IPIs. */ - cpumask_var_t cpumask; -#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index d52ac8c17190..e0e84332f51b 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -82,6 +82,8 @@ static inline unsigned long get_trans_granule(void) #ifdef CONFIG_ARM64_ERRATUM_4193714 +extern cpumask_t sme_active_cpus; + void sme_do_dvmsync(const struct cpumask *mask); static inline void sme_dvmsync(struct mm_struct *mm) @@ -92,42 +94,12 @@ static inline void sme_dvmsync(struct mm_struct *mm) sme_do_dvmsync(mm_cpumask(mm)); } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) +static inline void sme_dvmsync_batch(void) { if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) return; - /* - * Order the mm_cpumask() read after the hardware DVMSync. - */ - dsb(ish); - if (cpumask_empty(mm_cpumask(mm))) - return; - - /* - * Allocate the batch cpumask on first use. Fall back to an immediate - * IPI for this mm in case of failure. - */ - if (!cpumask_available(batch->cpumask) && - !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { - sme_do_dvmsync(mm_cpumask(mm)); - return; - } - - cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); -} - -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) -{ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - return; - - if (!cpumask_available(batch->cpumask)) - return; - - sme_do_dvmsync(batch->cpumask); - cpumask_clear(batch->cpumask); + sme_do_dvmsync(&sme_active_cpus); } #else @@ -135,11 +107,7 @@ static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) static inline void sme_dvmsync(struct mm_struct *mm) { } -static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm) -{ -} -static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void sme_dvmsync_batch(void) { } @@ -285,11 +253,11 @@ static inline void __tlbi_sync_s1ish(struct mm_struct *mm) sme_dvmsync(mm); } -static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +static inline void __tlbi_sync_s1ish_batch(void) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); - sme_dvmsync_batch(batch); + sme_dvmsync_batch(); } static inline void __tlbi_sync_s1ish_kernel(void) @@ -434,7 +402,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish_batch(batch); + __tlbi_sync_s1ish_batch(); } /* @@ -722,7 +690,6 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); - sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 5891f92c2035..681aa2bbc399 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -448,12 +448,14 @@ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 apci_id, return *pcpu; } + set_cpu_present(*pcpu, true); return 0; } EXPORT_SYMBOL(acpi_map_cpu); int acpi_unmap_cpu(int cpu) { + set_cpu_present(cpu, false); return 0; } EXPORT_SYMBOL(acpi_unmap_cpu); diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 25dc5afe9ba0..e7f1682a3059 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1355,6 +1355,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) * SME/CME erratum handling. */ static cpumask_t sme_dvmsync_cpus; +cpumask_t sme_active_cpus; /* * These helpers are only called from non-preemptible contexts, so @@ -1368,13 +1369,15 @@ void sme_set_active(void) return; cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + cpumask_set_cpu(cpu, &sme_active_cpus); /* * A subsequent (post ERET) SME access may use a stale address * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for - * the completion of cpumask_set_cpu() above as it appears in program - * order before the SME access. The post-TLBI+DSB read of mm_cpumask() - * will lead to the IPI being issued. + * the completion of the cpumask_set_cpu() operations above as they + * appear in program order before the SME access. The post-TLBI+DSB + * read of mm_cpumask() or sme_active_cpus will lead to the IPI being + * issued. * * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 */ @@ -1392,6 +1395,7 @@ void sme_clear_active(void) * completed on entering EL1. */ cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); + cpumask_clear_cpu(cpu, &sme_active_cpus); } static void sme_dvmsync_ipi(void *unused) diff --git a/arch/arm64/kernel/pi/relocate.c b/arch/arm64/kernel/pi/relocate.c index 2407d2696398..82592f3a5c1c 100644 --- a/arch/arm64/kernel/pi/relocate.c +++ b/arch/arm64/kernel/pi/relocate.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only // Copyright 2023 Google LLC // Authors: Ard Biesheuvel <ardb@google.com> -// Peter Collingbourne <pcc@google.com> +// Peter Collingbourne <peter@pcc.me.uk> #include <linux/elf.h> #include <linux/init.h> diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 43a0361a8bf0..4e0efad5caf2 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -174,12 +174,27 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; + + /* + * Save the outer kprobe's original DAIF flags before the nested + * kprobe calls kprobes_save_local_irqflag() and overwrites + * kcb->saved_irqflag. Without this, the outer kprobe will restore + * the wrong DAIF state and leave interrupts permanently masked. + */ + kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) { __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; + + /* + * Restore the outer kprobe's saved_irqflag so that when its + * single-step completes, kprobes_restore_local_irqflag() uses + * the correct original DAIF value. + */ + kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } static void __kprobes set_current_kprobe(struct kprobe *p) @@ -240,10 +255,16 @@ static int __kprobes reenter_kprobe(struct kprobe *p, switch (kcb->kprobe_status) { case KPROBE_HIT_SSDONE: case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SS: + /* + * A probe can be hit while another kprobe is preparing or + * executing its XOL single-step instruction. This is still a + * recoverable one-level reentry, so handle it in the same way as + * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. + */ kprobes_inc_nmissed_count(p); setup_singlestep(p, regs, kcb, 1); break; - case KPROBE_HIT_SS: case KPROBE_REENTER: pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); @@ -282,10 +303,32 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + /* + * Simulated kprobes execute in the debug trap context and have no + * XOL slot. Any page fault taken while a simulated kprobe is in + * progress cannot have been caused by kprobe single-stepping and + * must be left alone for the normal page fault handler, including + * fixup_exception. + */ + if (cur && !cur->ainsn.xol_insn) + return 0; + switch (kcb->kprobe_status) { case KPROBE_HIT_SS: case KPROBE_REENTER: /* + * A page fault taken while in KPROBE_HIT_SS or + * KPROBE_REENTER state is only attributable to kprobe + * single-stepping if the faulting PC points to the + * current kprobe's XOL instruction. If the fault occurred + * elsewhere (e.g. in perf or tracing code invoked from the + * debug exception path), leave it for the normal page fault + * handler to process. + */ + if (instruction_pointer(regs) != (unsigned long)cur->ainsn.xol_insn) + break; + + /* * We are here because the instruction being single * stepped caused a page fault. We reset the current * kprobe and the ip points back to the probe address diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 033643cd4e5e..581f80e9b9b7 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -341,41 +341,8 @@ void flush_thread(void) flush_gcs(); } -#ifdef CONFIG_ARM64_ERRATUM_4193714 - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ - /* - * Clear the inherited cpumask with memset() to cover both cases where - * cpumask_var_t is a pointer or an array. It will be allocated lazily - * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. - */ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - memset(&dst->tlb_ubc.arch.cpumask, 0, - sizeof(dst->tlb_ubc.arch.cpumask)); -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ - if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) - free_cpumask_var(tsk->tlb_ubc.arch.cpumask); -} - -#else - -static void arch_dup_tlbbatch_mask(struct task_struct *dst) -{ -} - -static void arch_release_tlbbatch_mask(struct task_struct *tsk) -{ -} - -#endif /* CONFIG_ARM64_ERRATUM_4193714 */ - void arch_release_task_struct(struct task_struct *tsk) { - arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -391,8 +358,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; - arch_dup_tlbbatch_mask(dst); - /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index d46022f72075..cdcdd160e5b6 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -535,23 +535,13 @@ void arch_unregister_cpu(int cpu) { acpi_handle acpi_handle = acpi_get_processor_handle(cpu); struct cpu *c = &per_cpu(cpu_devices, cpu); - acpi_status status; unsigned long long sta; - - if (!acpi_handle) { - pr_err_once("Removing a CPU without associated ACPI handle\n"); - return; - } + acpi_status status; status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return; - - /* For now do not allow anything that looks like physical CPU HP */ - if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) { + if (!ACPI_FAILURE(status) && + cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) pr_err_once("Changing CPU present bit is not supported\n"); - return; - } unregister_cpu(c); } @@ -566,6 +556,11 @@ struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu) } EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc); +static bool acpi_cpu_is_present(int cpu) +{ + return acpi_cpu_get_madt_gicc(cpu)->flags & ACPI_MADT_ENABLED; +} + /* * acpi_map_gic_cpu_interface - parse processor MADT entry * @@ -670,6 +665,10 @@ static void __init acpi_parse_and_init_cpus(void) early_map_cpu_to_node(i, acpi_numa_get_nid(i)); } #else +static bool acpi_cpu_is_present(int cpu) +{ + return false; +} #define acpi_parse_and_init_cpus(...) do { } while (0) #endif @@ -814,7 +813,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (err) continue; - set_cpu_present(cpu, true); + if (acpi_disabled || acpi_cpu_is_present(cpu)) + set_cpu_present(cpu, true); numa_store_cpu_info(cpu); } } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..9a6c72a18672 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -149,14 +149,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, &kvm->arch.flags); break; - case KVM_CAP_ARM_MTE: - mutex_lock(&kvm->lock); - if (system_supports_mte() && !kvm->created_vcpus) { - r = 0; - set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); + case KVM_CAP_ARM_MTE: { + struct kvm_memory_slot *memslot; + int bkt; + + guard(mutex)(&kvm->lock); + if (!system_supports_mte() || kvm->created_vcpus) + break; + + r = 0; + guard(mutex)(&kvm->slots_lock); + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) { + if (kvm_slot_has_gmem(memslot)) { + r = -EINVAL; + break; + } } - mutex_unlock(&kvm->lock); + if (r == 0) + set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); break; + + } case KVM_CAP_ARM_SYSTEM_SUSPEND: r = 0; set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index b8ded434c63f..640f2dc00a8b 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,14 +11,6 @@ #include <asm/kvm_mmu.h> #include <asm/lsui.h> -static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) -{ - wr->fst = fst; - wr->ptw = s1ptw; - wr->s2 = s1ptw; - wr->failed = true; -} - #define S1_MMU_DISABLED (-127) static int get_ia_size(struct s1_walk_info *wi) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index e688bc5139c1..3c82f392845d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2746,17 +2746,33 @@ static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr) (spsr & PSR_MODE32_BIT) || (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t || mode == PSR_MODE_EL1h))) { + u64 mask; + /* - * The guest is playing with our nerves. Preserve EL, SP, - * masks, flags from the existing PSTATE, and set IL. - * The HW will then generate an Illegal State Exception - * immediately after ERET. + * On an illegal exception return, the flags and masks are + * taken from the SPSR while PSTATE.{EL,SP,nRW} and, if + * FEAT_GCS, PSTATE.EXLOCK are unchanged (R_VWJHB). Set IL + * so the HW generates an Illegal State Exception right + * after ERET. */ - spsr = *vcpu_cpsr(vcpu); + mask = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | + PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + + if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, IMP)) + mask |= PSR_PAN_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, NMI, IMP)) + mask |= ALLINT_ALLINT; + /* FEAT_SPE_EXC and FEAT_TRBE_EXC also gate PSTATE.PM one day... */ + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR1_EL1, EBEP, IMP)) + mask |= BIT_ULL(32); /* SPSR_ELx.PM */ + + spsr &= mask; - spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | - PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT | - PSR_MODE_MASK | PSR_MODE32_BIT); + mask = PSR_MODE_MASK | PSR_MODE32_BIT; + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP)) + mask |= BIT_ULL(34); /* PSTATE.EXLOCK */ + + spsr |= *vcpu_cpsr(vcpu) & mask; spsr |= PSR_IL_BIT; } @@ -2784,7 +2800,7 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) * ERET handling, and the guest will have a little surprise. */ if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) { - esr &= ESR_ELx_ERET_ISS_ERETA; + esr &= (ESR_ELx_ERET_ISS_ERETA | ESR_ELx_IL); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC); kvm_inject_nested_sync(vcpu, esr); return; @@ -2826,6 +2842,7 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2, break; case except_type_serror: kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR); + vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2); break; default: WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type); @@ -2950,6 +2967,6 @@ int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr) * vSError injection. Manually populate EC for an emulated SError * exception. */ - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; return kvm_inject_nested(vcpu, esr, except_type_serror); } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 18131e395e24..4bf624a49591 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -448,16 +448,19 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) { + u64 spsr; + *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* * Finish potential single step before executing the prologue - * instruction. + * instruction. Modify the hardware SPSR_EL2 directly, as vcpu_cpsr() + * may hold a synthetic (vEL2) value for a guest hypervisor. */ - *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; - write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); + spsr = read_sysreg_el2(SYS_SPSR); + write_sysreg_el2(spsr & ~DBG_SPSR_SS, SYS_SPSR); return true; } @@ -602,8 +605,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) return false; break; case ESR_ELx_EC_SYS64: - if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu))) - return false; fallthrough; case ESR_ELx_EC_SVE: if (!sve_guest) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 1af722771178..a327c2bbb6b6 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -352,7 +352,7 @@ static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE)) @@ -372,7 +372,7 @@ static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; u64 pfn = hyp_phys_to_pfn(range->address); - if (!PAGE_ALIGNED(sz)) + if (!PAGE_ALIGNED(sz | range->address)) break; if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE)) @@ -476,11 +476,12 @@ static void __do_ffa_mem_xfer(const u64 func_id, DECLARE_REG(u32, fraglen, ctxt, 2); DECLARE_REG(u64, addr_mbz, ctxt, 3); DECLARE_REG(u32, npages_mbz, ctxt, 4); + u32 offset, nr_ranges, checked_offset, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; struct ffa_mem_region *buf; - u32 offset, nr_ranges, checked_offset; int ret = 0; + size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { @@ -488,8 +489,7 @@ static void __do_ffa_mem_xfer(const u64 func_id, goto out; } - if (fraglen < sizeof(struct ffa_mem_region) + - sizeof(struct ffa_mem_region_attributes)) { + if (fraglen < mem_region_len + ffa_emad_size_get(hyp_ffa_version)) { ret = FFA_RET_INVALID_PARAMETERS; goto out; } @@ -508,8 +508,13 @@ static void __do_ffa_mem_xfer(const u64 func_id, buf = hyp_buffers.tx; memcpy(buf, host_buffers.tx, fraglen); - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; @@ -574,9 +579,9 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, handle_lo, ctxt, 1); DECLARE_REG(u32, handle_hi, ctxt, 2); DECLARE_REG(u32, flags, ctxt, 3); + u32 offset, len, fraglen, fragoff, em_mem_access_off; struct ffa_mem_region_attributes *ep_mem_access; struct ffa_composite_mem_region *reg; - u32 offset, len, fraglen, fragoff; struct ffa_mem_region *buf; int ret = 0; u64 handle; @@ -599,16 +604,22 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, len = res->a1; fraglen = res->a2; - ep_mem_access = (void *)buf + - ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version); + if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) { + ret = FFA_RET_INVALID_PARAMETERS; + ffa_rx_release(res); + goto out_unlock; + } + + ep_mem_access = (void *)buf + em_mem_access_off; offset = ep_mem_access->composite_off; /* * We can trust the SPMD to get this right, but let's at least * check that we end up with something that doesn't look _completely_ * bogus. */ - if (WARN_ON(offset > len || - fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) { + if (offset + CONSTITUENTS_OFFSET(0) > len || + fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_ABORTED; ffa_rx_release(res); goto out_unlock; @@ -636,11 +647,16 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, ffa_rx_release(res); } + reg = (void *)buf + offset; + if (offset + CONSTITUENTS_OFFSET(reg->addr_range_cnt) > len) { + ret = FFA_RET_ABORTED; + goto out_unlock; + } + ffa_mem_reclaim(res, handle_lo, handle_hi, flags); if (res->a0 != FFA_SUCCESS) goto out_unlock; - reg = (void *)buf + offset; /* If the SPMD was happy, then we should be too. */ WARN_ON(ffa_host_unshare_ranges(reg->constituents, reg->addr_range_cnt)); @@ -864,7 +880,7 @@ out_unlock: bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { - struct arm_smccc_1_2_regs res; + struct arm_smccc_1_2_regs res = {0}; /* * There's no way we can tell what a non-standard SMC call might diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d3c69de698f4..d3df96ed8ba4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -932,6 +932,7 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) handle_host_mem_abort(host_ctxt); break; case ESR_ELx_EC_SYS64: + trace_hyp_enter(host_ctxt, HYP_REASON_SYS); if (handle_host_mte(esr)) break; fallthrough; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3b2c4fbc34d8..24d6f164129a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -1056,7 +1056,8 @@ static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa) /* Fake up a data abort (level 3 translation fault on write) */ vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | - ESR_ELx_WNR | ESR_ELx_FSC_FAULT | + ESR_ELx_IL | ESR_ELx_WNR | + ESR_ELx_FSC_FAULT | FIELD_PREP(ESR_ELx_FSC_LEVEL, 3); /* Shuffle the IPA around into the HPFAR */ diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8c3fbb413a06..b1411fb54139 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -268,6 +268,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) write_sysreg_el1(esr, SYS_ESR); write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR); + write_sysreg_el1(read_sysreg_el2(SYS_SPSR), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); } @@ -278,7 +279,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) */ static void inject_undef64(struct kvm_vcpu *vcpu) { - inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT)); + inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL); } static u64 read_id_reg(const struct kvm_vcpu *vcpu, diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 91a7dfad6686..b74dd5ce1efd 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1370,16 +1370,19 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, if (prot & KVM_PGTABLE_PROT_W) set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W; - ret = stage2_set_xn_attr(prot, &xn); - if (ret) - return ret; + if (prot & KVM_PGTABLE_PROT_X) { + ret = stage2_set_xn_attr(prot, &xn); + if (ret) + return ret; - set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; - clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN; + } ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags); if (!ret || ret == -EAGAIN) - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, + (ret == -EAGAIN) ? TLBI_TTL_UNKNOWN : level); return ret; } diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 2411b4c32932..9644c424819b 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -37,8 +37,6 @@ static struct hyp_trace_clock { u32 shift; struct delayed_work work; struct completion ready; - struct mutex lock; - bool running; } hyp_clock; static void __hyp_clock_work(struct work_struct *work) @@ -110,12 +108,9 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl { struct system_time_snapshot snap; - if (hyp_clock->running == enable) - return; - if (!enable) { cancel_delayed_work_sync(&hyp_clock->work); - hyp_clock->running = false; + return; } ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); @@ -128,7 +123,6 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl INIT_DELAYED_WORK(&hyp_clock->work, __hyp_clock_work); schedule_delayed_work(&hyp_clock->work, msecs_to_jiffies(CLOCK_INIT_MS)); wait_for_completion(&hyp_clock->ready); - hyp_clock->running = true; } /* Access to this struct within the trace_remote_callbacks are protected by the trace_remote lock */ @@ -160,6 +154,7 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; size_t backing_size; void *start; + int ret; backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); @@ -168,10 +163,16 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ if (!start) return -ENOMEM; + ret = __map_hyp(start, backing_size); + if (ret) { + free_pages_exact(start, backing_size); + return ret; + } + trace_buffer->desc->bpages_backing_start = (unsigned long)start; trace_buffer->desc->bpages_backing_size = backing_size; - return __map_hyp(start, backing_size); + return ret; } static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) @@ -228,18 +229,22 @@ static int hyp_trace_buffer_share_hyp(struct hyp_trace_buffer *trace_buffer) static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) { struct hyp_trace_buffer *trace_buffer = priv; + size_t desc_size, tb_desc_size; struct hyp_trace_desc *desc; - size_t desc_size; int ret; if (WARN_ON(trace_buffer->desc)) return ERR_PTR(-EINVAL); - desc_size = trace_buffer_desc_size(size, num_possible_cpus()); + tb_desc_size = trace_buffer_desc_size(size, num_possible_cpus()); + desc_size = size_add(tb_desc_size, offsetof(struct hyp_trace_desc, trace_buffer_desc)); if (desc_size == SIZE_MAX) return ERR_PTR(-E2BIG); desc_size = PAGE_ALIGN(desc_size); + if (!desc_size) + return ERR_PTR(-E2BIG); + desc = (struct hyp_trace_desc *)alloc_pages_exact(desc_size, GFP_KERNEL); if (!desc) return ERR_PTR(-ENOMEM); @@ -255,7 +260,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_desc; - ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, desc_size, size, + ret = trace_remote_alloc_buffer(&desc->trace_buffer_desc, tb_desc_size, size, cpu_possible_mask); if (ret) goto err_free_backing; @@ -304,9 +309,15 @@ static void hyp_trace_unload(struct trace_buffer_desc *desc, void *priv) static int hyp_trace_enable_tracing(bool enable, void *priv) { + int ret; + hyp_trace_clock_enable(&hyp_clock, enable); - return kvm_call_hyp_nvhe(__tracing_enable, enable); + ret = kvm_call_hyp_nvhe(__tracing_enable, enable); + if (ret) + hyp_trace_clock_enable(&hyp_clock, !enable); + + return ret; } static int hyp_trace_swap_reader_page(unsigned int cpu, void *priv) @@ -398,6 +409,7 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) static const char strs[][12] = { "smc", "hvc", + "sys", "psci", "host_abort", "guest_exit", diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 89982bd3345f..d6c4fc16f879 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -138,11 +138,10 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr pend_sync_exception(vcpu); /* - * Build an {i,d}abort, depending on the level and the - * instruction set. Report an external synchronous abort. + * Build an {i,d}abort, depending on the level. + * Report an external synchronous abort. */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + esr |= ESR_ELx_IL; /* * Here, the guest runs in AArch64 mode when in EL1. If we get @@ -170,14 +169,7 @@ void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr) static void inject_undef64(struct kvm_vcpu *vcpu) { - u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); - - /* - * Build an unknown exception, depending on the instruction - * set. - */ - if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_ELx_IL; + u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; kvm_inject_sync(vcpu, esr); } @@ -389,7 +381,7 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) */ if (!serror_is_masked(vcpu)) { pend_serror_exception(vcpu); - esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR); + esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); return 1; } diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index e2285ed8c91d..d1c3a352d5a2 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -126,6 +126,10 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) len = kvm_vcpu_dabt_get_as(vcpu); data = kvm_mmio_read_buf(run->mmio.data, len); + trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, + &data); + data = vcpu_data_host_to_guest(vcpu, data, len); + if (kvm_vcpu_dabt_issext(vcpu) && len < sizeof(unsigned long)) { mask = 1U << ((len * 8) - 1); @@ -135,9 +139,6 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) if (!kvm_vcpu_dabt_issf(vcpu)) data = data & 0xffffffff; - trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, - &data); - data = vcpu_data_host_to_guest(vcpu, data, len); vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data); } diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6c941aaa10c6..2d95203386ba 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2652,6 +2652,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (kvm_slot_has_gmem(new) && !kvm_memslot_is_gmem_only(new)) return -EINVAL; + /* guest_memfd is incompatible with MTE. */ + if (kvm_slot_has_gmem(new) && kvm_has_mte(kvm)) + return -EINVAL; + hva = new->userspace_addr; reg_end = hva + (new->npages << PAGE_SHIFT); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..dfb96edbdc43 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -24,6 +24,7 @@ struct vncr_tlb { struct s1_walk_result wr; u64 hpa; + bool hpa_writable; /* -1 when not mapped on a CPU */ int cpu; @@ -1401,15 +1402,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); - if (!memslot) + if (!memslot) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } *is_gmem = kvm_slot_has_gmem(memslot); if (!*is_gmem) { pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0, &writable, &page); - if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) + if (is_error_noslot_pfn(pfn)) { + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); return -EFAULT; + } } else { ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); if (ret) { @@ -1417,6 +1422,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) write_fault, false, false); return ret; } + + writable = !(memslot->flags & KVM_MEM_READONLY); + } + + /* + * FIXME: This check is too restrictive as KVM allows cacheable memory + * attributes for PFNMAP VMAs that have cacheable attributes in host + * stage-1. + */ + if (!pfn_is_map_memory(pfn)) { + kvm_release_faultin_page(vcpu->kvm, page, true, false); + fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false); + return -EINVAL; } scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { @@ -1427,116 +1445,100 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->gva = va; vt->hpa = pfn << PAGE_SHIFT; + vt->hpa_writable = writable; vt->valid = true; vt->cpu = -1; kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); - kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); + kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); } - if (vt->wr.pw) + if (vt->wr.pw && vt->hpa_writable) mark_page_dirty(vcpu->kvm, gfn); return 0; } -static void inject_vncr_perm(struct kvm_vcpu *vcpu) +static void handle_vncr_perm(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + u64 fsc; + + /* + * Promote to an external abort if the stage-1 permits writes but the + * HPA is read-only (e.g. RO memslot). + */ + if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable) + fsc = ESR_ELx_FSC_EXTABT; + /* + * Otherwise, inject a permission fault using the guest's translation + * level rather than the host's. + */ + else + fsc = ESR_ELx_FSC_PERM_L(vt->wr.level); - /* Adjust the fault level to reflect that of the guest's */ esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, - ESR_ELx_FSC_PERM_L(vt->wr.level)); + esr |= FIELD_PREP(ESR_ELx_FSC, fsc); kvm_inject_nested_sync(vcpu, esr); } -static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) -{ - struct vncr_tlb *vt = vcpu->arch.vncr_tlb; - - lockdep_assert_held_read(&vcpu->kvm->mmu_lock); - - if (!vt->valid) - return false; - - if (read_vncr_el2(vcpu) != vt->gva) - return false; - - if (vt->wr.nG) - return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid; - - return true; -} - int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) { struct vncr_tlb *vt = vcpu->arch.vncr_tlb; u64 esr = kvm_vcpu_get_esr(vcpu); + bool is_gmem = false; + bool perm; + int ret; WARN_ON_ONCE(!(esr & ESR_ELx_VNCR)); if (kvm_vcpu_abt_issea(vcpu)) return kvm_handle_guest_sea(vcpu); - if (esr_fsc_is_permission_fault(esr)) { - inject_vncr_perm(vcpu); - } else if (esr_fsc_is_translation_fault(esr)) { - bool valid, is_gmem = false; - int ret; - - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - valid = kvm_vncr_tlb_lookup(vcpu); - - if (!valid) - ret = kvm_translate_vncr(vcpu, &is_gmem); - else - ret = -EPERM; - - switch (ret) { - case -EAGAIN: - /* Let's try again... */ - break; - case -ENOMEM: - /* - * For guest_memfd, this indicates that it failed to - * create a folio to back the memory. Inform userspace. - */ - if (is_gmem) - return 0; - /* Otherwise, let's try again... */ - break; - case -EFAULT: - case -EIO: - case -EHWPOISON: - if (is_gmem) - return 0; - fallthrough; - case -EINVAL: - case -ENOENT: - case -EACCES: - /* - * Translation failed, inject the corresponding - * exception back to EL2. - */ - BUG_ON(!vt->wr.failed); + if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) { + KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr); + return -EIO; + } - esr &= ~ESR_ELx_FSC; - esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); + ret = kvm_translate_vncr(vcpu, &is_gmem); + switch (ret) { + case -EAGAIN: + /* Let's try again... */ + return 1; + case -ENOMEM: + /* + * For guest_memfd, this indicates that it failed to + * create a folio to back the memory. Inform userspace. + */ + if (is_gmem) + return 0; + /* Otherwise, let's try again... */ + break; + case -EFAULT: + case -EIO: + case -EHWPOISON: + if (is_gmem) + return 0; + fallthrough; + case -EINVAL: + case -ENOENT: + case -EACCES: + /* + * Translation failed, inject the corresponding + * exception back to EL2. + */ + esr &= ~ESR_ELx_FSC; + esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); - kvm_inject_nested_sync(vcpu, esr); - break; - case -EPERM: - /* Hack to deal with POE until we get kernel support */ - inject_vncr_perm(vcpu); - break; - case 0: - break; - } - } else { - WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); + kvm_inject_nested_sync(vcpu, esr); + break; + case 0: + perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr; + if (!perm) + handle_vncr_perm(vcpu); + break; } return 1; @@ -1574,7 +1576,7 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) vt->cpu = smp_processor_id(); - if (vt->wr.pw && vt->wr.pr) + if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) prot = PAGE_KERNEL_RO; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..428723b1b0f5 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -352,7 +352,7 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64 page = pfn_to_page(mapping->pfn); WARN_ON_ONCE(mapping->nr_pages != 1); unpin_user_pages_dirty_lock(&page, 1, true); - account_locked_vm(current->mm, 1, false); + account_locked_vm(kvm->mm, 1, false); pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); kfree(mapping); } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 4477f870c7b3..36ab3e492915 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -116,18 +116,27 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, kfree(irq); irq = oldirq; } else { - ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0)); + /* + * The entry is either empty or contains a dead LPI (refcount=0) + * from the deferred release path, pending cleanup by + * vgic_release_deleted_lpis(). Evict and free it if present. + */ + oldirq = __xa_store(&dist->lpi_xa, intid, irq, + GFP_NOWAIT | __GFP_ACCOUNT); + ret = xa_err(oldirq); + if (ret) { + xa_unlock_irqrestore(&dist->lpi_xa, flags); + kfree(irq); + + return ERR_PTR(ret); + } + + if (oldirq && !WARN_ON_ONCE(refcount_read(&oldirq->refcount))) + kfree_rcu(oldirq, rcu); } xa_unlock_irqrestore(&dist->lpi_xa, flags); - if (ret) { - xa_release(&dist->lpi_xa, intid); - kfree(irq); - - return ERR_PTR(ret); - } - /* * We "cache" the configuration table entries in our struct vgic_irq's. * However we only have those structs for mapped IRQs, so we read in @@ -508,6 +517,8 @@ static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db) struct kvm_io_device *kvm_io_dev; struct vgic_io_device *iodev; + guard(srcu)(&kvm->srcu); + kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db); if (!kvm_io_dev) return ERR_PTR(-EINVAL); diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9e841e7afd4a..7aa417440f6a 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -275,7 +275,13 @@ void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val) lr = vgic_v3_compute_lr(vcpu, irq) & ~ICH_LR_ACTIVE_BIT; } - if (lr & ICH_LR_HW) + /* + * In the nested state, the irq has already been deactivated via the HW + * bit in the LR. Deactivating again would be harmless except AmpereOne + * errata AC03_CPU_57, AC04_CPU_29 could cause irq delivery to break if + * the deactivation hits the highest priority pending irq. + */ + if ((lr & ICH_LR_HW) && !vgic_state_is_nested(vcpu)) vgic_v3_deactivate_phys(FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); vgic_v3_fold_lr(vcpu, lr); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5a4768d8cd4f..74bace10a22e 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -146,11 +146,7 @@ static __must_check bool __vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq) { - if (!__vgic_put_irq(kvm, irq)) - return false; - - irq->pending_release = true; - return true; + return __vgic_put_irq(kvm, irq); } void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) @@ -167,12 +163,14 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq) guard(spinlock_irqsave)(&dist->lpi_xa.xa_lock); } - if (!__vgic_put_irq(kvm, irq)) + if (!irq_is_lpi(kvm, irq->intid)) return; - xa_lock_irqsave(&dist->lpi_xa, flags); - vgic_release_lpi_locked(dist, irq); - xa_unlock_irqrestore(&dist->lpi_xa, flags); + if (refcount_dec_and_lock_irqsave(&irq->refcount, + &dist->lpi_xa.xa_lock, &flags)) { + vgic_release_lpi_locked(dist, irq); + xa_unlock_irqrestore(&dist->lpi_xa, flags); + } } static void vgic_release_deleted_lpis(struct kvm *kvm) @@ -184,7 +182,7 @@ static void vgic_release_deleted_lpis(struct kvm *kvm) xa_lock_irqsave(&dist->lpi_xa, flags); xa_for_each(&dist->lpi_xa, intid, irq) { - if (irq->pending_release) + if (!refcount_read(&irq->refcount)) vgic_release_lpi_locked(dist, irq); } @@ -203,6 +201,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu) list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) { if (irq_is_lpi(vcpu->kvm, irq->intid)) { raw_spin_lock(&irq->irq_lock); + irq->pending_latch = false; list_del(&irq->ap_list); irq->vcpu = NULL; raw_spin_unlock(&irq->irq_lock); @@ -792,7 +791,11 @@ retry: continue; } - /* This interrupt looks like it has to be migrated. */ + /* + * This interrupt looks like it has to be migrated, + * make sure it is kept alive while locks are dropped. + */ + vgic_get_irq_ref(irq); raw_spin_unlock(&irq->irq_lock); raw_spin_unlock(&vgic_cpu->ap_list_lock); @@ -815,15 +818,16 @@ retry: raw_spin_lock(&irq->irq_lock); /* - * If the affinity has been preserved, move the - * interrupt around. Otherwise, it means things have - * changed while the interrupt was unlocked, and we - * need to replay this. + * If the interrupt is still ours and its affinity has + * been preserved, move it around. Otherwise, it means + * things have changed while the interrupt was unlocked + * (it may even have been taken off the list with its + * affinity left untouched), and we need to replay this. * * In all cases, we cannot trust the list not to have * changed, so we restart from the beginning. */ - if (target_vcpu == vgic_target_oracle(irq)) { + if (irq->vcpu == vcpu && target_vcpu == vgic_target_oracle(irq)) { struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu; list_del(&irq->ap_list); @@ -836,6 +840,8 @@ retry: raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock); raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock); + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + if (target_vcpu_needs_kick) { kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu); kvm_vcpu_kick(target_vcpu); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 85e23388f9bb..0b52557652be 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -76,6 +76,8 @@ static void data_abort_decode(unsigned long esr) pr_alert(" SF = %lu, AR = %lu\n", (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); + pr_alert(" Xs = %llu\n", + (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT); } else { pr_alert(" ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n", esr & ESR_ELx_ISS_MASK, iss2); @@ -87,11 +89,10 @@ static void data_abort_decode(unsigned long esr) (iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT, (iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT); - pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu, Xs = %llu\n", + pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu\n", (iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT, (iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT, - (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT, - (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT); + (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT); } static void mem_abort_decode(unsigned long esr) diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 30772a909aea..8e799c1fe0aa 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -87,7 +87,7 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, p4dp = p4d_offset(pgdp, addr); pudp = pud_offset(p4dp, addr); pmdp = pmd_offset(pudp, addr); - if ((pte_t *)pmdp == ptep) { + if ((pte_t *)PTR_ALIGN_DOWN(pmdp, sizeof(*pmdp) * CONT_PMDS) == ptep) { *pgsize = PMD_SIZE; return CONT_PMDS; } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f2be501468ce..d4de88770ecf 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -49,8 +49,6 @@ #define NO_CONT_MAPPINGS BIT(1) #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ -DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key); - u64 kimage_voffset __ro_after_init; EXPORT_SYMBOL(kimage_voffset); @@ -1515,7 +1513,13 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, if (free_mapped) { /* CONT blocks are not supported in the vmemmap */ WARN_ON(pmd_cont(pmd)); - flush_tlb_kernel_range(addr, addr + PMD_SIZE); + /* + * Invalidating a block entry requires just + * a single overlapping TLB invalidation, + * so limit the range of the flush to a single + * page. + */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pmd_page(pmd), PMD_SIZE, altmap); } @@ -1545,7 +1549,8 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, if (pud_leaf(pud)) { pud_clear(pudp); if (free_mapped) { - flush_tlb_kernel_range(addr, addr + PUD_SIZE); + /* See comment in unmap_hotplug_pmd_range(). */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pud_page(pud), PUD_SIZE, altmap); } @@ -1857,8 +1862,7 @@ int pmd_clear_huge(pmd_t *pmdp) return 1; } -static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr, - bool acquire_mmap_lock) +int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) { pte_t *table; pmd_t pmd; @@ -1870,25 +1874,13 @@ static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr, return 1; } - /* See comment in pud_free_pmd_page for static key logic */ table = pte_offset_kernel(pmdp, addr); pmd_clear(pmdp); __flush_tlb_kernel_pgtable(addr); - if (static_branch_unlikely(&arm64_ptdump_lock_key) && acquire_mmap_lock) { - mmap_read_lock(&init_mm); - mmap_read_unlock(&init_mm); - } - pte_free_kernel(NULL, table); return 1; } -int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) -{ - /* If ptdump is walking the pagetables, acquire init_mm.mmap_lock */ - return __pmd_free_pte_page(pmdp, addr, /* acquire_mmap_lock = */ true); -} - int pud_free_pmd_page(pud_t *pudp, unsigned long addr) { pmd_t *table; @@ -1904,36 +1896,16 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr) } table = pmd_offset(pudp, addr); - - /* - * Our objective is to prevent ptdump from reading a PMD table which has - * been freed. In this race, if pud_free_pmd_page observes the key on - * (which got flipped by ptdump) then the mmap lock sequence here will, - * as a result of the mmap write lock/unlock sequence in ptdump, give - * us the correct synchronization. If not, this means that ptdump has - * yet not started walking the pagetables - the sequence of barriers - * issued by __flush_tlb_kernel_pgtable() guarantees that ptdump will - * observe an empty PUD. - */ - pud_clear(pudp); - __flush_tlb_kernel_pgtable(addr); - if (static_branch_unlikely(&arm64_ptdump_lock_key)) { - mmap_read_lock(&init_mm); - mmap_read_unlock(&init_mm); - } - pmdp = table; next = addr; end = addr + PUD_SIZE; do { if (pmd_present(pmdp_get(pmdp))) - /* - * PMD has been isolated, so ptdump won't see it. No - * need to acquire init_mm.mmap_lock. - */ - __pmd_free_pte_page(pmdp, next, /* acquire_mmap_lock = */ false); + pmd_free_pte_page(pmdp, next); } while (pmdp++, next += PMD_SIZE, next != end); + pud_clear(pudp); + __flush_tlb_kernel_pgtable(addr); pmd_free(NULL, table); return 1; } @@ -2187,7 +2159,7 @@ static int prevent_memory_remove_notifier(struct notifier_block *nb, } } - if (!can_unmap_without_split(pfn, arg->nr_pages)) + if (!can_unmap_without_split(arg->start_pfn, arg->nr_pages)) return NOTIFY_BAD; return NOTIFY_OK; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index 1c20144700d7..5a76c59b5ada 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -283,13 +283,6 @@ void note_page_flush(struct ptdump_state *pt_st) note_page(pt_st, 0, -1, pte_val(pte_zero)); } -static void arm64_ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm) -{ - static_branch_inc(&arm64_ptdump_lock_key); - ptdump_walk_pgd(st, mm, NULL); - static_branch_dec(&arm64_ptdump_lock_key); -} - void ptdump_walk(struct seq_file *s, struct ptdump_info *info) { unsigned long end = ~0UL; @@ -318,7 +311,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info) } }; - arm64_ptdump_walk_pgd(&st.ptdump, info->mm); + ptdump_walk_pgd(&st.ptdump, info->mm, NULL); } static void __init ptdump_initialize(void) @@ -360,7 +353,7 @@ bool ptdump_check_wx(void) } }; - arm64_ptdump_walk_pgd(&st.ptdump, &init_mm); + ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); if (st.wx_pages || st.uxn_pages) { pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n", diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index bc1788b1662b..7cb61aca3797 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -1806,7 +1806,7 @@ Res0 15:8 UnsignedEnum 7:4 BWE 0b0000 NI 0b0001 FEAT_BWE - 0b0002 FEAT_BWE2 + 0b0010 FEAT_BWE2 EndEnum UnsignedEnum 3:0 STEP 0b0000 NI diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index a648be5f723f..b7423d1ac568 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -14,7 +14,7 @@ #ifdef CONFIG_64BIT #define TASK_STRUCT_OFFSET 0 #else -#define TASK_STRUCT_OFFSET 2000 +#define TASK_STRUCT_OFFSET 2040 #endif .macro cpu_save_nonscratch thread diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 8f650c9ffecd..873e90990771 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -201,10 +201,12 @@ static void __init acpi_process_madt(void) } int pptt_enabled; +static int acpi_nr_packages; +static int acpi_package_ids[MAX_PACKAGES]; int __init parse_acpi_topology(void) { - int cpu, topology_id; + int i, cpu, topology_id; for_each_possible_cpu(cpu) { topology_id = find_acpi_cpu_topology(cpu, 0); @@ -222,6 +224,29 @@ int __init parse_acpi_topology(void) cpu_data[cpu].core = topology_id; } + + topology_id = find_acpi_cpu_topology_package(cpu); + if (topology_id < 0) { + pr_warn("Invalid BIOS PPTT\n"); + return -ENOENT; + } + + for (i = 0; i < acpi_nr_packages; i++) + if (acpi_package_ids[i] == topology_id) + break; + + if (i == acpi_nr_packages) + acpi_package_ids[acpi_nr_packages++] = topology_id; + + cpu_data[cpu].package = topology_id; + } + + for_each_possible_cpu(cpu) { + for (i = 0; i < acpi_nr_packages; i++) + if (cpu_data[cpu].package == acpi_package_ids[i]) { + cpu_data[cpu].package = i; /* Canonicalize */ + break; + } } pptt_enabled = 1; diff --git a/arch/loongarch/kernel/kgdb.c b/arch/loongarch/kernel/kgdb.c index 17664a6043b1..e7b59f8a4b05 100644 --- a/arch/loongarch/kernel/kgdb.c +++ b/arch/loongarch/kernel/kgdb.c @@ -252,7 +252,8 @@ static int kgdb_loongarch_notify(struct notifier_block *self, unsigned long cmd, if (atomic_read(&kgdb_active) != -1) kgdb_nmicallback(smp_processor_id(), regs); - if (kgdb_handle_exception(args->trapnr, args->signr, cmd, regs)) + if (kgdb_handle_exception(regs->csr_era == stepped_address ? 0 : args->trapnr, + args->signr, cmd, regs)) return NOTIFY_DONE; if (atomic_read(&kgdb_setting_breakpoint)) diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c index d7fafda1d541..1883cae93bc3 100644 --- a/arch/loongarch/kernel/machine_kexec.c +++ b/arch/loongarch/kernel/machine_kexec.c @@ -42,6 +42,7 @@ static unsigned long first_ind_entry; int machine_kexec_prepare(struct kimage *kimage) { int i; + char head[8]; char *bootloader = "kexec"; void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; @@ -59,7 +60,9 @@ int machine_kexec_prepare(struct kimage *kimage) } else { /* Find the command line */ for (i = 0; i < kimage->nr_segments; i++) { - if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { + if (copy_from_user(head, kimage->segment[i].buf, strlen(bootloader))) + continue; + if (!strncmp(bootloader, head, strlen(bootloader))) { if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; break; diff --git a/arch/loongarch/kernel/rethook_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S index d4ceb2fa2a5c..2e009fbea53f 100644 --- a/arch/loongarch/kernel/rethook_trampoline.S +++ b/arch/loongarch/kernel/rethook_trampoline.S @@ -71,27 +71,27 @@ cfi_ld s7, PT_R30 cfi_ld s8, PT_R31 LONG_L t0, sp, PT_CRMD - li.d t1, 0x7 /* mask bit[1:0] PLV, bit[2] IE */ + LONG_LI t1, 0x7 /* mask bit[1:0] PLV, bit[2] IE */ csrxchg t0, t1, LOONGARCH_CSR_CRMD .endm SYM_CODE_START(arch_rethook_trampoline) UNWIND_HINT_UNDEFINED - addi.d sp, sp, -PT_SIZE + PTR_ADDI sp, sp, -PT_SIZE save_all_base_regs - addi.d t0, sp, PT_SIZE - LONG_S t0, sp, PT_R3 + PTR_ADDI t0, sp, PT_SIZE + LONG_S t0, sp, PT_R3 - move a0, sp /* pt_regs */ + move a0, sp /* pt_regs */ - bl arch_rethook_trampoline_callback + bl arch_rethook_trampoline_callback /* use the result as the return-address */ - move ra, a0 + move ra, a0 restore_all_base_regs - addi.d sp, sp, PT_SIZE + PTR_ADDI sp, sp, PT_SIZE - jr ra + jr ra SYM_CODE_END(arch_rethook_trampoline) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index eaebb52bd36e..6fa4a22a58fd 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -603,6 +603,7 @@ void __init setup_arch(char **cmdline_p) memblock_init(); pagetable_init(); bootcmdline_init(cmdline_p); + jump_label_init(); /* Initialise the static keys for early params */ parse_early_param(); reserve_initrd_mem(); @@ -610,8 +611,6 @@ void __init setup_arch(char **cmdline_p) arch_mem_init(cmdline_p); resource_init(); - jump_label_init(); /* Initialise the static keys for paravirtualization */ - #ifdef CONFIG_SMP plat_smp_setup(); prefill_possible_map(); diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 5d792256bbb9..d4b5d1b6bb01 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -426,10 +426,10 @@ void loongson_init_secondary(void) numa_add_cpu(cpu); #endif per_cpu(cpu_state, cpu) = CPU_ONLINE; - cpu_data[cpu].package = - cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; + cpu_data[cpu].package = pptt_enabled ? cpu_data[cpu].package : + cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; cpu_data[cpu].global_id = cpu_logical_map(cpu); } diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 2738b4db1165..29c281bef28e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -835,7 +835,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext move_reg(ctx, t1, src); emit_sext_32(ctx, t1, is32); emit_insn(ctx, divd, dst, dst, t1); - emit_sext_32(ctx, dst, is32); + emit_zext_32(ctx, dst, is32); } break; @@ -852,7 +852,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext emit_sext_32(ctx, t1, is32); emit_sext_32(ctx, dst, is32); emit_insn(ctx, divd, dst, dst, t1); - emit_sext_32(ctx, dst, is32); + emit_zext_32(ctx, dst, is32); } break; @@ -870,7 +870,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext move_reg(ctx, t1, src); emit_sext_32(ctx, t1, is32); emit_insn(ctx, modd, dst, dst, t1); - emit_sext_32(ctx, dst, is32); + emit_zext_32(ctx, dst, is32); } break; @@ -887,7 +887,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext emit_sext_32(ctx, t1, is32); emit_sext_32(ctx, dst, is32); emit_insn(ctx, modd, dst, dst, t1); - emit_sext_32(ctx, dst, is32); + emit_zext_32(ctx, dst, is32); } break; @@ -2361,6 +2361,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kvfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 148f8a79d206..5a58cc13229e 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -549,6 +549,10 @@ config CACHE_COPYBACK endchoice endif # HAVE_CACHE_CB +config NR_CPUS + int + default "1" + # Coldfire cores that do not have a data cache configured can do coherent DMA. config COLDFIRE_COHERENT_DMA bool diff --git a/arch/m68k/coldfire/m523x.c b/arch/m68k/coldfire/m523x.c index 11d7423ef646..ec04c32bb03c 100644 --- a/arch/m68k/coldfire/m523x.c +++ b/arch/m68k/coldfire/m523x.c @@ -79,7 +79,7 @@ static void __init m523x_i2c_init(void) static void __init m523x_fec_init(void) { /* Set multi-function pins to ethernet use */ - mcf_write8(read8(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C); + mcf_write8(mcf_read8(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C); } /***************************************************************************/ diff --git a/arch/m68k/coldfire/m528x.c b/arch/m68k/coldfire/m528x.c index b244c9ba40a7..f9874bba9e45 100644 --- a/arch/m68k/coldfire/m528x.c +++ b/arch/m68k/coldfire/m528x.c @@ -110,14 +110,14 @@ void wildfiremod_halt(void) printk(KERN_INFO "WildFireMod hibernating...\n"); /* Set portE.5 to Digital IO */ - mcf_write16(read16(MCFGPIO_PEPAR) & ~(1 << (5 * 2)), MCFGPIO_PEPAR); + mcf_write16(mcf_read16(MCFGPIO_PEPAR) & ~(1 << (5 * 2)), MCFGPIO_PEPAR); /* Make portE.5 an output */ - mcf_write8(read8(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E); + mcf_write8(mcf_read8(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E); /* Now toggle portE.5 from low to high */ - mcf_write8(read8(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E); - mcf_write8(read8(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E); + mcf_write8(mcf_read8(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E); + mcf_write8(mcf_read8(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E); printk(KERN_EMERG "Failed to hibernate. Halting!\n"); } diff --git a/arch/m68k/include/asm/page_mm.h b/arch/m68k/include/asm/page_mm.h index ed782609ca41..0971a0651d49 100644 --- a/arch/m68k/include/asm/page_mm.h +++ b/arch/m68k/include/asm/page_mm.h @@ -55,10 +55,12 @@ static inline void clear_page(void *page) #define clear_user_page(addr, vaddr, page) \ do { clear_page(addr); \ flush_dcache_page(page); \ + (void)(vaddr); \ } while (0) #define copy_user_page(to, from, vaddr, page) \ do { copy_page(to, from); \ flush_dcache_page(page); \ + (void)(vaddr); \ } while (0) extern unsigned long m68k_memoffset; diff --git a/arch/microblaze/kernel/head.S b/arch/microblaze/kernel/head.S index 808019c3b7ac..9bd3e513c89b 100644 --- a/arch/microblaze/kernel/head.S +++ b/arch/microblaze/kernel/head.S @@ -39,6 +39,8 @@ #include <asm/processor.h> .section .data +/* The MMU requires a page aligned page directory. */ +.align 12 .global swapper_pg_dir swapper_pg_dir: .space PAGE_SIZE diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h index 3ff893a67c13..ae43fe79b2b5 100644 --- a/arch/openrisc/include/asm/processor.h +++ b/arch/openrisc/include/asm/processor.h @@ -26,6 +26,8 @@ | SPR_SR_DCE | SPR_SR_SM) #define USER_SR (SPR_SR_DME | SPR_SR_IME | SPR_SR_ICE \ | SPR_SR_DCE | SPR_SR_IEE | SPR_SR_TEE) +/* SR bits user space may change via sigreturn, the rest stay kernel owned */ +#define SPR_SR_USER_MASK (SPR_SR_F | SPR_SR_CY | SPR_SR_OV) /* * User space process size. This is hardcoded into a few places, diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c index f70a13ee0593..2be5af3b9832 100644 --- a/arch/openrisc/kernel/signal.c +++ b/arch/openrisc/kernel/signal.c @@ -74,6 +74,7 @@ static long save_fp_state(struct sigcontext __user *sc) static int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { + unsigned long old_sr = regs->sr; int err = 0; /* Always make any pending restarted system calls return -EINTR */ @@ -89,8 +90,8 @@ static int restore_sigcontext(struct pt_regs *regs, err |= __copy_from_user(®s->sr, &sc->regs.sr, sizeof(unsigned long)); err |= restore_fp_state(sc); - /* make sure the SM-bit is cleared so user-mode cannot fool us */ - regs->sr &= ~SPR_SR_SM; + /* keep the privileged SR bits kernel owned, restore only user flags */ + regs->sr = (old_sr & ~SPR_SR_USER_MASK) | (regs->sr & SPR_SR_USER_MASK); regs->orig_gpr11 = -1; /* Avoid syscall restart checks */ diff --git a/arch/powerpc/boot/simpleboot.c b/arch/powerpc/boot/simpleboot.c index c80691d83880..27591df41e9e 100644 --- a/arch/powerpc/boot/simpleboot.c +++ b/arch/powerpc/boot/simpleboot.c @@ -68,7 +68,7 @@ void platform_init(unsigned long r3, unsigned long r4, unsigned long r5, /* finally, setup the timebase */ node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-akebono.c b/arch/powerpc/boot/treeboot-akebono.c index e3cc2599869c..1b529037480f 100644 --- a/arch/powerpc/boot/treeboot-akebono.c +++ b/arch/powerpc/boot/treeboot-akebono.c @@ -146,7 +146,7 @@ void platform_init(char *userdata) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-currituck.c b/arch/powerpc/boot/treeboot-currituck.c index d53e8a592f81..5b5363b74f9f 100644 --- a/arch/powerpc/boot/treeboot-currituck.c +++ b/arch/powerpc/boot/treeboot-currituck.c @@ -102,7 +102,7 @@ void platform_init(void) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/include/asm/entry-common.h b/arch/powerpc/include/asm/entry-common.h index fc636c42e89a..c5adb5006361 100644 --- a/arch/powerpc/include/asm/entry-common.h +++ b/arch/powerpc/include/asm/entry-common.h @@ -66,6 +66,13 @@ static inline void srr_regs_clobbered(void) static inline void nap_adjust_return(struct pt_regs *regs) { #ifdef CONFIG_PPC_970_NAP + /* + * Adjust the nap return address before irq_exit_rcu(). irq_exit_rcu() + * may invoke softirqs with interrupts re-enabled, allowing a nested + * async interrupt to arrive. If _TLF_NAPPING is still set at that + * point, the nested interrupt would erroneously redirect its own + * return address to power4_idle_nap_return, corrupting the stack. + */ if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { /* Can avoid a test-and-clear because NMIs do not call this */ clear_thread_local_flags(_TLF_NAPPING); @@ -286,14 +293,6 @@ static inline void arch_interrupt_async_enter_prepare(struct pt_regs *regs) static inline void arch_interrupt_async_exit_prepare(struct pt_regs *regs) { - /* - * Adjust at exit so the main handler sees the true NIA. This must - * come before irq_exit() because irq_exit can enable interrupts, and - * if another interrupt is taken before nap_adjust_return has run - * here, then that interrupt would return directly to idle nap return. - */ - nap_adjust_return(regs); - arch_interrupt_exit_prepare(regs); } diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index fb42a664ae54..1b45a49e9bed 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -246,6 +246,7 @@ interrupt_handler void func(struct pt_regs *regs) \ instrumentation_begin(); \ irq_enter_rcu(); \ ____##func (regs); \ + nap_adjust_return(regs); \ irq_exit_rcu(); \ instrumentation_end(); \ arch_interrupt_async_exit_prepare(regs); \ diff --git a/arch/powerpc/include/asm/preempt.h b/arch/powerpc/include/asm/preempt.h deleted file mode 100644 index 000e2b9681f3..000000000000 --- a/arch/powerpc/include/asm/preempt.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_POWERPC_PREEMPT_H -#define __ASM_POWERPC_PREEMPT_H - -#include <asm-generic/preempt.h> - -#if defined(CONFIG_PREEMPT_DYNAMIC) -#include <linux/jump_label.h> -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -#define need_irq_preemption() \ - (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) -#else -#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION)) -#endif - -#endif /* __ASM_POWERPC_PREEMPT_H */ diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index fdeb97421785..d53c4dd4d8b6 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -53,9 +53,6 @@ struct pt_regs unsigned long esr; }; unsigned long result; - unsigned long exit_flags; - /* Maintain 16 byte interrupt stack alignment */ - unsigned long __pt_regs_pad[3]; }; }; #if defined(CONFIG_PPC64) || defined(CONFIG_PPC_KUAP) diff --git a/arch/powerpc/include/asm/serial.h b/arch/powerpc/include/asm/serial.h index cd6c18d0e66e..c3eb4a64b3f5 100644 --- a/arch/powerpc/include/asm/serial.h +++ b/arch/powerpc/include/asm/serial.h @@ -1,6 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - */ #ifndef _ASM_POWERPC_SERIAL_H #define _ASM_POWERPC_SERIAL_H @@ -18,4 +16,4 @@ extern void find_legacy_serial_ports(void); #define find_legacy_serial_ports() do { } while (0) #endif -#endif /* _PPC64_SERIAL_H */ +#endif /* _ASM_POWERPC_SERIAL_H */ diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index ee3b9adb5b67..0487e94d3416 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -57,6 +57,7 @@ struct thread_info { #ifdef CONFIG_SMP unsigned int cpu; #endif + unsigned long exit_flags; /* Exit Flags for entry/exit */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ unsigned long local_flags; /* private flags for thread */ #ifdef CONFIG_LIVEPATCH_64 diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 7b8c56962c31..49039074b33f 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -537,7 +537,7 @@ static inline void __user *mask_user_address(const void __user *ptr) if (IS_ENABLED(CONFIG_PPC64)) return mask_user_address_simple(ptr); - if (IS_ENABLED(CONFIG_E500)) + if (IS_ENABLED(CONFIG_PPC_E500)) return mask_user_address_isel(ptr); if (TASK_SIZE <= UL(SZ_2G) && border >= UL(SZ_2G)) return mask_user_address_simple(ptr); diff --git a/arch/powerpc/include/uapi/asm/ptrace.h b/arch/powerpc/include/uapi/asm/ptrace.h index a393b7f2760a..01e630149d48 100644 --- a/arch/powerpc/include/uapi/asm/ptrace.h +++ b/arch/powerpc/include/uapi/asm/ptrace.h @@ -55,8 +55,6 @@ struct pt_regs unsigned long dar; /* Fault registers */ unsigned long dsisr; /* on 4xx/Book-E used for ESR */ unsigned long result; /* Result of a system call */ - unsigned long exit_flags; /* System call exit flags */ - unsigned long __pt_regs_pad[3]; /* Maintain 16 byte interrupt stack alignment */ }; #endif /* __ASSEMBLER__ */ @@ -116,12 +114,10 @@ struct pt_regs #define PT_DAR 41 #define PT_DSISR 42 #define PT_RESULT 43 -#define PT_EXIT_FLAGS 44 -#define PT_PAD 47 /* 3 times */ -#define PT_DSCR 48 -#define PT_REGS_COUNT 48 +#define PT_DSCR 44 +#define PT_REGS_COUNT 44 -#define PT_FPR0 (PT_REGS_COUNT + 4) /* each FP reg occupies 2 slots in this space */ +#define PT_FPR0 48 /* each FP reg occupies 2 slots in this space */ #ifndef __powerpc64__ @@ -133,7 +129,7 @@ struct pt_regs #define PT_FPSCR (PT_FPR0 + 32) /* each FP reg occupies 1 slot in 64-bit space */ -#define PT_VR0 (PT_FPSCR + 2) /* <82> each Vector reg occupies 2 slots in 64-bit */ +#define PT_VR0 82 /* each Vector reg occupies 2 slots in 64-bit */ #define PT_VSCR (PT_VR0 + 32*2 + 1) #define PT_VRSAVE (PT_VR0 + 33*2) @@ -141,7 +137,7 @@ struct pt_regs /* * Only store first 32 VSRs here. The second 32 VSRs in VR0-31 */ -#define PT_VSR0 (PT_VRSAVE + 2) /* each VSR reg occupies 2 slots in 64-bit */ +#define PT_VSR0 150 /* each VSR reg occupies 2 slots in 64-bit */ #define PT_VSR31 (PT_VSR0 + 2*31) #endif /* __powerpc64__ */ diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 3af6c06af02f..e5853daa6a48 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -704,6 +704,15 @@ static void __init cpufeatures_setup_start(u32 isa) if (isa >= ISA_V3_1) { cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; + + /* + * CPU_FTR_P11_PVR is a kernel-internal flag to identify + * Power11 and later processors. While ISA v3.1 is supported + * by Power10+, this flag specifically indicates Power11+ + * for code that needs to distinguish between P10 and P11. + */ + if (PVR_VER(mfspr(SPRN_PVR)) >= PVR_POWER11) + cur_cpu_spec->cpu_features |= CPU_FTR_P11_PVR; } } diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index f04978080837..5b88bf72786c 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -89,15 +89,17 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, long scv) { unsigned long ti_flags; + unsigned long ret = 0; bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; kuap_assert_locked(); regs->result = r3; - regs->exit_flags = 0; - ti_flags = read_thread_flags(); + /* Clear exit_flags so only flags set during this exit are visible */ + current->thread_info.exit_flags = 0; + ti_flags = read_thread_flags(); if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { r3 = -r3; @@ -107,7 +109,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) { if (ti_flags & _TIF_RESTOREALL) - regs->exit_flags = _TIF_RESTOREALL; + ret = _TIF_RESTOREALL; else regs->gpr[3] = r3; clear_bits(_TIF_PERSYSCALL_MASK, ¤t_thread_info()->flags); @@ -116,7 +118,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, } if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) { - regs->exit_flags |= _TIF_RESTOREALL; + ret |= _TIF_RESTOREALL; } syscall_exit_to_user_mode(regs); @@ -132,17 +134,19 @@ again: /* Restore user access locks last */ kuap_user_restore(regs); - + ret |= current->thread_info.exit_flags; #ifdef CONFIG_PPC64 - regs->exit_result = regs->exit_flags; + regs->exit_result = ret; #endif - return regs->exit_flags; + return ret; } #ifdef CONFIG_PPC64 notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs) { + unsigned long ret; + /* * This is called when detecting a soft-pending interrupt as well as * an alternate-return interrupt. So we can't just have the alternate @@ -167,9 +171,11 @@ again: } kuap_user_restore(regs); - regs->exit_result |= regs->exit_flags; + ret = current_thread_info()->exit_flags & _TIF_RESTOREALL; + current_thread_info()->exit_flags &= ~_TIF_RESTOREALL; + regs->exit_result |= ret; - return regs->exit_result; + return ret; } #endif @@ -186,8 +192,10 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs) */ kuap_assert_locked(); + /* Clear exit_flags so only flags set during this exit are visible */ + current_thread_info()->exit_flags = 0; + local_irq_disable(); - regs->exit_flags = 0; again: check_return_regs_valid(regs); user_enter_irqoff(); @@ -200,9 +208,7 @@ again: /* Restore user access locks last */ kuap_user_restore(regs); - - ret = regs->exit_flags; - + ret = current_thread_info()->exit_flags & _TIF_RESTOREALL; #ifdef CONFIG_PPC64 regs->exit_result = ret; #endif diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 316d4f5ead8e..6cd180bc36ab 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -291,7 +291,6 @@ void __init pt_regs_check(void) CHECK_REG(PT_DAR, dar); CHECK_REG(PT_DSISR, dsisr); CHECK_REG(PT_RESULT, result); - CHECK_REG(PT_EXIT_FLAGS, exit_flags); #undef CHECK_REG BUILD_BUG_ON(PT_REGS_COUNT != sizeof(struct user_pt_regs) / sizeof(unsigned long)); diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index bb42a8b6c642..cc6498501610 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -356,6 +356,6 @@ void signal_fault(struct task_struct *tsk, struct pt_regs *regs, void arch_do_signal_or_restart(struct pt_regs *regs) { BUG_ON(regs != current->thread.regs); - regs->exit_flags |= _TIF_RESTOREALL; + current_thread_info()->exit_flags |= _TIF_RESTOREALL; do_signal(current); } diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 3460d1a5a97c..11145c40183d 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -377,7 +377,6 @@ void vtime_task_switch(struct task_struct *prev) } } -#ifdef CONFIG_NO_HZ_COMMON /** * vtime_reset - Fast forward vtime entry clocks * @@ -394,6 +393,7 @@ void vtime_reset(void) #endif } +#ifdef CONFIG_NO_HZ_COMMON /** * vtime_dyntick_start - Inform vtime about entry to idle-dynticks * @@ -933,6 +933,7 @@ static void __init set_decrementer_max(void) static void __init init_decrementer_clockevent(void) { register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } void secondary_cpu_time_init(void) @@ -948,6 +949,7 @@ void secondary_cpu_time_init(void) /* FIME: Should make unrelated change to move snapshot_timebase * call here ! */ register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } /* diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c index 57e897b60db8..cc9fb72cb4eb 100644 --- a/arch/powerpc/lib/vmx-helper.c +++ b/arch/powerpc/lib/vmx-helper.c @@ -46,7 +46,7 @@ int exit_vmx_usercopy(void) * set and we are preemptible. The hack here is to schedule a * decrementer to fire here and reschedule for us if necessary. */ - if (need_irq_preemption() && need_resched()) + if (need_resched()) set_dec(1); return 0; } diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index e3822f36c419..1d2db6d3e226 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -1023,10 +1023,7 @@ static int thread_imc_event_init(struct perf_event *event) static bool is_thread_imc_pmu(struct perf_event *event) { - if (!strncmp(event->pmu->name, "thread_imc", strlen("thread_imc"))) - return true; - - return false; + return strstarts(event->pmu->name, "thread_imc"); } static __be64 *get_event_base_addr(struct perf_event *event) diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c index 757811155587..c11deb2f50ed 100644 --- a/arch/powerpc/platforms/85xx/common.c +++ b/arch/powerpc/platforms/85xx/common.c @@ -42,6 +42,8 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = { { .compatible = "fsl,qoriq-pcie-v2.3", }, { .compatible = "fsl,qoriq-pcie-v2.2", }, { .compatible = "fsl,fman", }, + /* IFC NAND and NOR controllers */ + { .compatible = "fsl,ifc", }, {}, }; diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index f6de8c1169d5..de7494748fec 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -268,10 +268,12 @@ static int spufs_mem_mmap_access(struct vm_area_struct *vma, if (write && !(vma->vm_flags & VM_WRITE)) return -EACCES; + if (offset >= LS_SIZE) + return -EFAULT; if (spu_acquire(ctx)) return -EINTR; - if ((offset + len) > vma->vm_end) - len = vma->vm_end - offset; + if ((offset + len) > LS_SIZE) + len = LS_SIZE - offset; local_store = ctx->ops->get_ls(ctx); if (write) memcpy_toio(local_store + offset, buf, len); diff --git a/arch/powerpc/platforms/ps3/mm.c b/arch/powerpc/platforms/ps3/mm.c index 20fc5b68faee..315a32fd75b1 100644 --- a/arch/powerpc/platforms/ps3/mm.c +++ b/arch/powerpc/platforms/ps3/mm.c @@ -615,6 +615,7 @@ static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr, fail_map: for (iopage--; 0 <= iopage; iopage--) { + offset = (1 << r->page_size) * iopage; lv1_put_iopte(0, c->bus_addr + offset, c->lpar_addr + offset, diff --git a/arch/powerpc/platforms/ps3/spu.c b/arch/powerpc/platforms/ps3/spu.c index e4e0b45e1b9d..a71c0b0cfb07 100644 --- a/arch/powerpc/platforms/ps3/spu.c +++ b/arch/powerpc/platforms/ps3/spu.c @@ -191,8 +191,6 @@ static void spu_unmap(struct spu *spu) static int __init setup_areas(struct spu *spu) { - struct table {char* name; unsigned long addr; unsigned long size;}; - spu_pdata(spu)->shadow = ioremap_prot(spu_pdata(spu)->shadow_addr, sizeof(struct spe_shadow), pgprot_noncached_wc(PAGE_KERNEL_RO)); diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index f7052b131a4c..74910ce3a541 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -154,6 +154,7 @@ config HV_PERF_CTRS config VPA_PMU tristate "VPA PMU events" depends on KVM_BOOK3S_64_HV && HV_PERF_CTRS + default m help Enable access to the VPA PMU counters via perf. This enables code that support measurement for KVM on PowerVM(KoP) feature. diff --git a/arch/powerpc/platforms/pseries/kexec.c b/arch/powerpc/platforms/pseries/kexec.c index 431be156ca9b..29f7c97ff193 100644 --- a/arch/powerpc/platforms/pseries/kexec.c +++ b/arch/powerpc/platforms/pseries/kexec.c @@ -20,12 +20,15 @@ void pseries_kexec_cpu_down(int crash_shutdown, int secondary) { /* - * Don't risk a hypervisor call if we're crashing - * XXX: Why? The hypervisor is not crashing. It might be better - * to at least attempt unregister to avoid the hypervisor stepping - * on our memory. + * Ensure vpa/slb_shadow/dtl cleanup even while we are crashing. + * Why? The hypervisor is not crashing so at least attempt unregister to + * avoid the hypervisor stepping on our memory. If hypervisor or kexec + * kernel steps on the old memory allocated to these areas before the + * new kexec-kernel happens to allocate and register new areas, + * the hypervisor will see invalid content which may cause + * unexpected behavior. */ - if (firmware_has_feature(FW_FEATURE_SPLPAR) && !crash_shutdown) { + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { int ret; int cpu = smp_processor_id(); int hwcpu = hard_smp_processor_id(); diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c index 54b7ecf375b5..3280b7968cab 100644 --- a/arch/powerpc/platforms/pseries/lparcfg.c +++ b/arch/powerpc/platforms/pseries/lparcfg.c @@ -699,7 +699,7 @@ static ssize_t lparcfg_write(struct file *file, const char __user * buf, if (!firmware_has_feature(FW_FEATURE_SPLPAR)) return -EINVAL; - if (count > sizeof(kbuf)) + if (count == 0 || count > sizeof(kbuf)) return -EINVAL; if (copy_from_user(kbuf, buf, count)) diff --git a/arch/powerpc/platforms/pseries/papr-phy-attest.c b/arch/powerpc/platforms/pseries/papr-phy-attest.c index 20a0e1581302..350ba26e5962 100644 --- a/arch/powerpc/platforms/pseries/papr-phy-attest.c +++ b/arch/powerpc/platforms/pseries/papr-phy-attest.c @@ -230,10 +230,17 @@ static long papr_phy_attest_create_handle(struct papr_phy_attest_io_block __user return -ENOMEM; if (copy_from_user(¶ms->cmd, ulc, - sizeof(struct papr_phy_attest_io_block))) + sizeof(struct papr_phy_attest_io_block))) { + kfree(params); return -EFAULT; + } params->cmd_len = be32_to_cpu(params->cmd.length); + if (params->cmd_len == 0 || params->cmd_len > sizeof(params->cmd)) { + kfree(params); + return -EINVAL; + } + seq = (struct papr_rtas_sequence) { .begin = phy_attest_sequence_begin, .end = phy_attest_sequence_end, @@ -246,6 +253,9 @@ static long papr_phy_attest_create_handle(struct papr_phy_attest_io_block __user &papr_phy_attest_handle_ops, "[papr-physical-attestation]"); + if (fd < 0) + kfree(params); + return fd; } diff --git a/arch/powerpc/platforms/pseries/papr_platform_attributes.c b/arch/powerpc/platforms/pseries/papr_platform_attributes.c index c6159870de0e..9c3758aa54c6 100644 --- a/arch/powerpc/platforms/pseries/papr_platform_attributes.c +++ b/arch/powerpc/platforms/pseries/papr_platform_attributes.c @@ -271,11 +271,9 @@ retry: esi_buf_size = ESI_HDR_SIZE + (CURR_MAX_ESI_ATTRS * max_esi_attrs); temp_esi_buf = krealloc(esi_buf, esi_buf_size, GFP_KERNEL); - if (temp_esi_buf) - esi_buf = temp_esi_buf; - else - return -ENOMEM; - + if (!temp_esi_buf) + goto out_free_esi_buf; + esi_buf = temp_esi_buf; goto retry; } diff --git a/arch/powerpc/platforms/pseries/pci.c b/arch/powerpc/platforms/pseries/pci.c index 84e4ffe957a8..d11a64a086c1 100644 --- a/arch/powerpc/platforms/pseries/pci.c +++ b/arch/powerpc/platforms/pseries/pci.c @@ -132,7 +132,7 @@ static int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) /* First integer stores max config */ max_config_vfs = of_read_number(&max_vfs[0], 1); - if (max_config_vfs < num_vfs && num_vfs > MAX_VFS_FOR_MAP_PE) { + if (max_config_vfs < num_vfs || num_vfs > MAX_VFS_FOR_MAP_PE) { dev_err(&pdev->dev, "Num VFs %x > %x Configurable VFs\n", num_vfs, (num_vfs > MAX_VFS_FOR_MAP_PE) ? diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index db99725e752b..bf3d7ed3be01 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -128,7 +128,12 @@ static void smp_setup_cpu(int cpu) else if (cpu != boot_cpuid) xics_setup_cpu(); - if (firmware_has_feature(FW_FEATURE_SPLPAR)) + /* + * Initialize VPA on non-boot cpus since boot-cpu vpa was + * already initialized in pSeries_setup_arch() + */ + if (firmware_has_feature(FW_FEATURE_SPLPAR) && + cpu != boot_cpuid) vpa_init(cpu); cpumask_clear_cpu(cpu, of_spin_mask); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c0a6992933e4..f7028caaeae0 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -157,7 +157,7 @@ config RISCV select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) - select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C + select FUNCTION_ALIGNMENT_4B if DYNAMIC_FTRACE && RISCV_ISA_C select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index c174ac0ec46b..429e07589306 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -84,6 +84,12 @@ config ARCH_THEAD help This enables support for the RISC-V based T-HEAD SoCs. +config ARCH_ULTRARISC + bool "UltraRISC RISC-V SoCs" + help + This enables support for UltraRISC SoC platform hardware, + including boards based on the UR-DP1000. + config ARCH_VIRT bool "QEMU Virt Machine" select POWER_RESET diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index ce0cc737f870..1363e5bef35c 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -168,7 +168,7 @@ vdso_prepare: prepare0 endif endif -vdso-install-y += arch/riscv/kernel/vdso/vdso.so.dbg +vdso-install-$(CONFIG_MMU) += arch/riscv/kernel/vdso/vdso.so.dbg vdso-install-$(CONFIG_RISCV_USER_CFI) += arch/riscv/kernel/vdso_cfi/vdso-cfi.so.dbg vdso-install-$(CONFIG_COMPAT) += arch/riscv/kernel/compat_vdso/compat_vdso.so.dbg diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 74ba5acc12a4..ed605b5e3162 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -33,6 +33,7 @@ CONFIG_SOC_STARFIVE=y CONFIG_ARCH_SUNXI=y CONFIG_ARCH_TENSTORRENT=y CONFIG_ARCH_THEAD=y +CONFIG_ARCH_ULTRARISC=y CONFIG_ARCH_VIRT=y CONFIG_ARCH_CANAAN=y CONFIG_SMP=y diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index d0c61f86cba3..df80c9614df1 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -93,10 +93,8 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, for (alt = begin; alt < end; alt++) { if (alt->vendor_id != SIFIVE_VENDOR_ID) continue; - if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) { - WARN(1, "This errata id:%d is not in kernel errata list", alt->patch_id); + if (alt->patch_id >= ERRATA_SIFIVE_NUMBER) continue; - } tmp = (1U << alt->patch_id); if (cpu_req_errata & tmp) { diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 8cfe59483a8f..c2b0a2928f06 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -40,7 +40,7 @@ do { \ flush_icache_mm(vma->vm_mm, 0); \ } while (0) -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* This is accessed in assembly code. cpumask_var_t would be too complex. */ extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); extern char _end[]; @@ -56,7 +56,8 @@ static inline void mark_new_valid_map(void) #define flush_cache_vmap flush_cache_vmap static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (is_vmalloc_or_module_addr((void *)start)) + if (is_vmalloc_or_module_addr((void *)start) || + (start >= VMEMMAP_START && end <= VMEMMAP_END)) mark_new_valid_map(); } #define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 09bb5f57a9d3..92d5f831f349 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -102,12 +102,14 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) #define readsw(addr, buffer, count) __readsw(addr, buffer, count) #define readsl(addr, buffer, count) __readsl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) #define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) #define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) #define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) __io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) @@ -116,26 +118,32 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) #define writesw(addr, buffer, count) __writesw(addr, buffer, count) #define writesl(addr, buffer, count) __writesl(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) #define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) #define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) #define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) +#endif #ifdef CONFIG_64BIT __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) #define readsq(addr, buffer, count) __readsq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) #define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) +#endif __io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) #define writesq(addr, buffer, count) __writesq(addr, buffer, count) +#ifdef CONFIG_HAS_IOPORT __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) #endif +#endif #include <asm-generic/io.h> diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 60017ceec9d2..e2d5808169e4 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -209,13 +209,13 @@ struct kvm_vcpu_arch { /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index a06697846e69..d41acfb3959d 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -22,13 +22,13 @@ static inline cycles_t get_cycles(void) #else /* !CONFIG_64BIT */ static inline u32 get_cycles(void) { - return readl_relaxed(((u32 *)clint_time_val)); + return readl_relaxed(((u32 __iomem *)clint_time_val)); } #define get_cycles get_cycles static inline u32 get_cycles_hi(void) { - return readl_relaxed(((u32 *)clint_time_val) + 1); + return readl_relaxed(((u32 __iomem *)clint_time_val) + 1); } #define get_cycles_hi get_cycles_hi #endif /* CONFIG_64BIT */ diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 08df724e13b9..d799c4e56f80 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -137,7 +137,7 @@ SYM_CODE_START(handle_exception) .Lrestore_kernel_tpsp: csrr tp, CSR_SCRATCH -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) /* * The RISC-V kernel does not flush TLBs on all CPUS after each new * vmalloc mapping or kfence_unprotect(), which may result in diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index b430edfb83f4..be8b68514417 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -12,6 +12,7 @@ #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> +#include <asm/insn.h> #ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) @@ -63,7 +64,9 @@ static int __ftrace_modify_call(unsigned long source, unsigned long target, bool if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) return -EFAULT; - if (replaced[0] != call[0]) { + /* Bypass the check if the auipc insn is a kprobe breakpoint */ + if (replaced[0] != call[0] && + !(riscv_insn_is_ebreak(replaced[0]) || riscv_insn_is_c_ebreak(replaced[0]))) { pr_err("%p: expected (%08x) but got (%08x)\n", (void *)source, call[0], replaced[0]); return -EINVAL; diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 2306ce3e5f22..738df176ff6f 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -41,6 +41,9 @@ machine_kexec_prepare(struct kimage *image) if (image->segment[i].memsz <= sizeof(fdt)) continue; + if (!image->segment[i].buf) + continue; + if (image->file_mode) memcpy(&fdt, image->segment[i].buf, sizeof(fdt)); else if (copy_from_user(&fdt, image->segment[i].buf, sizeof(fdt))) diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 1659d31fd288..caf6762427c8 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -450,6 +450,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, if (cpusetsize > cpumask_size()) cpusetsize = cpumask_size(); + cpumask_clear(&cpus); ret = copy_from_user(&cpus, cpus_user, cpusetsize); if (ret) return -EFAULT; diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index bb57eb5d19df..36201613d2ca 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -289,7 +289,7 @@ free: } /* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) { schedule_on_each_cpu(check_vector_unaligned_access); riscv_hwprobe_complete_async_probe(); @@ -297,7 +297,7 @@ static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __alway return 0; } #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) { return 0; } @@ -411,4 +411,10 @@ static int __init check_unaligned_access_all_cpus(void) return 0; } -late_initcall(check_unaligned_access_all_cpus); +/* + * Run after clocksource_done_booting() so measure_cycles() uses a stable + * clocksource, but before rootfs_initcall() enables usermode helpers. Those + * helpers can reach hwprobe and populate the vDSO cache, so async hwprobe + * probes must be registered first. + */ +fs_initcall_sync(check_unaligned_access_all_cpus); diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index a842dc034571..43ee881f6c6f 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -69,9 +69,9 @@ CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif # Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) -CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) +CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO) # Force dependency $(obj)/$(vdso_o): $(obj)/$(vdso_so) diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index e82987dc3739..d6f96b1abe40 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -7,11 +7,19 @@ #include <asm/unistd.h> #include <asm/assembler.h> +/* + * WARNING: Do NOT add a CFI landing pad at the start of this function. + * Unwinders such as libgcc identify the sigreturn trampoline by matching the + * instruction sequence. Adding a landing pad here would break unwinding from + * signal handlers. + * + * This trampoline is used only for signal return and not via an indirect + * call/jump from userspace, so adding CFI landing pad is unnecessary. + */ .text SYM_FUNC_START(__vdso_rt_sigreturn) .cfi_startproc .cfi_signal_frame - vdso_lpad li a7, __NR_rt_sigreturn ecall .cfi_endproc diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index bafb009c5ce5..9a653b4ad40a 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -53,12 +53,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long mask, val; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (!kvm_riscv_aia_available()) return; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; + mask = vcpu->arch.irqs_pending_mask[1]; + if (mask) { + vcpu->arch.irqs_pending_mask[1] = 0; + val = vcpu->arch.irqs_pending[1] & mask; csr->hviph &= ~mask; csr->hviph |= val; @@ -69,6 +72,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (kvm_riscv_aia_available()) csr->vsieh = ncsr_read(CSR_VSIEH); } @@ -77,13 +82,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { unsigned long seip; +#ifdef CONFIG_32BIT + unsigned long flags; + bool pending; +#endif if (!kvm_riscv_aia_available()) return false; #ifdef CONFIG_32BIT - if (READ_ONCE(vcpu->arch.irqs_pending[1]) & - (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + pending = vcpu->arch.irqs_pending[1] & + (vcpu->arch.aia_context.guest_csr.vsieh & + upper_32_bits(mask)); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + if (pending) return true; #endif @@ -207,6 +221,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); +#ifdef CONFIG_32BIT + unsigned long flags; +#endif if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; @@ -219,8 +236,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = val; #ifdef CONFIG_32BIT - if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); + if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[1] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, + flags); + } #endif } diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index c4c3b79567f1..b0474fcf065a 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -5,11 +5,13 @@ */ #include <linux/bitops.h> +#include <linux/cpufeature.h> #include <linux/errno.h> #include <linux/kvm_host.h> #include <linux/module.h> #include <linux/pgtable.h> #include <asm/kvm_gstage.h> +#include <asm/hwcap.h> #ifdef CONFIG_64BIT unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3; @@ -171,8 +173,10 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, } if (pte_val(*ptep) != pte_val(map->pte)) { + bool was_invalid = !pte_val(*ptep); set_pte(ptep, map->pte); - if (gstage_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep) && + !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))) gstage_tlb_flush(gstage, current_level, map->addr); } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 082f9b261733..8a0aa5e0e216 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -41,6 +41,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, pgprot_t prot; unsigned long pfn; phys_addr_t addr, end; + unsigned long pgd_levels = kvm->arch.pgd_levels; struct kvm_mmu_memory_cache pcache = { .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, .gfp_zero = __GFP_ZERO, @@ -63,7 +64,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, if (!writable) map.pte = pte_wrprotect(map.pte); - ret = kvm_mmu_topup_memory_cache(&pcache, kvm->arch.pgd_levels); + ret = __kvm_mmu_topup_memory_cache(&pcache, pgd_levels, pgd_levels); if (ret) goto out; diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index cf6e231e76e2..977e36ab83d3 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -80,6 +80,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) { + unsigned long flags; bool loaded; /** @@ -104,8 +105,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) kvm_riscv_vcpu_aia_reset(vcpu); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_riscv_vcpu_pmu_reset(vcpu); @@ -151,6 +154,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); + raw_spin_lock_init(&vcpu->arch.irqs_pending_lock); spin_lock_init(&vcpu->arch.reset_state.lock); @@ -352,10 +356,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long mask, val; + unsigned long flags; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + + mask = vcpu->arch.irqs_pending_mask[0]; + if (mask) { + vcpu->arch.irqs_pending_mask[0] = 0; + val = vcpu->arch.irqs_pending[0] & mask; csr->hvip &= ~mask; csr->hvip |= val; @@ -363,11 +371,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) /* Flush AIA high interrupts */ kvm_riscv_vcpu_aia_flush_interrupts(vcpu); + + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); } void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) { unsigned long hvip; + unsigned long flags; struct kvm_vcpu_arch *v = &vcpu->arch; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; @@ -376,34 +387,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) /* Sync-up HVIP.VSSIP bit changes does by Guest */ hvip = ncsr_read(CSR_HVIP); + + raw_spin_lock_irqsave(&v->irqs_pending_lock, flags); + if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { if (hvip & (1UL << IRQ_VS_SOFT)) { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - set_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __set_bit(IRQ_VS_SOFT, v->irqs_pending); } else { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - clear_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __clear_bit(IRQ_VS_SOFT, v->irqs_pending); } } /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { if (!(hvip & (1UL << IRQ_PMU_OVF)) && - !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) - clear_bit(IRQ_PMU_OVF, v->irqs_pending); + !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) + __clear_bit(IRQ_PMU_OVF, v->irqs_pending); } /* Sync-up AIA high interrupts */ kvm_riscv_vcpu_aia_sync_interrupts(vcpu); + raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags); + /* Sync-up timer CSRs */ kvm_riscv_vcpu_timer_sync(vcpu); } int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, and external * interrupts when irq is one of the local interrupts @@ -416,9 +434,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - set_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __set_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_vcpu_kick(vcpu); @@ -427,6 +446,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, counter overflow and external * interrupts when irq is one of the local interrupts @@ -439,26 +460,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - clear_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __clear_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); return 0; } bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + unsigned long flags; unsigned long ie; + bool ret; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; - if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) - return true; + ret = vcpu->arch.irqs_pending[0] & ie; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); /* Check AIA high interrupts */ - return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + if (!ret) + ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + + return ret; } void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 0bb0c51e3c89..6c8530b9f29e 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -38,6 +38,25 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, return kvm_riscv_vcpu_mmio_store(vcpu, run, fault_addr, trap->htinst); + case EXC_INST_GUEST_PAGE_FAULT: { + /* + * No memslot backs this GPA and an instruction fetch + * cannot be emulated as MMIO. On bare metal a fetch + * from an unbacked physical address raises an + * instruction access fault, so reflect that back to + * the guest. + */ + struct kvm_cpu_trap inst_trap = { + .sepc = trap->sepc, + .scause = EXC_INST_ACCESS, + .stval = trap->stval, + .htval = 0, + .htinst = 0, + }; + + kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); + return 1; + } default: return -EOPNOTSUPP; }; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index bb920e8923c9..99b9107b1ac1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -50,19 +50,13 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - reg_val = riscv_cbom_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - reg_val = riscv_cboz_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - reg_val = riscv_cbop_block_size; + reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): reg_val = vcpu->arch.mvendorid; @@ -144,21 +138,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, } break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (kvm_riscv_isa_check_host(ZICBOM)) - return -ENOENT; - if (reg_val != riscv_cbom_block_size) + if (reg_val && reg_val != riscv_cbom_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (kvm_riscv_isa_check_host(ZICBOZ)) - return -ENOENT; - if (reg_val != riscv_cboz_block_size) + if (reg_val && reg_val != riscv_cboz_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): - if (kvm_riscv_isa_check_host(ZICBOP)) - return -ENOENT; - if (reg_val != riscv_cbop_block_size) + if (reg_val && reg_val != riscv_cbop_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): @@ -298,6 +286,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); + unsigned long flags; if (reg_num >= regs_max) return -ENOENT; @@ -311,8 +300,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = reg_val; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[0] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + } return 0; } @@ -614,20 +606,6 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, u64 size; u64 reg; - /* - * Avoid reporting config reg if the corresponding extension - * was not available. - */ - if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && - kvm_riscv_isa_check_host(ZICBOM)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && - kvm_riscv_isa_check_host(ZICBOZ)) - continue; - else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && - kvm_riscv_isa_check_host(ZICBOP)) - continue; - size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index bb46dcbfb24d..2025b664961c 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -586,7 +586,7 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } /* Start the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) @@ -658,7 +658,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } /* Stop the counters that have been configured and requested by the guest */ - for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) { pmc_index = array_index_nospec(i + ctr_base, RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c index ab39ac464ffd..1342adb3180c 100644 --- a/arch/riscv/kvm/vcpu_sbi_fwft.c +++ b/arch/riscv/kvm/vcpu_sbi_fwft.c @@ -327,9 +327,11 @@ static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) return SBI_ERR_DENIED_LOCKED; - conf->flags = flags; + ret = conf->feature->set(vcpu, conf, false, value); + if (ret == SBI_SUCCESS) + conf->flags = flags; - return conf->feature->set(vcpu, conf, false, value); + return ret; } static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 62d2fb77bb9b..3708616e2c32 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -10,6 +10,7 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> #include <asm/kvm_isa.h> @@ -129,11 +130,20 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, return -ENOENT; } } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { + unsigned long reg_offset; + if (reg_size != vlenb) return -EINVAL; WARN_ON(!cntx->vector.datap); - *reg_addr = cntx->vector.datap + - (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; + /* + * The reg_num is derived from the userspace-provided ONE_REG + * id. Sanitize it with array_index_nospec() to prevent + * speculative out-of-bounds access to the vector register + * buffer (32 vector registers: v0..v31). + */ + reg_offset = array_index_nospec( + reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); + *reg_addr = cntx->vector.datap + reg_offset * vlenb; } else { return -ENOENT; } diff --git a/arch/riscv/lib/strnlen.S b/arch/riscv/lib/strnlen.S index 53afa7b5b314..a8911605c248 100644 --- a/arch/riscv/lib/strnlen.S +++ b/arch/riscv/lib/strnlen.S @@ -83,8 +83,13 @@ strnlen_zbb: sub t3, t3, t2 slli t2, t2, 3 - /* Aligned boundary. */ + /* + * Aligned boundary. Use the address of the last valid byte + * (s + count - 1) to avoid loading a word past the count + * boundary in the loop below. count == 0 is handled above. + */ add t4, a0, a1 + addi t4, t4, -1 andi t4, t4, -SZREG /* Get the first word. */ @@ -120,6 +125,9 @@ strnlen_zbb: bgtu t3, a0, 2f + /* All remaining bytes are in the first word, no loop needed. */ + bgeu t0, t4, 2f + /* Prepare for the word comparison loop. */ addi t2, t0, SZREG li t3, -1 diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 5b1b3c88b4d1..01d252c741d2 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -37,7 +37,9 @@ #include "../kernel/head.h" +#if defined(CONFIG_64BIT) && defined(CONFIG_MMU) DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS); +#endif struct kernel_mapping kernel_map __ro_after_init; EXPORT_SYMBOL(kernel_map); @@ -61,7 +63,8 @@ EXPORT_SYMBOL(phys_ram_base); #ifdef CONFIG_SPARSEMEM_VMEMMAP #define VMEMMAP_ADDR_ALIGN max(1ULL << SECTION_SIZE_BITS, \ - MAX_FOLIO_VMEMMAP_ALIGN) + PFN_PHYS(MAX_FOLIO_VMEMMAP_ALIGN / \ + sizeof(struct page))) unsigned long vmemmap_start_pfn __ro_after_init; EXPORT_SYMBOL(vmemmap_start_pfn); @@ -162,7 +165,9 @@ static void print_vm_layout(void) { } void __init arch_mm_preinit(void) { - bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit); + bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit) && + memblock_start_of_DRAM() < dma32_phys_limit; + unsigned int swiotlb_flags = SWIOTLB_VERBOSE; #ifdef CONFIG_FLATMEM BUG_ON(!mem_map); #endif /* CONFIG_FLATMEM */ @@ -170,17 +175,22 @@ void __init arch_mm_preinit(void) if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb && dma_cache_alignment != 1) { /* - * If no bouncing needed for ZONE_DMA, allocate 1MB swiotlb - * buffer per 1GB of RAM for kmalloc() bouncing on - * non-coherent platforms. + * No 32-bit DMA bouncing needed (either all DRAM is within + * the 32-bit limit, or it all starts above it), but + * kmalloc() buffers whose sizes are not cache-line-aligned + * still require bouncing for non-coherent DMA. Use + * SWIOTLB_ANY so that the buffer can be allocated from high + * memory when DRAM starts above dma32_phys_limit. Allocate + * ~1 MB per 1 GB of RAM. */ unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); swiotlb = true; + swiotlb_flags |= SWIOTLB_ANY; } - swiotlb_init(swiotlb, SWIOTLB_VERBOSE); + swiotlb_init(swiotlb, swiotlb_flags); print_vm_layout(); } @@ -1616,7 +1626,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un for (; addr < end; addr = next) { next = pud_addr_end(addr, end); - pudp = pud_base + pud_index(addr); + pudp = pgtable_l4_enabled ? pud_base + pud_index(addr) : pud_base; pud = pudp_get(pudp); if (!pud_present(pud)) continue; @@ -1647,7 +1657,7 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un for (; addr < end; addr = next) { next = p4d_addr_end(addr, end); - p4dp = p4d_base + p4d_index(addr); + p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) continue; diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index eaa34c5bd3c1..b4182ca4435f 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -440,6 +440,7 @@ struct kvm_vcpu_arch { bool skey_enabled; /* Indicator if the access registers have been loaded from guest */ bool acrs_loaded; + bool initialized; struct kvm_s390_pv_vcpu pv; union diag318_info diag318_info; struct kvm_s390_mmu_cache *mc; @@ -476,6 +477,11 @@ struct s390_map_info { __u64 guest_addr; __u64 addr; struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; }; struct s390_io_adapter { diff --git a/arch/s390/kernel/diag/diag310.c b/arch/s390/kernel/diag/diag310.c index f411562aa7f6..c68ec9d28513 100644 --- a/arch/s390/kernel/diag/diag310.c +++ b/arch/s390/kernel/diag/diag310.c @@ -190,17 +190,18 @@ static int memtop_get_stride_len(unsigned long *res) static int memtop_get_page_count(unsigned long *res, unsigned long level) { static unsigned long memtop_pages[DIAG310_LEVELMAX]; - unsigned long pages; + unsigned long pages, idx; int rc; if (level > DIAG310_LEVELMAX || level < DIAG310_LEVELMIN) return -EINVAL; - pages = READ_ONCE(memtop_pages[level - 1]); + idx = array_index_nospec(level - 1, ARRAY_SIZE(memtop_pages)); + pages = READ_ONCE(memtop_pages[idx]); if (!pages) { rc = diag310_get_memtop_size(&pages, level); if (rc) return rc; - WRITE_ONCE(memtop_pages[level - 1], pages); + WRITE_ONCE(memtop_pages[idx], pages); } *res = pages; return 0; diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 7aa655664ecc..2076ac22e2c4 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -15,6 +15,7 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/perf_event.h> +#include <linux/nospec.h> #include <asm/cpu_mf.h> #include <asm/hwctrset.h> @@ -768,6 +769,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user)); ev = cpumf_generic_events_user[ev]; } } else if (!attr->exclude_kernel && attr->exclude_user) { @@ -778,6 +780,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type) if (!is_userspace_event(ev)) { if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) return -EOPNOTSUPP; + ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic)); ev = cpumf_generic_events_basic[ev]; } } diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index bd0df61d1907..2b989bebd220 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -65,6 +65,7 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); unsigned char ptff_function_mask[16]; +EXPORT_SYMBOL(ptff_function_mask); static unsigned long lpar_offset; static unsigned long initial_leap_seconds; diff --git a/arch/s390/kernel/vdso/note.S b/arch/s390/kernel/vdso/note.S index db19d0680a0a..d215781f26b5 100644 --- a/arch/s390/kernel/vdso/note.S +++ b/arch/s390/kernel/vdso/note.S @@ -4,6 +4,7 @@ * Here we can supply some information useful to userland. */ +#include <linux/build-salt.h> #include <linux/uts.h> #include <linux/version.h> #include <linux/elfnote.h> @@ -11,3 +12,5 @@ ELFNOTE_START(Linux, 0, "a") .long LINUX_VERSION_CODE ELFNOTE_END + +BUILD_SALT diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 5f1960ec982d..3f2d6e8902d7 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -570,6 +570,8 @@ static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *tabl else if (walk->ops->pte_entry) rc = dat_pte_walk_range(max(start, cur), min(end, next), dereference_pmd(crste.pmd), walk); + if (rc) + break; } } return rc; @@ -753,13 +755,15 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf return rc; } -int dat_reset_reference_bit(union asce asce, gfn_t gfn) +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) { union pgste pgste, old; union crste *crstep; union pte *ptep; int rc; + skey->skey = 0; + rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; @@ -769,21 +773,23 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn) if (!crste.h.fc || !crste.s.fc1.pr) return 0; - return page_reset_referenced(large_crste_to_phys(*crstep, gfn)); + skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; + return 0; } old = pgste_get_lock(ptep); pgste = old; if (!ptep->h.i) { - rc = page_reset_referenced(pte_origin(*ptep)); - pgste.hr = rc >> 1; + skey->skey = page_reset_referenced(pte_origin(*ptep)) << 1; + pgste.hr = skey->r; } - rc |= (pgste.gr << 1) | pgste.gc; + skey->r |= pgste.gr; + skey->c |= pgste.gc; pgste.gr = 0; dat_update_ptep_sd(old, pgste, ptep); pgste_set_unlock(ptep, pgste); - return rc; + return 0; } static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) @@ -844,6 +850,7 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal struct slot_priv *p = walk->priv; union crste dummy = { .val = p->token }; union pte new_pte, pte = READ_ONCE(*ptep); + union pgste pgste; new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); @@ -851,7 +858,11 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal if (pte.val == new_pte.val) return 0; - dat_ptep_xchg(ptep, new_pte, gfn, walk->asce, false); + pgste = pgste_get_lock(ptep); + pgste = __dat_ptep_xchg(ptep, pgste, new_pte, gfn, walk->asce, false); + pgste.cmma_d = 0; + pgste_set_unlock(ptep, pgste); + return 0; } diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index fad605305e05..141ee7b9f019 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -537,7 +537,7 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf union skey skey, bool nq); int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); -int dat_reset_reference_bit(union asce asce, gfn_t gfn); +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); long dat_reset_skeys(union asce asce, gfn_t start); unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/faultin.c index fee80047bd94..3cc45f7f5b2d 100644 --- a/arch/s390/kvm/faultin.c +++ b/arch/s390/kvm/faultin.c @@ -91,9 +91,9 @@ int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fa /* Access outside memory, addressing exception. */ if (is_noslot_pfn(f->pfn)) return PGM_ADDRESSING; - /* Signal pending: try again. */ - if (f->pfn == KVM_PFN_ERR_SIGPENDING) - return -EAGAIN; + /* Fatal signal pending: bail out. */ + if (is_sigpending_pfn(f->pfn)) + return -EINTR; /* Check if it's read-only memory; don't try to actually handle that case. */ if (f->pfn == KVM_PFN_ERR_RO_FAULT) return -EOPNOTSUPP; diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 298fbaecec28..8abb4f55b306 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -1374,8 +1374,13 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare /* Only allow one real-space gmap shadow. */ list_for_each_entry(sg, &parent->children, list) { if (sg->guest_asce.r) { - scoped_guard(write_lock, &parent->kvm->mmu_lock) + if (write_trylock(&parent->kvm->mmu_lock)) { gmap_unshadow(sg); + write_unlock(&parent->kvm->mmu_lock); + } else { + gmap_put(new); + return ERR_PTR(-EAGAIN); + } break; } } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 9e3e6b0d72ad..8f24bcd1a6d3 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -45,13 +45,16 @@ static struct kvm_s390_gib *gib; static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl sigp_ctrl; if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND)) return 0; + if (kvm_is_ucontrol(vcpu->kvm)) + return 0; BUG_ON(!kvm_s390_use_sca_entries()); + sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; if (src_id) *src_id = sigp_ctrl.scn; @@ -60,13 +63,16 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) { - struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1}; + struct esca_block *sca = vcpu->kvm->arch.sca; + union esca_sigp_ctrl *sigp_ctrl; int expect, rc; BUG_ON(!kvm_s390_use_sca_entries()); + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; old_val = READ_ONCE(*sigp_ctrl); old_val.c = 0; @@ -84,10 +90,13 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) static void sca_clear_ext_call(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl *sigp_ctrl; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized || kvm_is_ucontrol(vcpu->kvm)) return; + + /* Initialize after the above check, to prevent going out of bounds */ + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); WRITE_ONCE(sigp_ctrl->value, 0); @@ -2520,8 +2529,22 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) map->addr = host_addr; map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); if (!map->page) { - ret = -EINVAL; - goto out; + /* + * Long-term pinning may fail for memory types such as file-backed + * memory. Verify that short-term pinning succeeds so that the + * non-atomic irqfd path can handle interrupt injection. + */ + map->page = pin_map_page(kvm, host_addr, 0); + if (!map->page) { + ret = -EINVAL; + goto out; + } + unpin_user_page(map->page); + map->page = NULL; + map->pinned = false; + /* Add an entry to preserve MAP/UNMAP symmetry. */ + } else { + map->pinned = true; } spin_lock_irqsave(&adapter->maps_lock, flags); if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { @@ -2532,7 +2555,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) ret = -EINVAL; } spin_unlock_irqrestore(&adapter->maps_lock, flags); - if (ret) + if (ret && map->page) unpin_user_page(map->page); out: if (ret) @@ -2546,6 +2569,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) struct s390_map_info *map, *tmp, *map_to_free; struct page *map_page_to_put = NULL; u64 map_addr_to_mark = 0; + bool map_pinned = false; unsigned long flags; int found = 0, idx; @@ -2560,6 +2584,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) list_del(&map->list); map_page_to_put = map->page; map_addr_to_mark = map->guest_addr; + map_pinned = map->pinned; map_to_free = map; break; } @@ -2568,11 +2593,18 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) if (found) { kfree(map_to_free); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); - set_page_dirty_lock(map_page_to_put); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map_page_to_put); + if (map_pinned) { + /* + * Only long-term pinned pages need to be marked dirty + * and released. Fallback entries exist only for + * MAP/UNMAP symmetry. + */ + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } } return found ? 0 : -ENOENT; @@ -2598,11 +2630,13 @@ void kvm_s390_unmap_all_adapters(struct kvm *kvm) list_for_each_entry_safe(map, tmp, &local_list, list) { list_del(&map->list); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); - set_page_dirty_lock(map->page); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map->page); + if (map->pinned) { + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + } kfree(map); } } @@ -2929,8 +2963,11 @@ static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, return NULL; list_for_each_entry(map, &adapter->maps, list) { - if (map->addr == addr) + if (map->addr == addr) { + if (!map->pinned) + return NULL; return map; + } } return NULL; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 23c817595e28..518a69c55e85 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -571,7 +571,7 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op switch (op) { case KVM_S390_KEYOP_SSKE: r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0); - if (r >= 0) + if (r == 0 || r == 1) return skey.skey; break; case KVM_S390_KEYOP_ISKE: @@ -580,14 +580,14 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op return skey.skey; break; case KVM_S390_KEYOP_RRBE: - r = dat_reset_reference_bit(asce, gfn); - if (r > 0) - return r << 1; + r = dat_reset_reference_bit(asce, gfn, &skey); + if (!r) + return skey.skey; break; default: return -EINVAL; } - return r; + return r > 0 ? -EFAULT : r; } /* Section: device related */ @@ -1219,8 +1219,8 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) /* * Must be called with kvm->srcu held to avoid races on memslots, and with - * kvm->slots_lock to avoid races with ourselves, kvm_s390_vm_stop_migration(), - * and kvm_s390_get_cmma_bits(). + * kvm->slots_arch_lock to avoid races with ourselves, + * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits(). */ static int kvm_s390_vm_start_migration(struct kvm *kvm) { @@ -1265,7 +1265,7 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) } /* - * Must be called with kvm->slots_lock to avoid races with ourselves, + * Must be called with kvm->slots_arch_lock to avoid races with ourselves, * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits(). */ static int kvm_s390_vm_stop_migration(struct kvm *kvm) @@ -1280,8 +1280,10 @@ static int kvm_s390_vm_stop_migration(struct kvm *kvm) * PGSTEs might have cmma_d set. */ WRITE_ONCE(kvm->arch.migration_mode, 0); - if (kvm->arch.use_cmma) - kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); + if (!kvm->arch.use_cmma) + return 0; + + kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */ gmap_set_cmma_all_clean(kvm->arch.gmap); atomic64_set(&kvm->arch.cmma_dirty_pages, 0); @@ -1298,7 +1300,9 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, { int res = -ENXIO; - mutex_lock(&kvm->slots_lock); + guard(srcu)(&kvm->srcu); + guard(mutex)(&kvm->slots_arch_lock); + switch (attr->attr) { case KVM_S390_VM_MIGRATION_START: res = kvm_s390_vm_start_migration(kvm); @@ -1309,7 +1313,6 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, default: break; } - mutex_unlock(&kvm->slots_lock); return res; } @@ -2212,7 +2215,7 @@ static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) } kvfree(keys); - return r; + return r <= 0 ? r : -EFAULT; } static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) @@ -2274,7 +2277,7 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) kvm_s390_free_mmu_cache(mc); out: kvfree(keys); - return r; + return r <= 0 ? r : -EFAULT; } /* @@ -2384,7 +2387,7 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags); - return r; + return r <= 0 ? r : -EFAULT; } /** @@ -2932,6 +2935,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) case KVM_S390_INTERRUPT: { struct kvm_s390_interrupt s390int; + r = -EINVAL; + if (kvm_is_ucontrol(kvm)) + break; r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; @@ -2996,9 +3002,8 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); - r = kvm_s390_get_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + scoped_guard(mutex, &kvm->slots_arch_lock) + r = kvm_s390_get_cmma_bits(kvm, &args); if (!r) { r = copy_to_user(argp, &args, sizeof(args)); if (r) @@ -3012,9 +3017,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); + mutex_lock(&kvm->slots_arch_lock); r = kvm_s390_set_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + mutex_unlock(&kvm->slots_arch_lock); break; } case KVM_S390_PV_COMMAND: { @@ -3245,7 +3250,8 @@ static void kvm_s390_crypto_init(struct kvm *kvm) static void sca_dispose(struct kvm *kvm) { - free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); + if (kvm->arch.sca) + free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); kvm->arch.sca = NULL; } @@ -3459,7 +3465,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized) return; clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); @@ -3479,8 +3485,8 @@ static void sca_add_vcpu(struct kvm_vcpu *vcpu) if (!kvm_s390_use_sca_entries()) return; + WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block)); set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); - sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block); } static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) @@ -3611,6 +3617,9 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 || vcpu->kvm->arch.user_operexec) vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; + + /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */ + smp_store_release(&vcpu->arch.initialized, true); } static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr) @@ -3672,7 +3681,8 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + if (vcpu->arch.sie_block->cbrlo) + free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); vcpu->arch.sie_block->cbrlo = 0; } @@ -3790,21 +3800,21 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) return 0; } +DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T))) + int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) { - struct sie_page *sie_page; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + struct sie_page *sie_page __free(sie_page) = NULL; int rc; BUILD_BUG_ON(sizeof(struct sie_page) != 4096); - vcpu->arch.mc = kvm_s390_new_mmu_cache(); - if (!vcpu->arch.mc) + mc = kvm_s390_new_mmu_cache(); + if (!mc) return -ENOMEM; sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT); - if (!sie_page) { - kvm_s390_free_mmu_cache(vcpu->arch.mc); - vcpu->arch.mc = NULL; + if (!sie_page) return -ENOMEM; - } vcpu->arch.sie_block = &sie_page->sie_block; vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb); @@ -3846,10 +3856,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; if (kvm_is_ucontrol(vcpu->kvm)) { - rc = -ENOMEM; vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL); if (!vcpu->arch.gmap) - goto out_free_sie_block; + return -ENOMEM; } VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p", @@ -3857,20 +3866,19 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block); rc = kvm_s390_vcpu_setup(vcpu); - if (rc) - goto out_ucontrol_uninit; + if (rc) { + if (kvm_is_ucontrol(vcpu->kvm)) { + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) + gmap_remove_child(vcpu->arch.gmap); + vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); + } + return rc; + } + vcpu->arch.mc = no_free_ptr(mc); + sie_page = NULL; kvm_s390_update_topology_change_report(vcpu->kvm, 1); return 0; - -out_ucontrol_uninit: - if (kvm_is_ucontrol(vcpu->kvm)) { - gmap_remove_child(vcpu->arch.gmap); - vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); - } -out_free_sie_block: - free_page((unsigned long)(vcpu->arch.sie_block)); - return rc; } int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) @@ -5037,6 +5045,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS) return -EINVAL; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); if (guestdbg_exit_pending(vcpu)) { @@ -5445,6 +5457,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, struct kvm_s390_interrupt s390int; struct kvm_s390_irq s390irq = {}; + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; if (copy_from_user(&s390int, argp, sizeof(s390int))) return -EFAULT; if (s390int_to_s390irq(&s390int, &s390irq)) @@ -5521,6 +5535,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, long r; u16 rc, rrc; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); switch (ioctl) { @@ -5792,14 +5810,30 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, return 0; } +static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + if (pgste.cmma_d) { + pgste.cmma_d = 0; + atomic64_dec(walk->priv); + } + pgste_set_unlock(ptep, pgste); + return 0; +} + void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - struct kvm_s390_mmu_cache *mc = NULL; + const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; int rc = 0; + guard(mutex)(&kvm->slots_arch_lock); + if (change == KVM_MR_FLAGS_ONLY) return; @@ -5810,6 +5844,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, } scoped_guard(write_lock, &kvm->mmu_lock) { + if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { + _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, + kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, + &kvm->arch.cmma_dirty_pages); + } + switch (change) { case KVM_MR_DELETE: rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); @@ -5831,7 +5871,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, out: if (rc) pr_warn("failed to commit memory region\n"); - kvm_s390_free_mmu_cache(mc); return; } diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 5b075c38998e..50f5ec79600e 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -190,40 +190,61 @@ static int kvm_zpci_clear_airq(struct zpci_dev *zdev) return cc ? -EIO : 0; } -static inline void unaccount_mem(unsigned long nr_pages) +static inline void unaccount_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { - struct user_struct *user = get_uid(current_user()); + struct user_struct *user = kzdev->user_account; + struct mm_struct *mm_account = kzdev->mm_account; - if (user) + if (user) { atomic_long_sub(nr_pages, &user->locked_vm); - if (current->mm) - atomic64_sub(nr_pages, ¤t->mm->pinned_vm); + free_uid(user); + kzdev->user_account = NULL; + } + + if (mm_account) { + atomic64_sub(nr_pages, &mm_account->pinned_vm); + mmdrop(mm_account); + kzdev->mm_account = NULL; + } } -static inline int account_mem(unsigned long nr_pages) +static inline int account_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { struct user_struct *user = get_uid(current_user()); unsigned long page_limit, cur_pages, new_pages; + int rc = 0; page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; - if (new_pages > page_limit) - return -ENOMEM; + if (new_pages > page_limit) { + rc = -ENOMEM; + goto out; + } } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); - atomic64_add(nr_pages, ¤t->mm->pinned_vm); + if (current->mm) { + mmgrab(current->mm); + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + } + + kzdev->user_account = user; + kzdev->mm_account = current->mm; return 0; + +out: + free_uid(user); + return rc; } static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, bool assist) { struct page *pages[1], *aibv_page, *aisb_page = NULL; - unsigned int msi_vecs, idx; + unsigned int msi_vecs, idx, size; struct zpci_gaite *gaite; unsigned long hva, bit; struct kvm *kvm; @@ -237,6 +258,10 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, if (zdev->gisa == 0) return -EINVAL; + /* AIF already enabled for the device */ + if (zdev->kzdev->fib.fmt0.aibv != 0) + return -EINVAL; + kvm = zdev->kzdev->kvm; msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); @@ -246,6 +271,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return gisc; /* Replace AIBV address */ + size = BITS_TO_LONGS(msi_vecs + fib->fmt0.aibvo) * sizeof(unsigned long); + npages = DIV_ROUND_UP((fib->fmt0.aibv & ~PAGE_MASK) + size, PAGE_SIZE); + /* AIBV cannot span more than 1 page */ + if (npages > 1) { + rc = -EINVAL; + goto out; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); @@ -261,6 +294,12 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, /* Pin the guest AISB if one was specified */ if (fib->fmt0.sum == 1) { + /* AISB must be dword aligned */ + if (fib->fmt0.aisb & 0x7) { + rc = -EINVAL; + goto unpin1; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, @@ -275,20 +314,28 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(pcount)) + rc = account_mem(zdev->kzdev, pcount); + if (rc) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ mutex_lock(&aift->aift_lock); bit = airq_iv_alloc_bit(aift->sbv); - if (bit == -1UL) + if (bit == -1UL) { + rc = -ENOMEM; goto unlock; + } zdev->aisb = bit; /* store the summary bit number */ zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK | AIRQ_IV_GUESTVEC, phys_to_virt(fib->fmt0.aibv)); + if (!zdev->aibv) { + rc = -ENOMEM; + goto free_aisb; + } + spin_lock_irq(&aift->gait_lock); gaite = aift->gait + zdev->aisb; @@ -300,27 +347,50 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, gaite->gisc = fib->fmt0.isc; gaite->count++; - gaite->aisbo = fib->fmt0.aisbo; - gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb & - ~PAGE_MASK)); + if (fib->fmt0.sum == 1) { + gaite->aisbo = fib->fmt0.aisbo; + gaite->aisb = virt_to_phys(page_address(aisb_page) + + (fib->fmt0.aisb & ~PAGE_MASK)); + } else { + gaite->aisbo = 0; + gaite->aisb = 0; + } aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); - /* Update guest FIB for re-issue */ - fib->fmt0.aisbo = zdev->aisb & 63; - fib->fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8; - fib->fmt0.isc = gisc; - /* Save some guest fib values in the host for later use */ - zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.isc = gisc; zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; - mutex_unlock(&aift->aift_lock); /* Issue the clp to setup the irq now */ rc = kvm_zpci_set_airq(zdev); - return rc; + if (!rc) { + mutex_unlock(&aift->aift_lock); + return rc; + } + + /* Start cleanup */ + zdev->kzdev->fib.fmt0.isc = 0; + zdev->kzdev->fib.fmt0.aibv = 0; + + spin_lock_irq(&aift->gait_lock); + gaite->count--; + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = NULL; + spin_unlock_irq(&aift->gait_lock); + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; + +free_aisb: + airq_iv_free_bit(aift->sbv, zdev->aisb); + zdev->aisb = 0; unlock: + if (pcount > 0) + unaccount_mem(zdev->kzdev, pcount); mutex_unlock(&aift->aift_lock); unpin2: if (fib->fmt0.sum == 1) @@ -328,6 +398,7 @@ unpin2: unpin1: unpin_user_page(aibv_page); out: + kvm_s390_gisc_unregister(kvm, fib->fmt0.isc); return rc; } @@ -390,7 +461,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) pcount++; } if (pcount > 0) - unaccount_mem(pcount); + unaccount_mem(kzdev, pcount); out: mutex_unlock(&aift->aift_lock); diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index ff0972dd5e71..fdf8c7bf4ed0 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -22,6 +22,8 @@ struct kvm_zdev { struct kvm *kvm; struct zpci_fib fib; struct list_head entry; + struct user_struct *user_account; + struct mm_struct *mm_account; }; struct zpci_gaite { diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index ad0ddc433a73..b1ba24c346ef 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -289,6 +289,7 @@ static int handle_iske(struct kvm_vcpu *vcpu) static int handle_rrbe(struct kvm_vcpu *vcpu) { unsigned long gaddr; + union skey skey; int reg1, reg2; int rc; @@ -307,12 +308,12 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr); scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr)); + rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &skey); if (rc > 0) return kvm_s390_inject_program_int(vcpu, rc); if (rc < 0) return rc; - kvm_s390_set_psw_cc(vcpu, rc); + kvm_s390_set_psw_cc(vcpu, (skey.skey >> 1) & 3); return 0; } @@ -1260,8 +1261,9 @@ static int handle_essa(struct kvm_vcpu *vcpu) /* Retry the ESSA instruction */ kvm_s390_retry_instr(vcpu); } else { - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - i = __do_essa(vcpu, orc); + scoped_guard(mutex, &vcpu->kvm->slots_arch_lock) + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + i = __do_essa(vcpu, orc); if (i < 0) return i; /* Account for the possible extra cbrl entry */ diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index 1beacc841ca8..b02e0159d3cd 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -244,6 +244,24 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm) kvm->arch.pv.stor_var = NULL; } +static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base) +{ + if (free_stor_base) + free_pages(vcpu->arch.pv.stor_base, get_order(uv_info.guest_cpu_stor_len)); + free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); + vcpu->arch.sie_block->pv_handle_cpu = 0; + vcpu->arch.sie_block->pv_handle_config = 0; + memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); + vcpu->arch.sie_block->sdf = 0; + /* + * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). + * Use the reset value of gbea to avoid leaking the kernel pointer of + * the just freed sida. + */ + vcpu->arch.sie_block->gbea = 1; + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); +} + int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) { int cc; @@ -258,24 +276,9 @@ int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); /* Intended memory leak for something that should never happen. */ - if (!cc) - free_pages(vcpu->arch.pv.stor_base, - get_order(uv_info.guest_cpu_stor_len)); - - free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); - vcpu->arch.sie_block->pv_handle_cpu = 0; - vcpu->arch.sie_block->pv_handle_config = 0; - memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); - vcpu->arch.sie_block->sdf = 0; - /* - * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). - * Use the reset value of gbea to avoid leaking the kernel pointer of - * the just freed sida. - */ - vcpu->arch.sie_block->gbea = 1; - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_s390_pv_dispose_cpu(vcpu, !cc); - return cc ? EIO : 0; + return cc ? -EIO : 0; } int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) @@ -319,9 +322,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) uvcb.header.rrc); if (cc) { - u16 dummy; - - kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); + kvm_s390_pv_dispose_cpu(vcpu, true); return -EIO; } @@ -809,7 +810,7 @@ static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, return -EAGAIN; } - if (ret && ret != -EAGAIN) + if (ret && ret != -EAGAIN && ret != -EINTR) KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", uvcb.gaddr, *rc, *rrc); return ret; diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c index 458abd9bac70..9d74ceff136c 100644 --- a/arch/s390/lib/csum-partial.c +++ b/arch/s390/lib/csum-partial.c @@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w if (!cpu_has_vx()) { if (copy) memcpy(dst, src, len); - return cksm(dst, len, sum); + return cksm(src, len, sum); } kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); fpu_vlvgf(16, (__force u32)sum, 1); diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 2a222a7e14f4..ef7bfc87758c 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -64,7 +64,7 @@ static inline unsigned long mmap_base(unsigned long rnd, return PAGE_ALIGN(STACK_TOP - gap - rnd); } -static int get_align_mask(struct file *filp, unsigned long flags) +static unsigned long get_align_mask(struct file *filp, unsigned long flags) { if (filp && is_file_hugepages(filp)) return huge_page_mask_align(filp); diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 51e7a28af899..f3f79ba78410 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -188,6 +188,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, goto out_unlock_mmap; } + ret = -EFAULT; io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 2cc90055499a..8a70b3a62553 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -995,6 +995,9 @@ static int vector_mmsg_rx(struct vector_private *vp, int budget) */ dev_kfree_skb_irq(skb); vp->estats.rx_encaps_errors++; + (*skbuff_vector) = NULL; + mmsg_vector++; + skbuff_vector++; continue; } if (header_check > 0) { diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 07e0e64b9a98..06934f9691d6 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -27,6 +27,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS) KBUILD_CFLAGS += $(CC_FLAGS_DIALECT) KBUILD_CFLAGS += -fno-strict-aliasing -fPIE +KBUILD_CFLAGS += -fno-jump-tables KBUILD_CFLAGS += -Wundef KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING cflags-$(CONFIG_X86_32) := -march=i386 diff --git a/arch/x86/boot/compressed/acpi.c b/arch/x86/boot/compressed/acpi.c index f196b1d1ddf8..aed27604c11f 100644 --- a/arch/x86/boot/compressed/acpi.c +++ b/arch/x86/boot/compressed/acpi.c @@ -184,10 +184,15 @@ static unsigned long get_cmdline_acpi_rsdp(void) char val[MAX_ADDR_LEN] = { }; int ret; - ret = cmdline_find_option("acpi_rsdp", val, MAX_ADDR_LEN); + ret = cmdline_find_option("acpi_rsdp", val, sizeof(val)); if (ret < 0) return 0; + if (ret >= sizeof(val)) { + warn("acpi_rsdp= value too long; ignoring"); + return 0; + } + if (boot_kstrtoul(val, 16, &addr)) return 0; #endif diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c index 023bf1c3de8b..5b83beab89e1 100644 --- a/arch/x86/boot/early_serial_console.c +++ b/arch/x86/boot/early_serial_console.c @@ -117,7 +117,7 @@ static unsigned int probe_baud(int port) static void parse_console_uart8250(void) { char optstr[64], *options; - int baud = DEFAULT_BAUD; + int baud; int port = 0; /* @@ -136,10 +136,13 @@ static void parse_console_uart8250(void) else return; - if (options && (options[0] == ',')) - baud = simple_strtoull(options + 1, &options, 0); - else + if (options && (options[0] == ',')) { + baud = simple_strtoull(options + 1, NULL, 0); + if (!baud) + baud = DEFAULT_BAUD; + } else { baud = probe_baud(port); + } if (port) early_serial_init(port, baud); diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index ac0f900ebc47..1632d40e1f54 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -32,8 +32,15 @@ int memcmp(const void *s1, const void *s2, size_t len) { bool diff; - asm("repe cmpsb" - : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + + /* + * Make sure ZF is properly set in the len==0 case because in it, + * RCX==0 and the REPE; CMPSB won't get executed. + */ + asm volatile("test %3, %3\n\t" + "repe cmpsb" + : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len) + : : "cc", "memory"); return diff; } diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index c6d996593f32..253f0a585927 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -937,6 +937,8 @@ SYM_CODE_START(paranoid_entry) IBRS_ENTER save_reg=%r15 UNTRAIN_RET_FROM_CALL + HANDLE_INTR_SAFERET 8(%rsp) + RET SYM_CODE_END(paranoid_entry) @@ -1039,6 +1041,11 @@ SYM_CODE_START(error_entry) movl %ecx, %eax /* zero extend */ cmpq %rax, RIP+8(%rsp) je .Lbstep_iret + + VALIDATE_UNRET_END + + HANDLE_INTR_SAFERET 8(%rsp) + cmpq $.Lgs_change, RIP+8(%rsp) jne .Lerror_entry_done_lfence @@ -1057,7 +1064,6 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - VALIDATE_UNRET_END RET .Lbstep_iret: diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index 06f35a6b58a5..dc564688f3d7 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -259,13 +259,13 @@ void amd_brs_disable_all(void) amd_brs_disable(); } -static bool amd_brs_match_plm(struct perf_event *event, u64 to) +static bool amd_brs_match_plm(struct perf_event *event, u64 from, u64 to) { int type = event->attr.branch_sample_type; int plm_k = PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_HV; int plm_u = PERF_SAMPLE_BRANCH_USER; - if (!(type & plm_k) && kernel_ip(to)) + if (!(type & plm_k) && (kernel_ip(to) || kernel_ip(from))) return 0; if (!(type & plm_u) && !kernel_ip(to)) @@ -338,11 +338,11 @@ void amd_brs_drain(void) */ to = (u64)(((s64)to << shift) >> shift); - if (!amd_brs_match_plm(event, to)) - continue; - rdmsrq(brs_from(brs_idx), from); + if (!amd_brs_match_plm(event, from, to)) + continue; + perf_clear_branch_entry_bitfields(br+nr); br[nr].from = from; diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 6569048a8c1c..a787409f5a62 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -754,13 +754,11 @@ static void amd_pmu_enable_event(struct perf_event *event) x86_pmu_enable_event(event); } -static void amd_pmu_enable_all(int added) +static void __amd_pmu_enable_all(void) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx; - amd_brs_enable_all(); - for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { /* only activate events which are marked as active */ if (!test_bit(idx, cpuc->active_mask)) @@ -775,6 +773,12 @@ static void amd_pmu_enable_all(int added) } } +static void amd_pmu_enable_all(int added) +{ + amd_brs_enable_all(); + __amd_pmu_enable_all(); +} + static void amd_pmu_v2_enable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -1561,7 +1565,7 @@ static inline void amd_pmu_reload_virt(void) * set global enable bits once again */ amd_pmu_v2_disable_all(); - amd_pmu_enable_all(0); + __amd_pmu_enable_all(); amd_pmu_v2_enable_all(0); return; } diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 5b437dc8e4ce..9d9c961989d5 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -127,7 +127,8 @@ static void amd_pmu_lbr_filter(void) } /* If type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && kernel_ip(cpuc->lbr_entries[i].from))) { cpuc->lbr_entries[i].from = 0; /* mark invalid */ compress = true; } diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5f6c1ce9673b..6db5b5f79df9 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1858,7 +1858,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index b68892e6d58c..2ea6591bc7b9 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -12,6 +12,7 @@ #include <asm/msr-index.h> #include <asm/unwind_hints.h> #include <asm/percpu.h> +#include <asm/ptrace-abi.h> /* * Call depth tracking for Intel SKL CPUs to address the RSB underflow @@ -176,6 +177,50 @@ add $(BITS_PER_LONG/8), %_ASM_SP; \ lfence; +/* + * Helper for detecting if an interrupt occurred at an unsafe location within + * Safe-RET. If Safe-RET is interrupted after the CALL or LEA the RSB may get + * poisoned by the interrupt handler. + * + * The Safe-RET sequence is: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * The two CMPs below check whether RIP points to after the CALL or after the + * LEA. + * + * The LFENCE below is to address this particular speculation case: + * + * 1. Userspace runs and poisons the BTB around the safe-RET routine + * + * 2. Userspace triggers some kind of exception + * + * 3. Kernel executes error_entry() and mis-speculates the branch into thinking + * it actually came from kernel space + * + * 4. The kernel then further mis-speculates that the exception occurred due + * to an interrupted safe-RET + * + * 5. The handle_interrupted_saferet() routine speculatively executes and + * speculatively does a safe-RET. But this is unsafe since it was never + * untrained. + * + * The LFENCE fixes this by ensuring step 5 is never reached speculatively. + * Note that this LFENCE only occurs if safe-RET was actually interrupted (so + * it's outside of the normal path). + */ +#define __HANDLE_INTR_SAFERET(name, pt_regs) \ + cmpq $(name), RIP+pt_regs; \ + jb 1f; \ + cmpq $(name)+5, RIP+pt_regs; \ + ja 1f; \ + lfence; \ + leaq pt_regs, %rdi; \ + call handle_interrupted_saferet; \ + 1: + #ifdef __ASSEMBLER__ /* @@ -293,6 +338,14 @@ #define UNTRAIN_RET_FROM_CALL \ __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL) +.macro HANDLE_INTR_SAFERET pt_regs +#ifdef CONFIG_MITIGATION_SRSO + ALTERNATIVE_2 "", \ + __stringify(__HANDLE_INTR_SAFERET(srso_safe_ret, \pt_regs)), X86_FEATURE_SRSO, \ + __stringify(__HANDLE_INTR_SAFERET(srso_alias_safe_ret, \pt_regs)), X86_FEATURE_SRSO_ALIAS + +#endif +.endm .macro CALL_DEPTH_ACCOUNT #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING @@ -625,6 +678,10 @@ static __always_inline void x86_idle_clear_cpu_buffers(void) x86_clear_cpu_buffers(); } +void srso_safe_ret(void); +void srso_alias_safe_ret(void); +void handle_interrupted_saferet(struct pt_regs *regs); + #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index d2e8a849f180..5b1070ec85d9 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -46,7 +46,6 @@ obj-$(CONFIG_CPU_SUP_HYGON) += hygon.o obj-$(CONFIG_CPU_SUP_CYRIX_32) += cyrix.o obj-$(CONFIG_CPU_SUP_CENTAUR) += centaur.o obj-$(CONFIG_CPU_SUP_TRANSMETA_32) += transmeta.o -obj-$(CONFIG_CPU_SUP_UMC_32) += umc.o obj-$(CONFIG_CPU_SUP_ZHAOXIN) += zhaoxin.o obj-$(CONFIG_CPU_SUP_VORTEX_32) += vortex.o diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 487ac147e11f..d61df70d6875 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -515,11 +515,13 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) case 0x00 ... 0x2f: case 0x40 ... 0x4f: case 0x60 ... 0x7f: + case 0xd0 ... 0xd7: setup_force_cpu_cap(X86_FEATURE_ZEN5); break; case 0x50 ... 0x5f: case 0x80 ... 0xaf: - case 0xc0 ... 0xef: + case 0xc0 ... 0xcf: + case 0xd8 ... 0xef: setup_force_cpu_cap(X86_FEATURE_ZEN6); break; default: diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index d9af230c0512..82436b3534fa 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -3775,3 +3775,42 @@ void __warn_thunk(void) { WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); } + +#ifdef CONFIG_MITIGATION_SRSO +/* + * Called during exception/interrupt entry if interrupted during the + * safe-RET sequence. The safe-RET sequence consists of 3 instructions: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * An interrupt after the CALL or after the LEA could potentially lead + * to branch predictor poisoning and results in the sequence not being + * able to be safely resumed. + * + * Therefore, modify the regs state as if the remaining part of the + * safe-RET sequence executed so the interrupt returns back to the + * desired return target, instead of the to the safe-RET sequence. + */ +void noinstr handle_interrupted_saferet(struct pt_regs *regs) +{ + unsigned long rip = regs->ip; + + if (rip == (unsigned long) srso_safe_ret || + rip == (unsigned long) srso_alias_safe_ret) { + /* Modify stack pointer as if LEA executed: */ + regs->sp += 8; + } + + /* + * Adjust registers as if RET executed: + * + * 1. Read the return address off the stack and into rIP: + */ + regs->ip = *(unsigned long *)(regs->sp); + + /* 2. Pop rIP off the stack: */ + regs->sp += 8; +} +#endif /* CONFIG_MITIGATION_SRSO */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a3df21d26460..a3caddd411ec 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2664,3 +2664,13 @@ void __init arch_cpu_finalize_init(void) */ mem_encrypt_init(); } + +/* Control TLB flushing methods */ +static int __init tlbi_setup(char *str) +{ + if (!strcmp(str, "ipi")) + setup_clear_cpu_cap(X86_FEATURE_INVLPGB); + + return 1; +} +__setup("tlbi=", tlbi_setup); diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 9bba1e2f03af..cfb74be19994 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -2266,10 +2266,10 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c) mca_cfg.initialized = 1; + __mcheck_cpu_setup_timer(); __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(c); __mcheck_cpu_init_prepare_banks(); - __mcheck_cpu_setup_timer(); cr4_set_bits(X86_CR4_MCE); } diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index fd4eb1e561f7..d8736b5dfcd3 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1974,6 +1974,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; + bool full = false; if (!tdp_enabled || !hv_vcpu) return -EINVAL; @@ -1982,7 +1983,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count; i++) { + for (i = 0; i < count && !full; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -1991,11 +1992,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); } ++vcpu->stat.tlb_flush; diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 8c62c6d4d5c1..cb8ac4b9b0d7 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -488,8 +488,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) { ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry); - if (ret) + if (ret) { kvm->arch.nr_possible_bypass_irqs--; + irqfd->producer = NULL; + } } spin_unlock_irq(&kvm->irqfds.lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f30bbdddb5a..48b019114c19 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2052,7 +2052,7 @@ static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) ktimer->expired_tscdeadline = ktimer->tscdeadline; - if (!from_timer_fn && apic->apicv_active) { + if (!from_timer_fn && apic->apicv_active && vcpu->wants_to_run) { WARN_ON(kvm_get_running_vcpu() != vcpu); kvm_apic_inject_pending_timer_irqs(apic); return; @@ -3371,6 +3371,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { int vector; + + if (unlikely(!pv_eoi_enabled(vcpu))) { + __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + return; + } + /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host * and KVM_PV_EOI_ENABLED in guest memory as follows: @@ -3382,8 +3388,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: * -> host enabled PV EOI, guest executed EOI. */ - BUG_ON(!pv_eoi_enabled(vcpu)); - if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 234d0a95abf5..a61750f8e1e3 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -722,6 +722,26 @@ static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn, return &slot->arch.lpage_info[level - 2][idx]; } +static bool kvm_gfn_is_lpage_allowed(struct kvm *kvm, + const struct kvm_memory_slot *slot, + gfn_t gfn, int level) +{ + const struct kvm_memory_slot *other_slot; + + BUILD_BUG_ON(KVM_MAX_NR_ADDRESS_SPACES > 2); + + if (lpage_info_slot(gfn, slot, level)->disallow_lpage) + return false; + + if (kvm_arch_nr_memslot_as_ids(kvm) > 1) { + other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn); + if (other_slot && lpage_info_slot(gfn, other_slot, level)->disallow_lpage) + return false; + } + + return true; +} + /* * The most significant bit in disallow_lpage tracks whether or not memory * attributes are mixed, i.e. not identical for all gfns at the current level. @@ -2422,6 +2442,9 @@ static union kvm_mmu_page_role kvm_mmu_child_role(u64 *sptep, bool direct, role.direct = direct; role.passthrough = 0; + WARN_ON_ONCE(role.invalid); + role.invalid = 0; + /* * If the guest has 4-byte PTEs then that means it's using 32-bit, * 2-level, non-PAE paging. KVM shadows such guests with PAE paging @@ -2642,6 +2665,7 @@ static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, */ if (tdp_enabled && invalid_list && child->role.guest_mode && + !child->root_count && !atomic_long_read(&child->parent_ptes.val)) return kvm_mmu_prepare_zap_page(kvm, child, invalid_list); @@ -2967,7 +2991,7 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, * write-protected (see above), thus if the gfn can be mapped with a * hugepage and isn't write-tracked, it can't have a shadow page. */ - if (!lpage_info_slot(gfn, slot, PG_LEVEL_2M)->disallow_lpage) + if (kvm_gfn_is_lpage_allowed(kvm, slot, gfn, PG_LEVEL_2M)) return 0; /* @@ -4852,16 +4876,17 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault if (r != RET_PF_CONTINUE) return r; - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = direct_map(vcpu, fault); out_unlock: @@ -6630,7 +6655,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr); + kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); } if (!mmu->sync_spte) @@ -7574,7 +7599,9 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) static void mmu_destroy_caches(void) { kmem_cache_destroy(pte_list_desc_cache); + pte_list_desc_cache = NULL; kmem_cache_destroy(mmu_page_header_cache); + mmu_page_header_cache = NULL; } static void kvm_wake_nx_recovery_thread(struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 1b17b12393a8..7e8195a311bb 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -130,13 +130,23 @@ void __kvm_write_track_remove_gfn(struct kvm *kvm, kvm_mmu_gfn_allow_lpage(slot, gfn); } -/* - * check if the corresponding access on the specified guest page is tracked. - */ +static bool __kvm_gfn_is_write_tracked(const struct kvm_memory_slot *slot, + gfn_t gfn) +{ + int index; + + if (!slot) + return false; + + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); + return !!READ_ONCE(slot->arch.gfn_write_track[index]); +} + +/* check if write access is tracked on the specified guest page. */ bool kvm_gfn_is_write_tracked(struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn) { - int index; + const struct kvm_memory_slot *other_slot; if (!slot) return false; @@ -144,8 +154,18 @@ bool kvm_gfn_is_write_tracked(struct kvm *kvm, if (!kvm_page_track_write_tracking_enabled(kvm)) return false; - index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); - return !!READ_ONCE(slot->arch.gfn_write_track[index]); + BUILD_BUG_ON(KVM_MAX_NR_ADDRESS_SPACES > 2); + + if (__kvm_gfn_is_write_tracked(slot, gfn)) + return true; + + if (kvm_arch_nr_memslot_as_ids(kvm) > 1) { + other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn); + if (__kvm_gfn_is_write_tracked(other_slot, gfn)) + return true; + } + + return false; } #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index df3ae0c7ec2c..1ba840a73b7a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -864,15 +864,17 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault } #endif - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = FNAME(fetch)(vcpu, fault, &walker); out_unlock: diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 58e493a80cb0..16e09a8fe85e 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -240,14 +240,6 @@ static void avic_deactivate_vmcb(struct vcpu_svm *svm) if (!is_sev_es_guest(&svm->vcpu)) svm_set_intercept(svm, INTERCEPT_CR8_WRITE); - /* - * If running nested and the guest uses its own MSR bitmap, there - * is no need to update L0's msr bitmap - */ - if (is_guest_mode(&svm->vcpu) && - vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)) - return; - /* Enabling MSR intercept for x2APIC registers */ avic_set_x2apic_msr_interception(svm, true); } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 427229347876..0f0ea7896af5 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -97,6 +97,8 @@ static u64 sev_supported_vmsa_features __ro_after_init; static u8 sev_enc_bit; static DECLARE_RWSEM(sev_deactivate_lock); static DEFINE_MUTEX(sev_bitmap_lock); +/* Protects kvm_sev_info's enc_context_owner, mirror_vms and mirror_entry. */ +static DEFINE_MUTEX(sev_mirror_lock); unsigned int max_sev_asid; static unsigned int min_sev_asid; static unsigned int max_sev_es_asid; @@ -2018,7 +2020,6 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->asid = src->asid; dst->handle = src->handle; dst->pages_locked = src->pages_locked; - dst->enc_context_owner = src->enc_context_owner; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; @@ -2026,11 +2027,12 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) src->active = false; src->handle = 0; src->pages_locked = 0; - src->enc_context_owner = NULL; src->es_active = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); + mutex_lock(&sev_mirror_lock); + /* * If this VM has mirrors, "transfer" each mirror's refcount of the * source to the destination (this KVM). The caller holds a reference @@ -2047,12 +2049,15 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) * If this VM is a mirror, remove the old mirror from the owners list * and add the new mirror to the list. */ - if (is_mirroring_enc_context(dst_kvm)) { - struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(dst->enc_context_owner); + if (is_mirroring_enc_context(src_kvm)) { + struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(src->enc_context_owner); + dst->enc_context_owner = src->enc_context_owner; + src->enc_context_owner = NULL; list_del(&src->mirror_entry); list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms); } + mutex_unlock(&sev_mirror_lock); kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) { dst_svm = to_svm(dst_vcpu); @@ -2129,8 +2134,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (ret) return ret; + /* Do not allow SNP VM migration until additional state transfer is implemented */ if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { + sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { ret = -EINVAL; goto out_unlock; } @@ -2851,8 +2857,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disallow out-of-band SEV/SEV-ES init if the target is already an * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being * created after SEV/SEV-ES initialization, e.g. to init intercepts. + * Also do not allow SNP VM mirroring until additional state transfer is implemented. */ - if (sev_guest(kvm) || !sev_guest(source_kvm) || + if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; goto e_unlock; @@ -2869,11 +2876,14 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disappear until we're done with it */ source_sev = to_kvm_sev_info(source_kvm); - kvm_get_kvm(source_kvm); - list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); /* Set enc_context_owner and copy its encryption context over */ + mutex_lock(&sev_mirror_lock); + kvm_get_kvm(source_kvm); + list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); mirror_sev->enc_context_owner = source_kvm; + mutex_unlock(&sev_mirror_lock); + mirror_sev->active = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; @@ -2961,11 +2971,19 @@ void sev_vm_destroy(struct kvm *kvm) * Note, mirror VMs don't support registering encrypted regions. */ if (is_mirroring_enc_context(kvm)) { - struct kvm *owner_kvm = sev->enc_context_owner; + struct kvm *owner_kvm; - mutex_lock(&owner_kvm->lock); + mutex_lock(&sev_mirror_lock); + owner_kvm = sev->enc_context_owner; list_del(&sev->mirror_entry); - mutex_unlock(&owner_kvm->lock); + sev->enc_context_owner = NULL; + + /* + * The reference to owner_kvm cannot move after sev_mirror_lock is + * released. Release it before kvm_put_kvm() so that owner_kvm is + * never destroyed inside sev_mirror_lock. + */ + mutex_unlock(&sev_mirror_lock); kvm_put_kvm(owner_kvm); return; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4d2bacd00ec4..60495447f0ca 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -571,7 +571,12 @@ static int svm_enable_virtualization_cpu(void) return r; sd = per_cpu_ptr(&svm_data, me); - sd->asid_generation = 1; + /* + * Bump the current asid_generation value to ensure any vCPU that + * previously ran on this CPU sees a stale generation and is forced + * to acquire a new ASID, preventing a latent ASID collision. + */ + sd->asid_generation++; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; @@ -4222,18 +4227,31 @@ static void svm_flush_tlb_all(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) +static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) { - struct vcpu_svm *svm = to_svm(vcpu); + kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS); - invlpga(gva, svm->vmcb->control.asid); + svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) { - kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS); + struct vcpu_svm *svm = to_svm(vcpu); - svm_flush_tlb_asid(vcpu); + /* + * INVLPGA has had errata on Genoa and Turin, and even on older + * generations there were reports of Windows BSODs if INVLPGA + * was used for Hyper-V tlbflush. Use it only for shadow paging + * where it seems to be okay. + */ + if (!npt_enabled) { + invlpga(gva, svm->vmcb->control.asid); + return; + } + + svm_flush_tlb_guest(vcpu); + if (full) + *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 716be21fba33..d63e5878988a 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -109,6 +109,7 @@ struct kvm_sev_info { u64 ap_jump_table; /* SEV-ES AP Jump Table address */ u64 vmsa_features; u16 ghcb_version; /* Highest guest GHCB protocol version allowed */ + /* The three fields below are protected by sev_mirror_lock */ struct kvm *enc_context_owner; /* Owner of copied encryption context */ struct list_head mirror_vms; /* List of VMs mirroring */ struct list_head mirror_entry; /* Use as a list entry of mirrors */ diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 0db25bba17f6..93de876c318c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -490,7 +490,7 @@ TRACE_EVENT(kvm_inj_exception, TP_printk("%s%s%s%s%s", __print_symbolic(__entry->exception, kvm_trace_sym_exc), !__entry->has_error ? "" : " (", - !__entry->has_error ? "" : __print_symbolic(__entry->error_code, { }), + !__entry->has_error ? "" : __print_symbolic(__entry->error_code), !__entry->has_error ? "" : ")", __entry->reinjected ? " [reinjected]" : "") ); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 83d9921277ea..f204a0fc0a57 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -535,12 +535,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr); + vmx_flush_tlb_gva(vcpu, addr, full); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6957bb6f5cf7..ddf6df7bee93 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -336,6 +336,7 @@ static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) static void free_nested(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct vmcs *shadow_vmcs; if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) vmx_switch_vmcs(vcpu, &vmx->vmcs01); @@ -353,9 +354,15 @@ static void free_nested(struct kvm_vcpu *vcpu) vmx->nested.current_vmptr = INVALID_GPA; if (enable_shadow_vmcs) { vmx_disable_shadow_vmcs(vmx); - vmcs_clear(vmx->vmcs01.shadow_vmcs); - free_vmcs(vmx->vmcs01.shadow_vmcs); + + /* + * Keep the pointer visible until after VMCLEAR, so migration + * can clear an active shadow VMCS on the old CPU. + */ + shadow_vmcs = vmx->vmcs01.shadow_vmcs; + vmcs_clear(shadow_vmcs); vmx->vmcs01.shadow_vmcs = NULL; + free_vmcs(shadow_vmcs); } kfree(vmx->nested.cached_vmcs12); vmx->nested.cached_vmcs12 = NULL; @@ -582,6 +589,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; + u32 vtpr; + if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) return 0; @@ -591,6 +601,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) return -EINVAL; + /* + * Do the illegal vTPR vs. TPR Threshold consistency check if and only + * if KVM is configured to WARN on missed consistency checks, otherwise + * it's a waste of time. KVM needs to rely on hardware to fully detect + * an illegal combination due to the vTPR being writable by L1 at all + * times (it's an in-memory value, not a VMCS field). I.e. even if the + * check passes now, it might fail at the actual VM-Enter. + * + * If reading guest memory fails, skip the check as KVM's de facto ABI + * for VMX instruction accesses to non-existent memory is to provide + * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR + * is guaranteed to greater than or equal to the threshold. + * + * Note! Deliberately use the VM-scoped API when reading guest memory, + * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, + * and only perform the check when in KVM_RUN, to avoid a false failure + * if userspace hasn't yet configured memslots during state restore. + */ + if (warn_on_missed_cc && vcpu->wants_to_run && + nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && + !nested_cpu_has_vid(vmcs12) && + !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && + !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && + CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) + return -EINVAL; + return 0; } @@ -3104,38 +3140,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, return 0; } -static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) -{ - void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; - u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; - - /* - * Don't bother with the consistency checks if KVM isn't configured to - * WARN on missed consistency checks, as KVM needs to rely on hardware - * to fully detect an illegal vTPR vs. TRP Threshold combination due to - * the vTPR being writable by L1 at all times (it's an in-memory value, - * not a VMCS field). I.e. even if the check passes now, it might fail - * at the actual VM-Enter. - * - * Keying off the module param also allows treating an invalid vAPIC - * mapping as a consistency check failure without increasing the risk - * of breaking a "real" VM. - */ - if (!warn_on_missed_cc) - return 0; - - if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && - !nested_cpu_has_vid(vmcs12) && - !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && - (CC(!vapic) || - CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) - return -EINVAL; - - return 0; -} - static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { @@ -3661,19 +3665,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, &vmx->nested.pre_vmenter_ssp_tbl); /* - * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the - * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but - * not KVM, KVM must unwind its software model to the pre-VM-Entry host - * state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not - * L1's "real" CR3, which causes nested_vmx_restore_host_state() to - * corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the - * unwind naturally setting arch.cr3 to the correct value. Smashing - * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind, - * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be - * overwritten with a shadow CR3 prior to re-entering L1. + * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a + * VM-Fail detected by hardware but not KVM, KVM can unwind its + * software model to the pre-VM-Entry host state. When EPT is + * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, + * and so simply restoring from vmcs01.GUEST_CR3 would corrupt + * vcpu->arch.cr3. */ - if (!enable_ept) - vmcs_writel(GUEST_CR3, vcpu->arch.cr3); + vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu); vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); @@ -3685,11 +3684,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_KVM_INTERNAL_ERROR; } - if (nested_vmx_check_controls_late(vcpu, vmcs12)) { - vmx_switch_vmcs(vcpu, &vmx->vmcs01); - return NVMX_VMENTRY_VMFAIL; - } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; @@ -3774,6 +3768,8 @@ vmentry_fail_vmexit: if (!from_vmentry) return NVMX_VMENTRY_VMEXIT; + nested_put_vmcs12_pages(vcpu); + load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) @@ -4990,7 +4986,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); nested_ept_uninit_mmu_context(vcpu); - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3; kvm_register_mark_available(vcpu, VCPU_REG_CR3); /* diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 989ab29b8c6f..545b03d9d10b 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2797,7 +2797,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) goto out; } - if (init_vm->cpuid.padding) { + /* + * Reject the request if userspace changes cpuid.nent between the + * initial read and the subsequent copy. + */ + if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) { ret = -EINVAL; goto out; } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cc75feec05da..b8d745f6fd22 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3361,7 +3361,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index de9de0d2016c..dc8517f15bc4 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -160,6 +160,13 @@ struct nested_vmx { bool preemption_timer_expired; /* + * Used to restore L1's CR3 if hardware detects a VM-Fail Consistency + * Check that KVM does not, in which case KVM needs to unwind CR3 back + * to its pre-VM-Enter state, NOT to vmcs01.HOST_CR3. + */ + unsigned long pre_vmenter_cr3; + + /* * Used to snapshot MSRs that are conditionally loaded on VM-Enter in * order to propagate the guest's pre-VM-Enter value into vmcs02. For * emulation of VMLAUNCH/VMRESUME, the snapshot will be of L1's value. diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 81784befaaf4..9ffe03a0c223 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -101,7 +101,7 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) : [output] "=r" (value) : [field] "r" (field) - : "cc" + : "cc", "memory" : do_fail, do_exception); return value; @@ -145,7 +145,7 @@ do_exception: : ASM_CALL_CONSTRAINT, [output] "=&r" (value) : [field] "r" (field) - : "cc"); + : "cc", "memory"); return value; #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ @@ -192,7 +192,7 @@ do { \ asm goto("1: " __stringify(insn) " %0\n\t" \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1 : "cc" : error, fault); \ + : : op1 : "cc", "memory" : error, fault); \ return; \ error: \ instrumentation_begin(); \ @@ -208,7 +208,7 @@ do { \ asm goto("1: " __stringify(insn) " %1, %0\n\t" \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1, op2 : "cc" : error, fault); \ + : : op1, op2 : "cc", "memory" : error, fault);\ return; \ error: \ instrumentation_begin(); \ diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 409858074246..17595d52985c 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -82,7 +82,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index afcac1042947..47cb9eba113b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -13429,9 +13429,15 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm) * iterating over vCPUs in a different task while vCPUs are being freed * is unsafe, i.e. will lead to use-after-free. The PIT also needs to * be stopped before IRQ routing is freed. + * + * Do NOT free the in-kernel PIC or I/O APIC here (but as above, make + * sure to flush any background work), as KVM expects interrupt routing + * structures to be valid until vCPUs are destroyed. */ #ifdef CONFIG_KVM_IOAPIC kvm_free_pit(kvm); + if (kvm->arch.vioapic) + cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject); #endif kvm_mmu_pre_destroy_vm(kvm); diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 8f1fed0c3b83..f9ca1d8eabca 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -207,10 +207,24 @@ __EXPORT_THUNK(srso_alias_untrain_ret) .pushsection .text..__x86.rethunk_safe SYM_CODE_START_NOALIGN(srso_alias_safe_ret) + + /* + * Tell objtool that those are not function pointers referenced by + * __HANDLE_INTR_SAFERET(). Below too. + */ + ANNOTATE_NOENDBR + + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP UNWIND_HINT_FUNC + + ANNOTATE_NOENDBR ANNOTATE_UNRET_SAFE ret + /* End of Safe-RET sequence */ int3 SYM_FUNC_END(srso_alias_safe_ret) @@ -245,8 +259,14 @@ SYM_CODE_START_LOCAL_NOALIGN(srso_untrain_ret) * the stack. */ SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP ret + /* End of Safe-RET sequence */ + int3 int3 /* end of movabs */ diff --git a/arch/x86/video/video-common.c b/arch/x86/video/video-common.c index 152789f00fcd..8ed82fff7638 100644 --- a/arch/x86/video/video-common.c +++ b/arch/x86/video/video-common.c @@ -43,21 +43,26 @@ bool video_is_primary_device(struct device *dev) if (!pci_is_display(pdev)) return false; - if (pdev == vga_default_device()) - return true; - #ifdef CONFIG_SCREEN_INFO numres = screen_info_resources(si, res, ARRAY_SIZE(res)); - for (i = 0; i < numres; ++i) { - if (!(res[i].flags & IORESOURCE_MEM)) - continue; + if (numres > 0) { + for (i = 0; i < numres; ++i) { + if (!(res[i].flags & IORESOURCE_MEM)) + continue; + + if (pci_find_resource(pdev, &res[i])) + return true; + } - if (pci_find_resource(pdev, &res[i])) - return true; + return false; } #endif - return false; + /* + * No framebuffer was set up by the firmware/bootloader, so fall back + * to the default VGA device. + */ + return pdev == vga_default_device(); } EXPORT_SYMBOL(video_is_primary_device); diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 8bcdce98f6dc..cff285d8ad8e 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -536,6 +536,8 @@ int snp_prepare(void) goto unlock; } + wbinvd_on_all_cpus(); + /* * MtrrFixDramModEn is not shared between threads on a core, * therefore it must be set on all CPUs prior to enabling SNP. |