diff options
| author | Mark Brown <broonie@kernel.org> | 2026-10-27 13:26:15 +0000 |
|---|---|---|
| committer | Mark Brown <broonie@kernel.org> | 2026-10-27 13:26:15 +0000 |
| commit | c2f570c342c74591abdddfc188a64240a97bb2d5 (patch) | |
| tree | e4c38675b9fdb90d489347c65afe29d1ee31aa10 | |
| parent | 365cdd7e39664d568d68bc901593be600b57e29c (diff) | |
| parent | a5ae1bf65cee84bcc121d659e7e22a767853f5f5 (diff) | |
| download | linux-next-c2f570c342c74591abdddfc188a64240a97bb2d5.tar.gz linux-next-c2f570c342c74591abdddfc188a64240a97bb2d5.zip | |
Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
58 files changed, 1170 insertions, 485 deletions
diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst index bb1e0cbab36f..8454a6ae6fd1 100644 --- a/Documentation/arch/riscv/hwprobe.rst +++ b/Documentation/arch/riscv/hwprobe.rst @@ -423,3 +423,11 @@ The following keys are defined: * :c:macro:`RISCV_HWPROBE_EXT_ZA64RS`: The Za64rs extension is supported, as defined in the RISC-V Profiles specification starting from commit b1d80660 ("Updated to ratified state.") + + * :c:macro:`RISCV_HWPROBE_EXT_B`: The B extension is supported, as defined + in version 1.0 of the Bit-Manipulation ISA extensions, and implies the + presence of the Zba, Zbb, and Zbs sub-extensions. + + * :c:macro:`RISCV_HWPROBE_EXT_ZIC64B`: The Zic64b extension is supported, + as defined in the RISC-V Profiles specification starting from commit + b1d80660 ("Updated to ratified state.") diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index 577d7ae5576b..79491ef8f86a 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -537,6 +537,12 @@ properties: in commit 64074bc ("Update version numbers for Zfh/Zfinx") of riscv-isa-manual. + - const: zic64b + description: + The standard Zic64b extension for 64-byte naturally aligned cache + blocks, as ratified in RISC-V Profiles Version 1.0, with commit + b1d806605f87 ("Updated to ratified state.") + - const: zicbom description: The standard Zicbom extension for base cache management operations as @@ -1211,6 +1217,54 @@ allOf: not: contains: const: zilsd + # Zic64b mandates 64-byte naturally aligned cache blocks + - if: + properties: + riscv,isa-extensions: + contains: + const: zic64b + required: + - riscv,isa-extensions + then: + properties: + riscv,cbom-block-size: + const: 64 + riscv,cbop-block-size: + const: 64 + riscv,cboz-block-size: + const: 64 + # All three Zicbo* extensions require their block size property as there's no + # default. + - if: + properties: + riscv,isa-extensions: + contains: + const: zicbom + required: + - riscv,isa-extensions + then: + required: + - riscv,cbom-block-size + - if: + properties: + riscv,isa-extensions: + contains: + const: zicbop + required: + - riscv,isa-extensions + then: + required: + - riscv,cbop-block-size + - if: + properties: + riscv,isa-extensions: + contains: + const: zicboz + required: + - riscv,isa-extensions + then: + required: + - riscv,cboz-block-size additionalProperties: true ... diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 6c9d61d83d71..e7891a2a89da 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -215,7 +215,7 @@ config RISCV select PCI_ECAM if (ACPI && PCI) select PCI_MSI if PCI select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED - select RISCV_ALTERNATIVE + select RISCV_ALTERNATIVE_EARLY if 64BIT select RISCV_APLIC select RISCV_IMSIC select RISCV_INTC @@ -537,17 +537,8 @@ config RISCV_COMBO_SPINLOCKS endchoice -config RISCV_ALTERNATIVE - bool - help - This Kconfig allows the kernel to automatically patch the - erratum or cpufeature required by the execution platform at run - time. The code patching overhead is minimal, as it's only done - once at boot and once on each module load. - config RISCV_ALTERNATIVE_EARLY bool - depends on RISCV_ALTERNATIVE help Allows early patching of the kernel for special errata @@ -576,7 +567,6 @@ config RISCV_ISA_SUPM config RISCV_ISA_SVNAPOT bool "Svnapot extension support for supervisor mode NAPOT pages" depends on 64BIT && MMU - depends on RISCV_ALTERNATIVE default y help Enable support for the Svnapot ISA-extension when it is detected @@ -610,10 +600,33 @@ config RISCV_ISA_SSQOSID cache partitioning scheme and use the MCID to track how much cache a process, or a group of processes, is using. +config RISCV_ISA_SV48 + bool "Page-Based 48-bit Virtual-Memory System" + depends on 64BIT + default y + help + Add support for the Page-Based 48-bit Virtual-Memory System. + + Enable this option for systems for which a 39-bit virtual address + space is insufficient. But if 39-bit is enough, disabling this + option will result in smaller kernel size and slighly better + performance. + +config RISCV_ISA_SV57 + bool "Page-Based 57-bit Virtual-Memory System" + depends on 64BIT + default y + help + Add support for the Page-Based 57-bit Virtual-Memory System. + + Enable this option for systems for which a 48-bit virtual address + space is insufficient. But if 48-bit is enough, disabling this + option will result in smaller kernel size and slighly better + performance. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU - depends on RISCV_ALTERNATIVE default y help Add support for the Svpbmt ISA-extension (Supervisor-mode: @@ -683,7 +696,6 @@ config RISCV_ISA_V_PREEMPTIVE config RISCV_ISA_ZAWRS bool "Zawrs extension support for more efficient busy waiting" - depends on RISCV_ALTERNATIVE default y help The Zawrs extension defines instructions to be used in polling loops @@ -702,7 +714,6 @@ config TOOLCHAIN_HAS_ZABHA config RISCV_ISA_ZABHA bool "Zabha extension support for atomic byte/halfword operations" depends on TOOLCHAIN_HAS_ZABHA - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zabha ISA-extension to implement kernel @@ -719,7 +730,6 @@ config TOOLCHAIN_HAS_ZACAS config RISCV_ISA_ZACAS bool "Zacas extension support for atomic CAS" - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zacas ISA-extension to implement kernel atomic @@ -764,7 +774,6 @@ config RISCV_ISA_ZBA config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" - depends on RISCV_ALTERNATIVE default y help Add support for enabling optimisations in the kernel when the @@ -789,7 +798,6 @@ config RISCV_ISA_ZBC bool "Zbc extension support for carry-less multiplication instructions" depends on TOOLCHAIN_HAS_ZBC depends on MMU - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the Zbc @@ -811,7 +819,6 @@ config TOOLCHAIN_HAS_ZBKB config RISCV_ISA_ZBKB bool "Zbkb extension support for bit manipulation instructions" depends on TOOLCHAIN_HAS_ZBKB - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the ZBKB @@ -825,7 +832,6 @@ config RISCV_ISA_ZBKB config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU - depends on RISCV_ALTERNATIVE default y select RISCV_DMA_NONCOHERENT select DMA_DIRECT_REMAP @@ -841,7 +847,6 @@ config RISCV_ISA_ZICBOM config RISCV_ISA_ZICBOZ bool "Zicboz extension support for faster zeroing of memory" - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zicboz extension (cbo.zero instruction) @@ -854,7 +859,6 @@ config RISCV_ISA_ZICBOZ config RISCV_ISA_ZICBOP bool "Zicbop extension support for cache block prefetch" depends on MMU - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the ZICBOP @@ -869,7 +873,6 @@ config RISCV_ISA_ZICBOP config RISCV_ISA_SVRSW60T59B bool "Svrsw60t59b extension support for using PTE bits 60 and 59" depends on MMU && 64BIT - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the Svrsw60t59b @@ -1131,6 +1134,9 @@ config ARCH_DEFAULT_CRASH_DUMP config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION def_bool CRASH_RESERVE +config ARCH_SUPPORTS_CRASH_HOTPLUG + def_bool y + config COMPAT bool "Kernel support for 32-bit U-mode" default 64BIT @@ -1202,7 +1208,6 @@ config RISCV_USER_CFI bool "riscv userspace control flow integrity" depends on 64BIT && MMU && \ $(cc-option,-mabi=lp64 -march=rv64ima_zicfiss_zicfilp -fcf-protection=full) - depends on RISCV_ALTERNATIVE select RISCV_SBI select ARCH_HAS_USER_SHADOW_STACK select ARCH_USES_HIGH_VMA_FLAGS diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index 622ade116a2d..4f77b3c6d7fd 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -2,7 +2,7 @@ menu "CPU errata selection" config ERRATA_ANDES bool "Andes AX45MP errata" - depends on RISCV_ALTERNATIVE && RISCV_SBI + depends on RISCV_SBI help All Andes errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all Andes errata. Please say "Y" @@ -23,7 +23,7 @@ config ERRATA_ANDES_CMO config ERRATA_MIPS bool "MIPS errata" - depends on RISCV_ALTERNATIVE + select RISCV_ALTERNATIVE_EARLY help All MIPS errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all MIPS errata. Please say "Y" @@ -44,9 +44,20 @@ config ERRATA_MIPS_P8700_PAUSE_OPCODE If you are not using the P8700 processor, say n. +config ERRATA_MIPS_P8700_WFI + bool "Replace WFI with mips.pause for MIPS P8700" + depends on ERRATA_MIPS && 64BIT + default n + help + The RISCV MIPS P8700 has bugs with the WFI instruction. + This errata replaces all WFI instructions with the MIPS + P8700 pause opcode to avoid these issues. + + If you are not using the P8700 processor, say n. + config ERRATA_SIFIVE bool "SiFive errata" - depends on RISCV_ALTERNATIVE && 64BIT + depends on 64BIT help All SiFive errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all SiFive errata. Please say "Y" @@ -112,7 +123,6 @@ config ERRATA_SIFIVE_MAL_9092 config ERRATA_THEAD bool "T-HEAD errata" - depends on RISCV_ALTERNATIVE help All T-HEAD errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all T-HEAD errata. Please say "Y" diff --git a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi index f66a382c95bd..5943407020e3 100644 --- a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi @@ -31,7 +31,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -67,7 +67,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -103,7 +103,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -139,7 +139,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -175,7 +175,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -211,7 +211,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -247,7 +247,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -283,7 +283,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -319,7 +319,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -355,7 +355,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -391,7 +391,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -427,7 +427,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -463,7 +463,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -499,7 +499,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -535,7 +535,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -571,7 +571,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -607,7 +607,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -643,7 +643,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -679,7 +679,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -715,7 +715,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -751,7 +751,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -787,7 +787,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -823,7 +823,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -859,7 +859,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -895,7 +895,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -931,7 +931,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -967,7 +967,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1003,7 +1003,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1039,7 +1039,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1075,7 +1075,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1111,7 +1111,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1147,7 +1147,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1183,7 +1183,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1219,7 +1219,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1255,7 +1255,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1291,7 +1291,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1327,7 +1327,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1363,7 +1363,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1399,7 +1399,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1435,7 +1435,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1471,7 +1471,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1507,7 +1507,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1543,7 +1543,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1579,7 +1579,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1615,7 +1615,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1651,7 +1651,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1687,7 +1687,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1723,7 +1723,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1759,7 +1759,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1795,7 +1795,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1831,7 +1831,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1867,7 +1867,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1903,7 +1903,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1939,7 +1939,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -1975,7 +1975,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2011,7 +2011,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2047,7 +2047,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2083,7 +2083,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2119,7 +2119,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2155,7 +2155,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2191,7 +2191,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2227,7 +2227,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2263,7 +2263,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", @@ -2299,7 +2299,7 @@ "svinval", "svnapot", "svpbmt", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", - "zfa", "zfbfmin", "zfh", "zfhmin", + "zfa", "zfbfmin", "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", "zicboz", "ziccrse", "zicntr", "zicond","zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi index 3fabfd34114b..542bfc0d19f5 100644 --- a/arch/riscv/boot/dts/spacemit/k1.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1.dtsi @@ -57,12 +57,12 @@ device_type = "cpu"; reg = <0>; clocks = <&syscon_apmu CLK_CPU_C0_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -88,12 +88,12 @@ device_type = "cpu"; reg = <1>; clocks = <&syscon_apmu CLK_CPU_C0_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -119,12 +119,12 @@ device_type = "cpu"; reg = <2>; clocks = <&syscon_apmu CLK_CPU_C0_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -150,12 +150,12 @@ device_type = "cpu"; reg = <3>; clocks = <&syscon_apmu CLK_CPU_C0_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -181,12 +181,12 @@ device_type = "cpu"; reg = <4>; clocks = <&syscon_apmu CLK_CPU_C1_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -212,12 +212,12 @@ device_type = "cpu"; reg = <5>; clocks = <&syscon_apmu CLK_CPU_C1_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -243,12 +243,12 @@ device_type = "cpu"; reg = <6>; clocks = <&syscon_apmu CLK_CPU_C1_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; @@ -274,12 +274,12 @@ device_type = "cpu"; reg = <7>; clocks = <&syscon_apmu CLK_CPU_C1_CORE>; - riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zic64b_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", - "zicbop", "zicboz", "zicntr", "zicond", "zicsr", - "zifencei", "zihintpause", "zihpm", "zfh", "zba", - "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zic64b", + "zicbom", "zicbop", "zicboz", "zicntr", "zicond", + "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", + "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", "sscofpmf", "sstc", "svinval", "svnapot", "svpbmt"; riscv,cbom-block-size = <64>; riscv,cbop-block-size = <64>; diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi index a5fb72e23dcf..e89d2cc5ca8c 100644 --- a/arch/riscv/boot/dts/spacemit/k3.dtsi +++ b/arch/riscv/boot/dts/spacemit/k3.dtsi @@ -38,9 +38,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -79,9 +79,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -120,9 +120,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -161,9 +161,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -202,9 +202,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -243,9 +243,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -284,9 +284,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", @@ -325,9 +325,9 @@ "svinval", "svnapot", "svpbmt", "za64rs", "zawrs", "zba", "zbb", "zbc", "zbs", "zca", "zcb", "zcd", "zcmop", "zfa", "zfbfmin", - "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", - "ziccamoa", "ziccif", "zicclsm", "ziccrse", "zicntr", - "zicond", "zicsr", "zifencei", "zihintntl", + "zfh", "zfhmin", "zic64b", "zicbom", "zicbop", + "zicboz", "ziccamoa", "ziccif", "zicclsm", "ziccrse", + "zicntr", "zicond", "zicsr", "zifencei", "zihintntl", "zihintpause", "zihpm", "zimop", "zkt", "zvbb", "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", diff --git a/arch/riscv/errata/mips/Makefile b/arch/riscv/errata/mips/Makefile index 6278c389b801..137e700d9d3f 100644 --- a/arch/riscv/errata/mips/Makefile +++ b/arch/riscv/errata/mips/Makefile @@ -1,5 +1,11 @@ ifdef CONFIG_RISCV_ALTERNATIVE_EARLY CFLAGS_errata.o := -mcmodel=medany +ifdef CONFIG_FTRACE +CFLAGS_REMOVE_errata.o = $(CC_FLAGS_FTRACE) +endif +ifdef CONFIG_KASAN +KASAN_SANITIZE_errata.o := n +endif endif obj-y += errata.o diff --git a/arch/riscv/errata/mips/errata.c b/arch/riscv/errata/mips/errata.c index 2c3dc2259e93..6e7155527450 100644 --- a/arch/riscv/errata/mips/errata.c +++ b/arch/riscv/errata/mips/errata.c @@ -7,12 +7,13 @@ #include <linux/module.h> #include <asm/text-patching.h> #include <asm/alternative.h> +#include <asm/cacheflush.h> #include <asm/errata_list.h> #include <asm/vendorid_list.h> #include <asm/vendor_extensions.h> #include <asm/vendor_extensions/mips.h> -static inline bool errata_probe_pause(void) +static inline bool errata_probe_pause(unsigned int stage) { if (!IS_ENABLED(CONFIG_ERRATA_MIPS_P8700_PAUSE_OPCODE)) return false; @@ -20,16 +21,36 @@ static inline bool errata_probe_pause(void) if (!riscv_isa_vendor_extension_available(MIPS_VENDOR_ID, XMIPSEXECTL)) return false; + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return false; + + return true; +} + +static inline bool errata_probe_wfi(unsigned int stage) +{ + if (!IS_ENABLED(CONFIG_ERRATA_MIPS_P8700_WFI)) + return false; + + if (!riscv_isa_vendor_extension_available(MIPS_VENDOR_ID, XMIPSEXECTL)) + return false; + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return false; + return true; } -static u32 mips_errata_probe(void) +static u32 mips_errata_probe(unsigned int stage) { u32 cpu_req_errata = 0; - if (errata_probe_pause()) + if (errata_probe_pause(stage)) cpu_req_errata |= BIT(ERRATA_MIPS_P8700_PAUSE_OPCODE); + if (errata_probe_wfi(stage)) + cpu_req_errata |= BIT(ERRATA_MIPS_P8700_WFI); + return cpu_req_errata; } @@ -38,30 +59,34 @@ void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage) { struct alt_entry *alt; - u32 cpu_req_errata = mips_errata_probe(); + u32 cpu_req_errata = mips_errata_probe(stage); u32 tmp; + void *oldptr, *altptr; BUILD_BUG_ON(ERRATA_MIPS_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE); - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - return; - for (alt = begin; alt < end; alt++) { if (alt->vendor_id != MIPS_VENDOR_ID) continue; - if (alt->patch_id >= ERRATA_MIPS_NUMBER) { - WARN(1, "MIPS errata id:%d not in kernel errata list\n", - alt->patch_id); + if (alt->patch_id >= ERRATA_MIPS_NUMBER) continue; - } tmp = (1U << alt->patch_id); if (cpu_req_errata & tmp) { - mutex_lock(&text_mutex); - patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), - alt->alt_len); - mutex_unlock(&text_mutex); + oldptr = ALT_OLD_PTR(alt); + altptr = ALT_ALT_PTR(alt); + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) { + memcpy(oldptr, altptr, alt->alt_len); + } else { + mutex_lock(&text_mutex); + patch_text_nosync(oldptr, altptr, alt->alt_len); + mutex_unlock(&text_mutex); + } } } + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + local_flush_icache_all(); } diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 9619bd5c8eba..629d884578da 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -2,8 +2,6 @@ #ifndef __ASM_ALTERNATIVE_MACROS_H #define __ASM_ALTERNATIVE_MACROS_H -#ifdef CONFIG_RISCV_ALTERNATIVE - #ifdef __ASSEMBLER__ .macro ALT_ENTRY oldptr newptr vendor_id patch_id new_len @@ -108,28 +106,6 @@ __ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1), \ new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2)) -#else /* CONFIG_RISCV_ALTERNATIVE */ -#ifdef __ASSEMBLER__ - -.macro ALTERNATIVE_CFG old_c - \old_c -.endm - -#define __ALTERNATIVE_CFG(old_c, ...) ALTERNATIVE_CFG old_c -#define __ALTERNATIVE_CFG_2(old_c, ...) ALTERNATIVE_CFG old_c - -#else /* !__ASSEMBLER__ */ - -#define __ALTERNATIVE_CFG(old_c, ...) old_c "\n" -#define __ALTERNATIVE_CFG_2(old_c, ...) old_c "\n" - -#endif /* __ASSEMBLER__ */ - -#define _ALTERNATIVE_CFG(old_c, ...) __ALTERNATIVE_CFG(old_c) -#define _ALTERNATIVE_CFG_2(old_c, ...) __ALTERNATIVE_CFG_2(old_c) - -#endif /* CONFIG_RISCV_ALTERNATIVE */ - /* * Usage: * ALTERNATIVE(old_content, new_content, vendor_id, patch_id, CONFIG_k) diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 8407d1d535b8..6be7b2b6ade9 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -10,8 +10,6 @@ #ifndef __ASSEMBLER__ -#ifdef CONFIG_RISCV_ALTERNATIVE - #include <linux/init.h> #include <linux/kernel.h> #include <linux/types.h> @@ -35,7 +33,7 @@ void __init apply_early_boot_alternatives(void); void apply_module_alternatives(void *start, size_t length); void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, - int patch_offset); + int patch_offset, bool early); struct alt_entry { s32 old_offset; /* offset relative to original instruction or data */ @@ -61,13 +59,5 @@ void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, void riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage); -#else /* CONFIG_RISCV_ALTERNATIVE */ - -static inline void apply_boot_alternatives(void) { } -static inline void apply_early_boot_alternatives(void) { } -static inline void apply_module_alternatives(void *start, size_t length) { } - -#endif /* CONFIG_RISCV_ALTERNATIVE */ - #endif #endif diff --git a/arch/riscv/include/asm/cpufeature-macros.h b/arch/riscv/include/asm/cpufeature-macros.h index a8103edbf51f..adaf9e3fb25c 100644 --- a/arch/riscv/include/asm/cpufeature-macros.h +++ b/arch/riscv/include/asm/cpufeature-macros.h @@ -47,20 +47,14 @@ static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_unlikely(STANDARD_EXT, ext); - - return __riscv_isa_extension_available(NULL, ext); + return __riscv_has_extension_unlikely(STANDARD_EXT, ext); } static __always_inline bool riscv_has_extension_likely(const unsigned long ext) { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_likely(STANDARD_EXT, ext); - - return __riscv_isa_extension_available(NULL, ext); + return __riscv_has_extension_likely(STANDARD_EXT, ext); } #endif /* _ASM_CPUFEATURE_MACROS_H */ diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 5efa72823475..99799cc6c198 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -36,6 +36,8 @@ extern const struct seq_operations cpuinfo_op; /* Per-cpu ISA extensions. */ extern struct riscv_isainfo hart_isa[NR_CPUS]; +extern DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX); + extern u32 thead_vlenb_of; void __init riscv_user_isa_enable(void); @@ -136,8 +138,7 @@ static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsign { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_likely(STANDARD_EXT, ext)) + if (__riscv_has_extension_likely(STANDARD_EXT, ext)) return true; return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); @@ -147,8 +148,7 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_unlikely(STANDARD_EXT, ext)) + if (__riscv_has_extension_unlikely(STANDARD_EXT, ext)) return true; return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 6694b5ccdcf8..6fdb445c0f45 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -25,6 +25,16 @@ ALTERNATIVE(__stringify(RISCV_PTR do_page_fault), \ __stringify(RISCV_PTR sifive_cip_453_page_fault_trp), \ SIFIVE_VENDOR_ID, ERRATA_SIFIVE_CIP_453, \ CONFIG_ERRATA_SIFIVE_CIP_453) + +#ifdef CONFIG_ERRATA_MIPS_P8700_WFI +#define ALT_WFI \ +ALTERNATIVE("wfi; .rept 7; nop; .endr;", \ + ".rept 8; .4byte 0x00501013; .endr;", MIPS_VENDOR_ID, \ + ERRATA_MIPS_P8700_WFI, CONFIG_ERRATA_MIPS_P8700_WFI) +#else +#define ALT_WFI wfi +#endif + #else /* !__ASSEMBLER__ */ #define ALT_SFENCE_VMA_ASID(asid) \ @@ -53,6 +63,33 @@ asm(ALTERNATIVE( \ : /* no inputs */ \ : "memory") +#ifdef CONFIG_ERRATA_MIPS_P8700_WFI +/* + * A WFI instruction after a mispredicted branch can result in erroneous address translation + * on the MIPS P8700. + * The number of MIPS_PAUSE instructions required was determined by measuring + * Coremark performance on a one-core, two-hart configuration. Eight PAUSE instructions matched + * the latency of a single WFI instruction. + * Usually, the maximum number of cycles per PAUSE instruction is 63. Exceptionally, if the hart's + * LR reservation bit is set, the maximum number of cycles is extended to 255 + * (and the hart is woken up once it is cleared). This case should not affect the patched sites. + */ +#define ALT_RISCV_WFI() \ +asm volatile(ALTERNATIVE( \ + "wfi\n" /* Original RISC-V wfi insn */ \ + __nops(7), \ + ".rept 8;" MIPS_PAUSE ".endr;\n", /* Replacement: mips.pause for P8700 */ \ + MIPS_VENDOR_ID, /* Vendor ID to match */ \ + ERRATA_MIPS_P8700_WFI, /* patch_id */ \ + CONFIG_ERRATA_MIPS_P8700_WFI) \ + : /* no outputs */ \ + : /* no inputs */ \ + : "memory") +#else +#define ALT_RISCV_WFI() \ + __asm__ __volatile__ ("wfi") +#endif + /* * _val is marked as "will be overwritten", so need to set it to 0 * in the default case. diff --git a/arch/riscv/include/asm/errata_list_vendors.h b/arch/riscv/include/asm/errata_list_vendors.h index c62a82f60468..02b5ab0d23a8 100644 --- a/arch/riscv/include/asm/errata_list_vendors.h +++ b/arch/riscv/include/asm/errata_list_vendors.h @@ -23,8 +23,9 @@ #endif #ifdef CONFIG_ERRATA_MIPS -#define ERRATA_MIPS_P8700_PAUSE_OPCODE 0 -#define ERRATA_MIPS_NUMBER 1 +#define ERRATA_MIPS_P8700_PAUSE_OPCODE 0 +#define ERRATA_MIPS_P8700_WFI 1 +#define ERRATA_MIPS_NUMBER 2 #endif #endif /* ASM_ERRATA_LIST_VENDORS_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index f8db798b2654..b5952db9259f 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -11,6 +11,7 @@ #include <uapi/asm/hwcap.h> #define RISCV_ISA_EXT_A ('a' - 'a') +#define RISCV_ISA_EXT_B ('b' - 'a') #define RISCV_ISA_EXT_C ('c' - 'a') #define RISCV_ISA_EXT_D ('d' - 'a') #define RISCV_ISA_EXT_F ('f' - 'a') @@ -122,6 +123,9 @@ #define RISCV_ISA_EXT_ZICCAMOA 113 #define RISCV_ISA_EXT_ZICCIF 114 #define RISCV_ISA_EXT_ZA64RS 115 +#define RISCV_ISA_EXT_ZIC64B 116 +#define RISCV_ISA_EXT_SV48 117 +#define RISCV_ISA_EXT_SV57 118 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index c3005573e8c9..ad348d8b8857 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -142,7 +142,7 @@ #define RVG_OPCODE_JAL 0x6f #define RVG_OPCODE_SYSTEM 0x73 #define RVG_SYSTEM_CSR_OFF 20 -#define RVG_SYSTEM_CSR_MASK GENMASK(12, 0) +#define RVG_SYSTEM_CSR_MASK GENMASK(11, 0) /* parts of opcode for RVF, RVD and RVQ */ #define RVFDQ_FL_FS_WIDTH_OFF 12 diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 3ab5f2e3212b..d75961ed3324 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -65,6 +65,32 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke label: return true; } +#else +#include <asm/asm.h> + +#define JUMP_TABLE_ENTRY(key, label) \ + .pushsection __jump_table, "aw"; \ + .align RISCV_LGPTR; \ + .long 1b - ., label - .; \ + RISCV_PTR key - .; \ + .popsection + +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + .align 2; \ + .option push; \ + .option norelax; \ + .option norvc; \ +1: nop; \ + .option pop; \ + JUMP_TABLE_ENTRY(key, label) +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + .align 2; \ + .option push; \ + .option norelax; \ + .option norvc; \ +1: j label; \ + .option pop; \ + JUMP_TABLE_ENTRY(key, label) #endif /* __ASSEMBLER__ */ #endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/include/asm/kexec.h b/arch/riscv/include/asm/kexec.h index b9ee8346cc8c..e218835eb10c 100644 --- a/arch/riscv/include/asm/kexec.h +++ b/arch/riscv/include/asm/kexec.h @@ -75,4 +75,17 @@ int load_extra_segments(struct kimage *image, unsigned long kernel_start, unsigned long cmdline_len); #endif +#ifdef CONFIG_CRASH_HOTPLUG +struct kimage; + +void arch_crash_handle_hotplug_event(struct kimage *image, void *arg); +#define arch_crash_handle_hotplug_event arch_crash_handle_hotplug_event + +int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags); +#define arch_crash_hotplug_support arch_crash_hotplug_support + +unsigned int arch_crash_get_elfcorehdr_size(void); +#define crash_get_elfcorehdr_size arch_crash_get_elfcorehdr_size +#endif + #endif diff --git a/arch/riscv/include/asm/kvm_nacl.h b/arch/riscv/include/asm/kvm_nacl.h index f45407bcaa26..662717e2c45c 100644 --- a/arch/riscv/include/asm/kvm_nacl.h +++ b/arch/riscv/include/asm/kvm_nacl.h @@ -98,7 +98,7 @@ do { \ #define nacl_sync_hfence(__e) \ sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_HFENCE, \ - (__e), 0, 0, 0, 0, 0) + (__e)) #define nacl_hfence_mkconfig(__type, __order, __vmid, __asid) \ ({ \ @@ -198,7 +198,7 @@ do { \ #define nacl_sync_csr(__csr) \ sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_CSR, \ - (__csr), 0, 0, 0, 0, 0) + (__csr)) /* * Each ncsr_xyz() macro defined below has it's own static-branch so every diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h index cc2a184cfc2e..b2f91abebd61 100644 --- a/arch/riscv/include/asm/pci.h +++ b/arch/riscv/include/asm/pci.h @@ -15,6 +15,11 @@ #define PCIBIOS_MIN_IO 4 #define PCIBIOS_MIN_MEM 16 +/* + * Set to 1 if the kernel should re-assign all PCI bus numbers + */ +#define pcibios_assign_all_busses() (pci_has_flag(PCI_REASSIGN_ALL_BUS)) + #if defined(CONFIG_PCI) && defined(CONFIG_NUMA) static inline int pcibus_to_node(struct pci_bus *bus) { diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 770ce18a7328..0133f7ba41d7 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { unsigned long pfn = virt_to_pfn(pud); set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); @@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { unsigned long pfn = virt_to_pfn(pud); set_p4d_safe(p4d, @@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { unsigned long pfn = virt_to_pfn(p4d); set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); @@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { unsigned long pfn = virt_to_pfn(p4d); set_pgd_safe(pgd, @@ -82,21 +82,21 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, #define pud_free pud_free static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) __pud_free(mm, pud); } static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long addr) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); } static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); } #endif /* __PAGETABLE_PMD_FOLDED */ diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h index 00f3369570a8..5003a7e0f0b0 100644 --- a/arch/riscv/include/asm/pgtable-32.h +++ b/arch/riscv/include/asm/pgtable-32.h @@ -33,7 +33,7 @@ _PAGE_WRITE | _PAGE_EXEC | \ _PAGE_USER | _PAGE_GLOBAL)) -static const __maybe_unused int pgtable_l4_enabled; -static const __maybe_unused int pgtable_l5_enabled; +static inline bool pgtable_l4_enabled(void) { return false; } +static inline bool pgtable_l5_enabled(void) { return false; } #endif /* _ASM_RISCV_PGTABLE_32_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index ae23182b572c..eb7708e35cf9 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -10,14 +10,48 @@ #include <linux/const.h> #include <asm/errata_list.h> -extern bool pgtable_l4_enabled; -extern bool pgtable_l5_enabled; +extern bool _pgtable_l4_enabled; +extern bool _pgtable_l5_enabled; + +#ifdef USE_EARLY_PGTABLE_LEVELS +static __always_inline bool pgtable_l5_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57)) + return false; + + return _pgtable_l5_enabled; +} + +static __always_inline bool pgtable_l4_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48)) + return false; + + return _pgtable_l4_enabled; +} +#else +static __always_inline bool pgtable_l5_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57)) + return false; + + return riscv_has_extension_likely(RISCV_ISA_EXT_SV57); +} + +static __always_inline bool pgtable_l4_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48)) + return false; + + return riscv_has_extension_likely(RISCV_ISA_EXT_SV48); +} +#endif #define PGDIR_SHIFT_L3 30 #define PGDIR_SHIFT_L4 39 #define PGDIR_SHIFT_L5 48 -#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ - (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) +#define PGDIR_SHIFT (pgtable_l5_enabled() ? PGDIR_SHIFT_L5 : \ + (pgtable_l4_enabled() ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) /* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE - 1)) @@ -26,8 +60,8 @@ extern bool pgtable_l5_enabled; #define P4D_SHIFT_L3 30 #define P4D_SHIFT_L4 39 #define P4D_SHIFT_L5 39 -#define P4D_SHIFT (pgtable_l5_enabled ? P4D_SHIFT_L5 : \ - (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) +#define P4D_SHIFT (pgtable_l5_enabled() ? P4D_SHIFT_L5 : \ + (pgtable_l4_enabled() ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) #define P4D_MASK (~(P4D_SIZE - 1)) @@ -233,7 +267,7 @@ static inline struct page *pud_page(pud_t pud) #define mm_p4d_folded mm_p4d_folded static inline bool mm_p4d_folded(struct mm_struct *mm) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return false; return true; @@ -242,7 +276,7 @@ static inline bool mm_p4d_folded(struct mm_struct *mm) #define mm_pud_folded mm_pud_folded static inline bool mm_pud_folded(struct mm_struct *mm) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return false; return true; @@ -266,7 +300,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) WRITE_ONCE(*p4dp, p4d); else set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); @@ -274,7 +308,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) static inline int p4d_none(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (p4d_val(p4d) == 0); return 0; @@ -282,7 +316,7 @@ static inline int p4d_none(p4d_t p4d) static inline int p4d_present(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (p4d_val(p4d) & _PAGE_PRESENT); return 1; @@ -290,7 +324,7 @@ static inline int p4d_present(p4d_t p4d) static inline int p4d_bad(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return !p4d_present(p4d); return 0; @@ -298,7 +332,7 @@ static inline int p4d_bad(p4d_t p4d) static inline void p4d_clear(p4d_t *p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) set_p4d(p4d, __p4d(0)); } @@ -314,7 +348,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d) static inline pud_t *p4d_pgtable(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); @@ -333,7 +367,7 @@ pud_t *pud_offset(p4d_t *p4d, unsigned long address); static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) WRITE_ONCE(*pgdp, pgd); else set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); @@ -341,7 +375,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) static inline int pgd_none(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (pgd_val(pgd) == 0); return 0; @@ -349,7 +383,7 @@ static inline int pgd_none(pgd_t pgd) static inline int pgd_present(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (pgd_val(pgd) & _PAGE_PRESENT); return 1; @@ -357,7 +391,7 @@ static inline int pgd_present(pgd_t pgd) static inline int pgd_bad(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return !pgd_present(pgd); return 0; @@ -365,13 +399,13 @@ static inline int pgd_bad(pgd_t pgd) static inline void pgd_clear(pgd_t *pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) set_pgd(pgd, __pgd(0)); } static inline p4d_t *pgd_pgtable(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index d48f90140841..2ebd5fa5adc5 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -75,8 +75,8 @@ #define VA_BITS_SV48 48 #define VA_BITS_SV57 57 -#define VA_BITS (pgtable_l5_enabled ? \ - VA_BITS_SV57 : (pgtable_l4_enabled ? VA_BITS_SV48 : VA_BITS_SV39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + VA_BITS_SV57 : (pgtable_l4_enabled() ? VA_BITS_SV48 : VA_BITS_SV39)) #else #define VA_BITS VA_BITS_SV32 #endif diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 815715c67f94..e9d339bcb7e9 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -17,6 +17,7 @@ #include <asm/alternative-macros.h> #include <asm/hwcap.h> #include <asm/usercfi.h> +#include <asm/errata_list.h> #define arch_get_mmap_end(addr, len, flags) \ ({ \ @@ -187,7 +188,7 @@ extern unsigned long __get_wchan(struct task_struct *p); static inline void wait_for_interrupt(void) { - __asm__ __volatile__ ("wfi"); + ALT_RISCV_WFI(); } extern phys_addr_t dma32_phys_limit; @@ -225,6 +226,10 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg); long get_tagged_addr_ctrl(struct task_struct *task); #define SET_TAGGED_ADDR_CTRL(arg) set_tagged_addr_ctrl(current, arg) #define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current) + +bool riscv_have_user_pmlen(u8 len); +#else +static inline bool riscv_have_user_pmlen(u8 len) { return false; } #endif #endif /* __ASSEMBLER__ */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 5725e0ca4dda..fd6266a7b339 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -7,9 +7,11 @@ #ifndef _ASM_RISCV_SBI_H #define _ASM_RISCV_SBI_H +#include <linux/args.h> #include <linux/types.h> #include <linux/cpumask.h> #include <linux/jump_label.h> +#include <linux/tracepoint-defs.h> #ifdef CONFIG_RISCV_SBI enum sbi_ext_id { @@ -563,14 +565,81 @@ struct sbiret { long value; }; +#ifdef CONFIG_TRACEPOINTS +DECLARE_TRACEPOINT(sbi_call); +DECLARE_TRACEPOINT(sbi_return); +extern void do_trace_sbi_call(int ext, int fid); +extern void do_trace_sbi_return(int ext, long error, long value); +#else +static inline void do_trace_sbi_call(int ext, int fid) {}; +static inline void do_trace_sbi_return(int ext, long error, long value) {}; +#endif + void sbi_init(void); long __sbi_base_ecall(int fid); -struct sbiret __sbi_ecall(unsigned long arg0, unsigned long arg1, - unsigned long arg2, unsigned long arg3, - unsigned long arg4, unsigned long arg5, - int fid, int ext); -#define sbi_ecall(e, f, a0, a1, a2, a3, a4, a5) \ - __sbi_ecall(a0, a1, a2, a3, a4, a5, f, e) + +#define __sbi_ecall_args2(e, f) \ + uintptr_t __ta7 = (uintptr_t)(e); \ + uintptr_t __ta6 = (uintptr_t)(f) +#define __sbi_ecall_args3(e, f, a0) \ + __sbi_ecall_args2(e, f); uintptr_t __ta0 = (uintptr_t)(a0) +#define __sbi_ecall_args4(e, f, a0, a1) \ + __sbi_ecall_args3(e, f, a0); uintptr_t __ta1 = (uintptr_t)(a1) +#define __sbi_ecall_args5(e, f, a0, a1, a2) \ + __sbi_ecall_args4(e, f, a0, a1); uintptr_t __ta2 = (uintptr_t)(a2) +#define __sbi_ecall_args6(e, f, a0, a1, a2, a3) \ + __sbi_ecall_args5(e, f, a0, a1, a2); uintptr_t __ta3 = (uintptr_t)(a3) +#define __sbi_ecall_args7(e, f, a0, a1, a2, a3, a4) \ + __sbi_ecall_args6(e, f, a0, a1, a2, a3); uintptr_t __ta4 = (uintptr_t)(a4) +#define __sbi_ecall_args8(e, f, a0, a1, a2, a3, a4, a5) \ + __sbi_ecall_args7(e, f, a0, a1, a2, a3, a4); uintptr_t __ta5 = (uintptr_t)(a5) + +#define __sbi_ecall_regs2 \ + register uintptr_t __a7 asm ("a7") = __ta7; \ + register uintptr_t __a6 asm ("a6") = __ta6 +#define __sbi_ecall_regs3 __sbi_ecall_regs2; register uintptr_t __a0 asm ("a0") = __ta0 +#define __sbi_ecall_regs4 __sbi_ecall_regs3; register uintptr_t __a1 asm ("a1") = __ta1 +#define __sbi_ecall_regs5 __sbi_ecall_regs4; register uintptr_t __a2 asm ("a2") = __ta2 +#define __sbi_ecall_regs6 __sbi_ecall_regs5; register uintptr_t __a3 asm ("a3") = __ta3 +#define __sbi_ecall_regs7 __sbi_ecall_regs6; register uintptr_t __a4 asm ("a4") = __ta4 +#define __sbi_ecall_regs8 __sbi_ecall_regs7; register uintptr_t __a5 asm ("a5") = __ta5 + +#define __sbi_ecall_constraints1 "r" (__a7) +#define __sbi_ecall_constraints2 __sbi_ecall_constraints1, "r" (__a6) +#define __sbi_ecall_constraints3 __sbi_ecall_constraints2, "r" (__a0) +#define __sbi_ecall_constraints4 __sbi_ecall_constraints3, "r" (__a1) +#define __sbi_ecall_constraints5 __sbi_ecall_constraints4, "r" (__a2) +#define __sbi_ecall_constraints6 __sbi_ecall_constraints5, "r" (__a3) +#define __sbi_ecall_constraints7 __sbi_ecall_constraints6, "r" (__a4) +#define __sbi_ecall_constraints8 __sbi_ecall_constraints7, "r" (__a5) + +#define __sbi_ecall_trace_call() \ + if (tracepoint_enabled(sbi_call)) \ + do_trace_sbi_call(__ta7, __ta6) + +#define __sbi_ecall_trace_return() \ + if (tracepoint_enabled(sbi_return)) \ + do_trace_sbi_return(__ta7, __ret.error, __ret.value) + +/* + * Clear a1 to avoid leaking unrelated kernel state through tracepoints in case + * the register doesn't get overwritten by the ecall nor the arguments. + */ +#define sbi_ecall(A...) \ +({ \ + CONCATENATE(__sbi_ecall_args, COUNT_ARGS(A))(A); \ + __sbi_ecall_trace_call(); \ + register uintptr_t __r0 asm ("a0"); \ + register uintptr_t __r1 asm ("a1") = 0; \ + CONCATENATE(__sbi_ecall_regs, COUNT_ARGS(A)); \ + asm volatile ("ecall" \ + : "=r" (__r0), "=r" (__r1) \ + : CONCATENATE(__sbi_ecall_constraints, COUNT_ARGS(A)) \ + : "memory"); \ + struct sbiret __ret = {.error = __r0, .value = __r1}; \ + __sbi_ecall_trace_return(); \ + __ret; \ +}) #ifdef CONFIG_RISCV_SBI_V01 void sbi_console_putchar(int ch); diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 1f8dcae3af51..cd09f32e1ea3 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -377,12 +377,12 @@ static inline void __switch_to_vector(struct task_struct *prev, struct pt_regs *regs; if (riscv_preempt_v_started(prev)) { - if (riscv_v_is_on()) { - WARN_ON(prev->thread.riscv_v_flags & RISCV_V_CTX_DEPTH_MASK); + if (!(current->thread.riscv_v_flags & RISCV_V_CTX_DEPTH_MASK)) { + /* Voluntary schedule(): nesting_end closed any dirty. */ + WARN_ON(riscv_preempt_v_dirty(prev)); riscv_v_disable(); prev->thread.riscv_v_flags |= RISCV_PREEMPT_V_IN_SCHEDULE; - } - if (riscv_preempt_v_dirty(prev)) { + } else if (riscv_preempt_v_dirty(prev)) { __riscv_v_vstate_save(&prev->thread.kernel_vstate, prev->thread.kernel_vstate.datap); riscv_preempt_v_clear_dirty(prev); diff --git a/arch/riscv/include/asm/vendor_extensions.h b/arch/riscv/include/asm/vendor_extensions.h index 7437304a71b9..b00149be2627 100644 --- a/arch/riscv/include/asm/vendor_extensions.h +++ b/arch/riscv/include/asm/vendor_extensions.h @@ -54,11 +54,9 @@ static __always_inline bool riscv_has_vendor_extension_likely(const unsigned lon if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_likely(vendor, - ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); + return __riscv_has_extension_likely(vendor, + ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); - return __riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, ext); } static __always_inline bool riscv_has_vendor_extension_unlikely(const unsigned long vendor, @@ -67,11 +65,9 @@ static __always_inline bool riscv_has_vendor_extension_unlikely(const unsigned l if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_unlikely(vendor, - ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); + return __riscv_has_extension_unlikely(vendor, + ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); - return __riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, ext); } static __always_inline bool riscv_cpu_has_vendor_extension_likely(const unsigned long vendor, @@ -80,8 +76,7 @@ static __always_inline bool riscv_cpu_has_vendor_extension_likely(const unsigned if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_likely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) + if (__riscv_has_extension_likely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) return true; return __riscv_isa_vendor_extension_available(cpu, vendor, ext); @@ -94,8 +89,7 @@ static __always_inline bool riscv_cpu_has_vendor_extension_unlikely(const unsign if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_unlikely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) + if (__riscv_has_extension_unlikely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) return true; return __riscv_isa_vendor_extension_available(cpu, vendor, ext); diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h index fefe94dc98e2..4d8b80ce570b 100644 --- a/arch/riscv/include/asm/vmalloc.h +++ b/arch/riscv/include/asm/vmalloc.h @@ -3,15 +3,14 @@ #define _ASM_RISCV_VMALLOC_H #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP - -extern bool pgtable_l4_enabled, pgtable_l5_enabled; +#include <asm/pgtable.h> #define IOREMAP_MAX_ORDER (PUD_SHIFT) #define arch_vmap_pud_supported arch_vmap_pud_supported static inline bool arch_vmap_pud_supported(pgprot_t prot) { - return pgtable_l4_enabled || pgtable_l5_enabled; + return pgtable_l4_enabled() || pgtable_l5_enabled(); } #define arch_vmap_pmd_supported arch_vmap_pmd_supported diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h index c52bb7bbbabe..96b7cf854e09 100644 --- a/arch/riscv/include/uapi/asm/hwcap.h +++ b/arch/riscv/include/uapi/asm/hwcap.h @@ -21,6 +21,7 @@ #define COMPAT_HWCAP_ISA_F (1 << ('F' - 'A')) #define COMPAT_HWCAP_ISA_D (1 << ('D' - 'A')) #define COMPAT_HWCAP_ISA_C (1 << ('C' - 'A')) +#define COMPAT_HWCAP_ISA_B (1 << ('B' - 'A')) #define COMPAT_HWCAP_ISA_V (1 << ('V' - 'A')) #endif /* _UAPI_ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index e6a394fbe7fc..9a121ee3b436 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -122,6 +122,8 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZICCIF _BITULL(3) #define RISCV_HWPROBE_EXT_ZICCRSE _BITULL(4) #define RISCV_HWPROBE_EXT_ZA64RS _BITULL(5) +#define RISCV_HWPROBE_EXT_B _BITULL(6) +#define RISCV_HWPROBE_EXT_ZIC64B _BITULL(7) /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ebe1c3588177..f54c336bf547 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -48,7 +48,7 @@ always-$(KBUILD_BUILTIN) += vmlinux.lds obj-y += head.o obj-y += soc.o -obj-$(CONFIG_RISCV_ALTERNATIVE) += alternative.o +obj-y += alternative.o obj-y += cpu.o obj-y += cpufeature.o obj-y += entry.o @@ -111,6 +111,7 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KEXEC_CORE) += kexec_relocate.o crash_save_regs.o machine_kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o kexec_image.o machine_kexec_file.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o +obj-$(CONFIG_CRASH_HOTPLUG) += crash.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index c0c9306022c5..bbb215349452 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -75,7 +75,8 @@ static u32 riscv_instruction_at(void *p) } static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, - u32 jalr_insn, int patch_offset) + u32 jalr_insn, int patch_offset, + bool early) { u32 call[2] = { auipc_insn, jalr_insn }; s32 imm; @@ -88,10 +89,15 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm); /* patch the call place again */ - patch_text_nosync(ptr, call, sizeof(u32) * 2); + if (early) { + memcpy(ptr, call, sizeof(call)); + } else { + patch_text_nosync(ptr, call, sizeof(call)); + } } -static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) +static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset, + bool early) { s32 imm; @@ -103,11 +109,14 @@ static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) riscv_insn_insert_jtype_imm(&jal_insn, imm); /* patch the call place again */ - patch_text_nosync(ptr, &jal_insn, sizeof(u32)); + if (early) + memcpy(ptr, &jal_insn, sizeof(u32)); + else + patch_text_nosync(ptr, &jal_insn, sizeof(u32)); } void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, - int patch_offset) + int patch_offset, bool early) { int num_insn = len / sizeof(u32); int i; @@ -131,7 +140,8 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, continue; riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), - insn, insn2, patch_offset); + insn, insn2, patch_offset, + early); i++; } @@ -144,7 +154,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, continue; riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32), - insn, patch_offset); + insn, patch_offset, early); } } } diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 3dbc8cc557dd..af84b97af3a1 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -292,9 +292,9 @@ static void print_mmu(struct seq_file *f) #if defined(CONFIG_32BIT) sv_type = "sv32"; #elif defined(CONFIG_64BIT) - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) sv_type = "sv57"; - else if (pgtable_l4_enabled) + else if (pgtable_l4_enabled()) sv_type = "sv48"; else sv_type = "sv39"; diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index ee6e4b5cc39e..1a4d132a785a 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -29,7 +29,7 @@ static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, struct sbiret ret; ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_START, - hartid, saddr, priv, 0, 0, 0); + hartid, saddr, priv); return sbi_err_map_linux_errno(ret.error); } @@ -39,7 +39,7 @@ static int sbi_hsm_hart_stop(void) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STOP); return sbi_err_map_linux_errno(ret.error); } @@ -49,7 +49,7 @@ static int sbi_hsm_hart_get_status(unsigned long hartid) struct sbiret ret; ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_STATUS, - hartid, 0, 0, 0, 0, 0); + hartid); if (ret.error) return sbi_err_map_linux_errno(ret.error); else diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 7329792978cc..72bce616491f 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -39,7 +39,7 @@ static bool any_cpu_has_zicbom; unsigned long elf_hwcap __read_mostly; /* Host ISA bitmap */ -static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; +DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; /* Per-cpu ISA extensions. */ struct riscv_isainfo hart_isa[NR_CPUS]; @@ -136,6 +136,24 @@ static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_ext_zic64b_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + /* + * Zic64b mandates 64-byte naturally aligned cache blocks; cross-check the + * cbom/cbop/cboz block-size (when declared) device-tree properties to + * avoid inconsistency. + */ + if ((riscv_cbom_block_size && riscv_cbom_block_size != 64) || + (riscv_cbop_block_size && riscv_cbop_block_size != 64) || + (riscv_cboz_block_size && riscv_cboz_block_size != 64)) { + pr_err("Zic64b detected in ISA string, disabling as a CBO block size is not 64 bytes\n"); + return -EINVAL; + } + + return 0; +} + static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -484,6 +502,12 @@ static const unsigned int riscv_c_exts[] = { RISCV_ISA_EXT_ZCD, }; +static const unsigned int riscv_b_exts[] = { + RISCV_ISA_EXT_ZBA, + RISCV_ISA_EXT_ZBB, + RISCV_ISA_EXT_ZBS, +}; + /* * The canonical order of ISA extension names in the ISA string is defined in * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA @@ -532,9 +556,11 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q), __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts), + __RISCV_ISA_EXT_SUPERSET(b, RISCV_ISA_EXT_B, riscv_b_exts), __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts, riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H), + __RISCV_ISA_EXT_DATA_VALIDATE(zic64b, RISCV_ISA_EXT_ZIC64B, riscv_ext_zic64b_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), @@ -640,6 +666,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), + __RISCV_ISA_EXT_DATA(sv48, RISCV_ISA_EXT_SV48), + __RISCV_ISA_EXT_DATA(sv57, RISCV_ISA_EXT_SV57), }; const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); @@ -938,6 +966,13 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); } +#ifndef CONFIG_32BIT + if (_pgtable_l4_enabled) + set_bit(RISCV_ISA_EXT_SV48, source_isa); + if (_pgtable_l5_enabled) + set_bit(RISCV_ISA_EXT_SV57, source_isa); +#endif + /* * "V" in ISA strings is ambiguous in practice: it should mean * just the standard V-1.0 but vendors aren't well behaved. @@ -1098,6 +1133,13 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) riscv_isa_set_ext(ext, source_isa); } +#ifndef CONFIG_32BIT + if (_pgtable_l4_enabled) + set_bit(RISCV_ISA_EXT_SV48, source_isa); + if (_pgtable_l5_enabled) + set_bit(RISCV_ISA_EXT_SV57, source_isa); +#endif + riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); riscv_fill_cpu_vendor_ext(cpu_node, cpu); @@ -1161,8 +1203,15 @@ void __init riscv_fill_hwcap(void) isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F; isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D; isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; + isa2hwcap[RISCV_ISA_EXT_B] = COMPAT_HWCAP_ISA_B; isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; + /* + * riscv_isa may have been touched by "early" ISA parsing, so zero it + * here to get a fresh slate. + */ + bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); + if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); } else { @@ -1245,7 +1294,6 @@ void riscv_clear_hypervisor_csr(void) csr_clear(CSR_HSTATUS, HSTATUS_HU); } -#ifdef CONFIG_RISCV_ALTERNATIVE /* * Alternative patch sites consider 48 bits when determining when to patch * the old instruction sequence with the new. These bits are broken into a @@ -1279,6 +1327,17 @@ static bool riscv_cpufeature_patch_check(u16 id, u16 value) return false; } +static bool __init_or_module riscv_is_isa_ext_early_id(u16 id) +{ + switch (id) { + case RISCV_ISA_EXT_SV48: + case RISCV_ISA_EXT_SV57: + return true; + } + + return false; +} + void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage) @@ -1286,9 +1345,7 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *alt; void *oldptr, *altptr; u16 id, value, vendor; - - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - return; + bool early = stage == RISCV_ALTERNATIVES_EARLY_BOOT; for (alt = begin; alt < end; alt++) { id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); @@ -1303,6 +1360,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, * vendor extension. */ if (id < RISCV_ISA_EXT_MAX) { + if (early && !riscv_is_isa_ext_early_id(id)) + continue; /* * This patch should be treated as errata so skip * processing here. @@ -1317,6 +1376,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, if (!riscv_cpufeature_patch_check(id, value)) continue; } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) { + if (early) + continue; if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) continue; @@ -1328,10 +1389,20 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, oldptr = ALT_OLD_PTR(alt); altptr = ALT_ALT_PTR(alt); - mutex_lock(&text_mutex); - patch_text_nosync(oldptr, altptr, alt->alt_len); - riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); - mutex_unlock(&text_mutex); + if (early) { + /* oldptr is writable through the MMU-off kernel mapping. */ + memcpy(oldptr, altptr, alt->alt_len); + riscv_alternative_fix_offsets(oldptr, alt->alt_len, + oldptr - altptr, true); + } else { + mutex_lock(&text_mutex); + patch_text_nosync(oldptr, altptr, alt->alt_len); + riscv_alternative_fix_offsets(oldptr, alt->alt_len, + oldptr - altptr, false); + mutex_unlock(&text_mutex); + } } + + if (early) + local_flush_icache_all(); } -#endif diff --git a/arch/riscv/kernel/crash.c b/arch/riscv/kernel/crash.c new file mode 100644 index 000000000000..fe5a4b306aae --- /dev/null +++ b/arch/riscv/kernel/crash.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RISC-V crash hotplug support + * + * Copyright (C) 2025 Bytedance Ltd. + */ + +#include <linux/kexec.h> +#include <linux/crash_core.h> +#include <linux/mm.h> +#include <linux/vmalloc.h> + +#ifdef CONFIG_CRASH_HOTPLUG + +#undef pr_fmt +#define pr_fmt(fmt) "crash hp: " fmt + +int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags) +{ +#ifdef CONFIG_KEXEC_FILE + if (image->file_mode) + return 1; +#endif + /* + * For the kexec_load() syscall path, the user space kexec tool + * needs to indicate that the elfcorehdr segment is excluded from + * SHA verification by setting the appropriate flags. + */ + return (kexec_flags & KEXEC_UPDATE_ELFCOREHDR || + kexec_flags & KEXEC_CRASH_HOTPLUG_SUPPORT); +} + +unsigned int arch_crash_get_elfcorehdr_size(void) +{ + unsigned int sz; + + /* kernel_map, VMCOREINFO and maximum CPUs */ + sz = 2 + CONFIG_NR_CPUS; + if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) + sz += CONFIG_CRASH_MAX_MEMORY_RANGES; + sz *= sizeof(Elf64_Phdr); + return sz; +} + +/** + * arch_crash_handle_hotplug_event() - Handle hotplug elfcorehdr changes + * @image: a pointer to kexec_crash_image + * @arg: struct memory_notify handler for memory hotplug case and + * NULL for CPU hotplug case. + * + * Prepare the new elfcorehdr and replace the existing elfcorehdr. + */ +void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) +{ + void *elfbuf = NULL, *old_elfcorehdr; + unsigned long mem, memsz; + unsigned long elfsz = 0; + + /* + * As crash_prepare_elf64_headers() has already described all + * possible CPUs, there is no need to update the elfcorehdr + * for additional CPU changes. + */ + if ((image->file_mode || image->elfcorehdr_updated) && + ((image->hp_action == KEXEC_CRASH_HP_ADD_CPU) || + (image->hp_action == KEXEC_CRASH_HP_REMOVE_CPU))) + return; + + /* + * Create the new elfcorehdr reflecting the changes to CPU and/or + * memory resources. + */ + if (crash_prepare_headers(true, &elfbuf, &elfsz, NULL)) { + pr_err("unable to create new elfcorehdr"); + goto out; + } + + /* + * Obtain address and size of the elfcorehdr segment, and + * check it against the new elfcorehdr buffer. + */ + mem = image->segment[image->elfcorehdr_index].mem; + memsz = image->segment[image->elfcorehdr_index].memsz; + if (elfsz > memsz) { + pr_err("update elfcorehdr elfsz %lu > memsz %lu", + elfsz, memsz); + goto out; + } + + /* + * Copy new elfcorehdr over the old elfcorehdr at destination. + */ + old_elfcorehdr = kmap_local_page(pfn_to_page(mem >> PAGE_SHIFT)); + if (!old_elfcorehdr) { + pr_err("mapping elfcorehdr segment failed\n"); + goto out; + } + + /* + * Temporarily invalidate the crash image while the + * elfcorehdr is updated. + */ + xchg(&kexec_crash_image, NULL); + memcpy_flushcache(old_elfcorehdr, elfbuf, elfsz); + xchg(&kexec_crash_image, image); + kunmap_local(old_elfcorehdr); + pr_debug("updated elfcorehdr\n"); + +out: + vfree(elfbuf); +} + +#endif /* CONFIG_CRASH_HOTPLUG */ diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index f6a8ca49e627..1631fd6bc786 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -15,6 +15,8 @@ #include <asm/image.h> #include <asm/scs.h> #include <asm/usercfi.h> +#include <asm/alternative.h> +#include <asm/errata_list.h> #include "efi-header.S" __HEAD @@ -189,7 +191,7 @@ secondary_start_sbi: * - receive an early trap, before setup_trap_vector finished * - fail in smp_callin(), as a successful one wouldn't return */ - wfi + ALT_WFI j .Lsecondary_park .align 2 diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c index 77e98b504485..b9481a3e40e3 100644 --- a/arch/riscv/kernel/kernel_mode_vector.c +++ b/arch/riscv/kernel/kernel_mode_vector.c @@ -182,14 +182,6 @@ static int riscv_v_start_kernel_context(void) get_cpu_vector_context(); __riscv_flush_vector_context(); put_cpu_vector_context(); - /* - * A voluntary context switch caused by put_cpu_vector_context() can - * raise the NEED_RESTORE flag if preempt_v starts too early due to a - * failed risv_v_is_on() check. - * - * This causes the next context_nesting_end pollute the v-reg from - * the stale context memory in kernel-mode vector. - */ riscv_v_start(RISCV_PREEMPT_V); return 0; } @@ -227,6 +219,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs) __riscv_v_vstate_clean(regs); riscv_preempt_v_reset_flags(); } + riscv_preempt_v_clear_dirty(current); } } #else diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 9c13a6f1ea2a..9b6cedf914e9 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -51,7 +51,7 @@ static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi, struct sbiret ret; ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM, - lo, hi, flags, 0, 0, 0); + lo, hi, flags); if (ret.error) { if (lo == SBI_SHMEM_DISABLE && hi == SBI_SHMEM_DISABLE) pr_warn("Failed to disable steal-time shmem"); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 7cc5a6a5c020..34c1d498d1ca 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -302,6 +302,18 @@ enum { static bool have_user_pmlen_7; static bool have_user_pmlen_16; +bool riscv_have_user_pmlen(u8 len) +{ + switch (len) { + case PMLEN_7: + return have_user_pmlen_7; + case PMLEN_16: + return have_user_pmlen_16; + default: + return false; + } +} + /* * Control the relaxed ABI allowing tagged user addresses into the kernel. */ diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index c443337056ab..af1ccea222f1 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -57,7 +57,7 @@ static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask *cpu_mas */ void sbi_console_putchar(int ch) { - sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); + sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch); } EXPORT_SYMBOL(sbi_console_putchar); @@ -70,7 +70,7 @@ int sbi_console_getchar(void) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0); return ret.error; } @@ -83,7 +83,7 @@ EXPORT_SYMBOL(sbi_console_getchar); */ void sbi_shutdown(void) { - sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); + sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0); } EXPORT_SYMBOL(sbi_shutdown); @@ -97,9 +97,9 @@ static void __sbi_set_timer_v01(uint64_t stime_value) { #if __riscv_xlen == 32 sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, - stime_value >> 32, 0, 0, 0, 0); + stime_value >> 32); #else - sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); + sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value); #endif } @@ -107,8 +107,7 @@ static void __sbi_send_ipi_v01(unsigned int cpu) { unsigned long hart_mask = __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu)); - sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask), - 0, 0, 0, 0, 0); + sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask)); } static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, @@ -126,17 +125,16 @@ static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, switch (fid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, - (unsigned long)&hart_mask, 0, 0, 0, 0, 0); + (unsigned long)&hart_mask); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, - (unsigned long)&hart_mask, start, size, - 0, 0, 0); + (unsigned long)&hart_mask, start, size); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, (unsigned long)&hart_mask, start, size, - arg4, 0, 0); + arg4); break; default: pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); @@ -180,10 +178,9 @@ static void __sbi_set_timer_v02(uint64_t stime_value) { #if __riscv_xlen == 32 sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, - stime_value >> 32, 0, 0, 0, 0); + stime_value >> 32); #else - sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, - 0, 0, 0, 0); + sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value); #endif } @@ -193,7 +190,7 @@ static void __sbi_send_ipi_v02(unsigned int cpu) struct sbiret ret = {0}; ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, - 1UL, cpuid_to_hartid_map(cpu), 0, 0, 0, 0); + 1UL, cpuid_to_hartid_map(cpu)); if (ret.error) { result = sbi_err_map_linux_errno(ret.error); pr_err("%s: hbase = [%lu] failed (error [%d])\n", @@ -212,32 +209,32 @@ static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask, switch (fid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: - ret = sbi_ecall(ext, fid, hmask, hbase, 0, 0, 0, 0); + ret = sbi_ecall(ext, fid, hmask, hbase); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, 0, 0); + size); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, arg4, 0); + size, arg4); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, 0, 0); + size); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, arg4, 0); + size, arg4); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, 0, 0); + size); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: ret = sbi_ecall(ext, fid, hmask, hbase, start, - size, arg4, 0); + size, arg4); break; default: pr_err("unknown function ID [%lu] for SBI extension [%d]\n", @@ -334,7 +331,7 @@ int sbi_fwft_set(u32 feature, unsigned long value, unsigned long flags) return -EOPNOTSUPP; ret = sbi_ecall(SBI_EXT_FWFT, SBI_EXT_FWFT_SET, - feature, value, flags, 0, 0, 0); + feature, value, flags); return sbi_err_map_linux_errno(ret.error); } @@ -510,8 +507,7 @@ EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); static void sbi_srst_reset(unsigned long type, unsigned long reason) { - sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason, - 0, 0, 0, 0); + sbi_ecall(SBI_EXT_SRST, SBI_EXT_SRST_RESET, type, reason); pr_warn("%s: type=0x%lx reason=0x%lx failed\n", __func__, type, reason); } @@ -544,8 +540,7 @@ long sbi_probe_extension(int extid) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, - 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid); if (!ret.error) return ret.value; @@ -607,10 +602,10 @@ int sbi_debug_console_write(const char *bytes, unsigned int num_bytes) if (IS_ENABLED(CONFIG_32BIT)) ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE, num_bytes, lower_32_bits(base_addr), - upper_32_bits(base_addr), 0, 0, 0); + upper_32_bits(base_addr)); else ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE, - num_bytes, base_addr, 0, 0, 0, 0); + num_bytes, base_addr, 0); if (ret.error == SBI_ERR_FAILURE) return -EIO; @@ -636,10 +631,10 @@ int sbi_debug_console_read(char *bytes, unsigned int num_bytes) if (IS_ENABLED(CONFIG_32BIT)) ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_READ, num_bytes, lower_32_bits(base_addr), - upper_32_bits(base_addr), 0, 0, 0); + upper_32_bits(base_addr)); else ret = sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_READ, - num_bytes, base_addr, 0, 0, 0, 0); + num_bytes, base_addr, 0); if (ret.error == SBI_ERR_FAILURE) return -EIO; diff --git a/arch/riscv/kernel/sbi_ecall.c b/arch/riscv/kernel/sbi_ecall.c index 24aabb4fbde3..b783a46fff1c 100644 --- a/arch/riscv/kernel/sbi_ecall.c +++ b/arch/riscv/kernel/sbi_ecall.c @@ -9,7 +9,7 @@ long __sbi_base_ecall(int fid) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_BASE, fid); if (!ret.error) return ret.value; else @@ -17,32 +17,18 @@ long __sbi_base_ecall(int fid) } EXPORT_SYMBOL(__sbi_base_ecall); -struct sbiret __sbi_ecall(unsigned long arg0, unsigned long arg1, - unsigned long arg2, unsigned long arg3, - unsigned long arg4, unsigned long arg5, - int fid, int ext) +#ifdef CONFIG_TRACEPOINTS +void do_trace_sbi_call(int ext, int fid) { - struct sbiret ret; - trace_sbi_call(ext, fid); +} +EXPORT_SYMBOL(do_trace_sbi_call); +EXPORT_TRACEPOINT_SYMBOL(sbi_call); - register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); - register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); - register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); - register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); - register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); - register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); - register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); - register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); - asm volatile ("ecall" - : "+r" (a0), "+r" (a1) - : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) - : "memory"); - ret.error = a0; - ret.value = a1; - - trace_sbi_return(ext, ret.error, ret.value); - - return ret; +void do_trace_sbi_return(int ext, long error, long value) +{ + trace_sbi_return(ext, error, value); } -EXPORT_SYMBOL(__sbi_ecall); +EXPORT_SYMBOL(do_trace_sbi_return); +EXPORT_TRACEPOINT_SYMBOL(sbi_return); +#endif diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 8930b62b15e7..07027cfeba4b 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -15,7 +15,6 @@ #include <linux/kexec.h> #include <linux/kgdb.h> #include <linux/percpu.h> -#include <linux/profile.h> #include <linux/smp.h> #include <linux/sched.h> #include <linux/seq_file.h> diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index db220966f782..3d74ac644049 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -118,7 +118,7 @@ static int sbi_system_suspend(unsigned long sleep_type, struct sbiret ret; ret = sbi_ecall(SBI_EXT_SUSP, SBI_EXT_SUSP_SYSTEM_SUSPEND, - sleep_type, resume_addr, opaque, 0, 0, 0); + sleep_type, resume_addr, opaque); if (ret.error) return sbi_err_map_linux_errno(ret.error); @@ -156,7 +156,7 @@ static int sbi_suspend_finisher(unsigned long suspend_type, struct sbiret ret; ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND, - suspend_type, resume_addr, opaque, 0, 0, 0); + suspend_type, resume_addr, opaque); return (ret.error) ? sbi_err_map_linux_errno(ret.error) : 0; } diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index d427d2f02c60..cd7cf272e6e1 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -207,6 +207,8 @@ static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, EXT_KEY(isainfo->isa, ZICCIF, pair->value, missing); EXT_KEY(isainfo->isa, ZICCRSE, pair->value, missing); EXT_KEY(isainfo->isa, ZA64RS, pair->value, missing); + EXT_KEY(isainfo->isa, B, pair->value, missing); + EXT_KEY(isainfo->isa, ZIC64B, pair->value, missing); } /* Now turn off reporting features if any CPU is missing it. */ diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c index dec0ba5eff5e..86141e5bab82 100644 --- a/arch/riscv/kernel/usercfi.c +++ b/arch/riscv/kernel/usercfi.c @@ -15,6 +15,7 @@ #include <linux/syscalls.h> #include <linux/prctl.h> #include <asm/csr.h> +#include <asm/switch_to.h> #include <asm/usercfi.h> unsigned long riscv_nousercfi __read_mostly; @@ -66,12 +67,7 @@ void set_shstk_status(struct task_struct *task, bool enable) task->thread_info.user_cfi_state.ubcfi_en = enable ? 1 : 0; - if (enable) - task->thread.envcfg |= ENVCFG_SSE; - else - task->thread.envcfg &= ~ENVCFG_SSE; - - csr_write(CSR_ENVCFG, task->thread.envcfg); + envcfg_update_bits(task, ENVCFG_SSE, enable ? ENVCFG_SSE : 0); } void set_shstk_lock(struct task_struct *task, bool lock) @@ -96,12 +92,7 @@ void set_indir_lp_status(struct task_struct *task, bool enable) task->thread_info.user_cfi_state.ufcfi_en = enable ? 1 : 0; - if (enable) - task->thread.envcfg |= ENVCFG_LPE; - else - task->thread.envcfg &= ~ENVCFG_LPE; - - csr_write(CSR_ENVCFG, task->thread.envcfg); + envcfg_update_bits(task, ENVCFG_LPE, enable ? ENVCFG_LPE : 0); } void set_indir_lp_lock(struct task_struct *task, bool lock) @@ -206,10 +197,13 @@ int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr) */ int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr) { + unsigned long __user *token_ptr = (unsigned long __user *)shstk_ptr; unsigned long token = 0; - token = amo_user_shstk((unsigned long __user *)shstk_ptr, 0); + if (!access_ok(token_ptr, sizeof(token))) + return -EFAULT; + token = amo_user_shstk(token_ptr, 0); if (token == -1) return -EFAULT; diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c index a5cda9a65156..87bcdc0c76ec 100644 --- a/arch/riscv/kvm/nacl.c +++ b/arch/riscv/kvm/nacl.c @@ -73,7 +73,7 @@ int kvm_riscv_nacl_enable(void) nacl = this_cpu_ptr(&kvm_riscv_nacl); ret = sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SET_SHMEM, - nacl->shmem_phys, 0, 0, 0, 0, 0); + nacl->shmem_phys, 0, 0); rc = sbi_err_map_linux_errno(ret.error); if (rc) return rc; @@ -87,7 +87,7 @@ void kvm_riscv_nacl_disable(void) return; sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SET_SHMEM, - SBI_SHMEM_DISABLE, SBI_SHMEM_DISABLE, 0, 0, 0, 0); + SBI_SHMEM_DISABLE, SBI_SHMEM_DISABLE, 0); } void kvm_riscv_nacl_exit(void) @@ -118,8 +118,7 @@ static long nacl_probe_feature(long feature_id) if (!kvm_riscv_nacl_available()) return 0; - ret = sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_PROBE_FEATURE, - feature_id, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_PROBE_FEATURE, feature_id); return ret.value; } diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index cf8586a937de..ad1457e789fb 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -5,6 +5,7 @@ #include <asm/csr.h> #include <asm/hwcap.h> #include <asm/alternative-macros.h> +#include <asm/jump_label.h> .macro fixup op reg addr lbl 100: @@ -77,33 +78,11 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) bltu a2, a3, .Lbyte_copy_tail #if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) - /* - * Copy first bytes until dst is aligned to word boundary. - * a0 - start of dst - * t1 - start of aligned dst - */ - addi t1, a0, SZREG-1 - andi t1, t1, ~(SZREG-1) - /* dst is already aligned, skip */ - beq a0, t1, .Lskip_align_dst -1: - /* a5 - one byte for copying data */ - fixup lb a5, 0(a1), 10f - addi a1, a1, 1 /* src */ - fixup sb a5, 0(a0), 10f - addi a0, a0, 1 /* dst */ - bltu a0, t1, 1b /* t1 - start of aligned dst */ - -.Lskip_align_dst: - /* - * Now dst is aligned. - * Use shift-copy if src is misaligned. - * Use word-copy if both src and dst are aligned because - * can not use shift-copy which do not require shifting - */ - /* a1 - start of src */ - andi a3, a1, SZREG-1 - bnez a3, .Lshift_copy +#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) && defined(CONFIG_JUMP_LABEL) + ARCH_STATIC_BRANCH_JUMP_ASM(fast_unaligned_access_speed_key + 1, .Lalign_dst) +#else + j .Lalign_dst +#endif #endif .Lword_copy: /* @@ -139,6 +118,34 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) j .Lbyte_copy_tail #if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) +.Lalign_dst: + /* + * Copy first bytes until dst is aligned to word boundary. + * a0 - start of dst + * t1 - start of aligned dst + */ + addi t1, a0, SZREG-1 + andi t1, t1, ~(SZREG-1) + /* dst is already aligned, skip */ + beq a0, t1, .Lskip_align_dst +1: + /* a5 - one byte for copying data */ + fixup lb a5, 0(a1), 10f + addi a1, a1, 1 /* src */ + fixup sb a5, 0(a0), 10f + addi a0, a0, 1 /* dst */ + bltu a0, t1, 1b /* t1 - start of aligned dst */ + +.Lskip_align_dst: + /* + * Now dst is aligned. + * Use shift-copy if src is misaligned. + * Use word-copy if both src and dst are aligned because + * can not use shift-copy which do not require shifting + */ + /* a1 - start of src */ + andi a3, a1, SZREG-1 + beqz a3, .Lword_copy .Lshift_copy: /* diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index b50de11605c6..070ee69eabb3 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -6,6 +6,11 @@ * Nick Kossifidis <mick@ics.forth.gr> */ +#ifdef CONFIG_64BIT +/* riscv_has_extension_likely() cannot be used this early */ +#define USE_EARLY_PGTABLE_LEVELS +#endif + #include <linux/init.h> #include <linux/mm.h> #include <linux/memblock.h> @@ -53,10 +58,8 @@ u64 satp_mode __ro_after_init = SATP_MODE_32; EXPORT_SYMBOL(satp_mode); #ifdef CONFIG_64BIT -bool pgtable_l4_enabled __ro_after_init = true; -bool pgtable_l5_enabled __ro_after_init = true; -EXPORT_SYMBOL(pgtable_l4_enabled); -EXPORT_SYMBOL(pgtable_l5_enabled); +bool _pgtable_l4_enabled __ro_after_init = true; +bool _pgtable_l5_enabled __ro_after_init = true; #endif phys_addr_t phys_ram_base __ro_after_init; @@ -653,23 +656,23 @@ static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t } #define pgd_next_t p4d_t -#define alloc_pgd_next(__va) (pgtable_l5_enabled ? \ - pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ +#define alloc_pgd_next(__va) (pgtable_l5_enabled() ? \ + pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled() ? \ pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) -#define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \ - pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ +#define get_pgd_next_virt(__pa) (pgtable_l5_enabled() ? \ + pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled() ? \ pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa))) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) -#define fixmap_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ +#define fixmap_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)fixmap_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) -#define trampoline_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ +#define trampoline_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)trampoline_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) #else #define pgd_next_t pte_t @@ -714,11 +717,11 @@ static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr if (debug_pagealloc_enabled()) return PAGE_SIZE; - if (pgtable_l5_enabled && + if (pgtable_l5_enabled() && !(pa & (P4D_SIZE - 1)) && !(va & (P4D_SIZE - 1)) && size >= P4D_SIZE) return P4D_SIZE; - if (pgtable_l4_enabled && + if (pgtable_l4_enabled() && !(pa & (PUD_SIZE - 1)) && !(va & (PUD_SIZE - 1)) && size >= PUD_SIZE) return PUD_SIZE; @@ -770,14 +773,14 @@ u64 __pi_set_satp_mode_from_fdt(uintptr_t dtb_pa); static void __init disable_pgtable_l5(void) { - pgtable_l5_enabled = false; + _pgtable_l5_enabled = false; kernel_map.page_offset = PAGE_OFFSET_L4; satp_mode = SATP_MODE_48; } static void __init disable_pgtable_l4(void) { - pgtable_l4_enabled = false; + _pgtable_l4_enabled = false; kernel_map.page_offset = PAGE_OFFSET_L3; satp_mode = SATP_MODE_39; } @@ -840,6 +843,14 @@ static __init void set_satp_mode(uintptr_t dtb_pa) kernel_map.page_offset = PAGE_OFFSET_L5; + /* Fall back to SV48 if RISCV_ISA_SV57 isn't enabled */ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57) && satp_mode_limit == 0) + satp_mode_limit = SATP_MODE_48; + + /* Fall back to SV39 if RISCV_ISA_SV48 isn't enabled */ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48) && satp_mode_limit == SATP_MODE_48) + satp_mode_limit = SATP_MODE_39; + if (satp_mode_limit == SATP_MODE_48) { disable_pgtable_l5(); } else if (satp_mode_limit == SATP_MODE_39) { @@ -868,7 +879,7 @@ retry: create_pgd_mapping(early_pg_dir, set_satp_mode_pmd, - pgtable_l5_enabled ? + pgtable_l5_enabled() ? (uintptr_t)early_p4d : (uintptr_t)early_pud, PGDIR_SIZE, PAGE_TABLE); @@ -880,7 +891,7 @@ retry: local_flush_tlb_all(); if (hw_satp != identity_satp) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { disable_pgtable_l5(); memset(early_pg_dir, 0, PAGE_SIZE); goto retry; @@ -893,6 +904,10 @@ out: memset(early_p4d, 0, PAGE_SIZE); memset(early_pud, 0, PAGE_SIZE); memset(early_pmd, 0, PAGE_SIZE); + if (pgtable_l4_enabled()) + set_bit(RISCV_ISA_EXT_SV48, riscv_isa); + if (pgtable_l5_enabled()) + set_bit(RISCV_ISA_EXT_SV57, riscv_isa); } #endif @@ -1121,11 +1136,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) #ifndef __PAGETABLE_PMD_FOLDED /* Setup fixmap P4D and PUD */ - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) create_p4d_mapping(fixmap_p4d, FIXADDR_START, (uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE); /* Setup fixmap PUD and PMD */ - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) create_pud_mapping(fixmap_pud, FIXADDR_START, (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); create_pmd_mapping(fixmap_pmd, FIXADDR_START, @@ -1133,10 +1148,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) /* Setup trampoline PGD and PMD */ create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr, (uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE); - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) create_pud_mapping(trampoline_pud, kernel_map.virt_addr, (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, @@ -1407,7 +1422,7 @@ static void __init preallocate_pgd_pages_range(unsigned long start, unsigned lon if (!p4d) goto failed; - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) continue; lvl = "pud"; @@ -1415,7 +1430,7 @@ static void __init preallocate_pgd_pages_range(unsigned long start, unsigned lon if (!pud) goto failed; - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) continue; lvl = "pmd"; @@ -1627,13 +1642,13 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un for (; addr < end; addr = next) { next = pud_addr_end(addr, end); - pudp = pgtable_l4_enabled ? pud_base + pud_index(addr) : pud_base; + pudp = pgtable_l4_enabled() ? pud_base + pud_index(addr) : pud_base; pud = pudp_get(pudp); if (!pud_present(pud)) continue; if (pud_leaf(pud)) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { pud_clear(pudp); if (is_vmemmap) free_vmemmap_storage(pud_page(pud), PUD_SIZE, altmap); @@ -1644,7 +1659,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un pmd_base = pmd_offset(pudp, 0); remove_pmd_mapping(pmd_base, addr, next, is_vmemmap, altmap); - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) free_pmd_table(pmd_base, pudp, is_vmemmap); } } @@ -1658,13 +1673,13 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un for (; addr < end; addr = next) { next = p4d_addr_end(addr, end); - p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; + p4dp = pgtable_l5_enabled() ? p4d_base + p4d_index(addr) : p4d_base; p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) continue; if (p4d_leaf(p4d)) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { p4d_clear(p4dp); if (is_vmemmap) free_vmemmap_storage(p4d_page(p4d), P4D_SIZE, altmap); @@ -1675,7 +1690,7 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un pud_base = pud_offset(p4dp, 0); remove_pud_mapping(pud_base, addr, next, is_vmemmap, altmap); - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) free_pud_table(pud_base, p4dp); } } diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index 1f3aa9611187..84f87c8277c3 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -168,7 +168,7 @@ static void __init kasan_early_clear_pud(p4d_t *p4dp, pud_t *pudp, *base_pud; unsigned long next; - if (!pgtable_l4_enabled) { + if (!pgtable_l4_enabled()) { pudp = (pud_t *)p4dp; } else { base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); @@ -193,7 +193,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp, p4d_t *p4dp, *base_p4d; unsigned long next; - if (!pgtable_l5_enabled) { + if (!pgtable_l5_enabled()) { p4dp = (p4d_t *)pgdp; } else { base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); @@ -203,7 +203,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp, do { next = p4d_addr_end(vaddr, end); - if (pgtable_l4_enabled && IS_ALIGNED(vaddr, P4D_SIZE) && + if (pgtable_l4_enabled() && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) { p4d_clear(p4dp); continue; @@ -221,7 +221,7 @@ static void __init kasan_early_clear_pgd(pgd_t *pgdp, do { next = pgd_addr_end(vaddr, end); - if (pgtable_l5_enabled && IS_ALIGNED(vaddr, PGDIR_SIZE) && + if (pgtable_l5_enabled() && IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { pgd_clear(pgdp); continue; @@ -239,7 +239,7 @@ static void __init kasan_early_populate_pud(p4d_t *p4dp, phys_addr_t phys_addr; unsigned long next; - if (!pgtable_l4_enabled) { + if (!pgtable_l4_enabled()) { pudp = (pud_t *)p4dp; } else { base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); @@ -277,7 +277,7 @@ static void __init kasan_early_populate_p4d(pgd_t *pgdp, * Note that this test is then completely equivalent to * p4dp = p4d_offset(pgdp, vaddr) */ - if (!pgtable_l5_enabled) { + if (!pgtable_l5_enabled()) { p4dp = (p4d_t *)pgdp; } else { base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); @@ -336,7 +336,7 @@ asmlinkage void __init kasan_early_init(void) (__pa((uintptr_t)kasan_early_shadow_pte)), PAGE_TABLE)); - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { for (i = 0; i < PTRS_PER_PUD; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN @@ -344,7 +344,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { for (i = 0; i < PTRS_PER_P4D; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN @@ -461,7 +461,7 @@ static void __init create_tmp_mapping(void) memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(pgd_t) * PTRS_PER_PGD); /* Copy the last p4d since it is shared with the kernel mapping. */ - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { ptr = (p4d_t *)pgd_page_vaddr(pgdp_get(pgd_offset_k(KASAN_SHADOW_END))); memcpy(tmp_p4d, ptr, sizeof(p4d_t) * PTRS_PER_P4D); set_pgd(&tmp_pg_dir[pgd_index(KASAN_SHADOW_END)], @@ -472,7 +472,7 @@ static void __init create_tmp_mapping(void) } /* Copy the last pud since it is shared with the kernel mapping. */ - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { ptr = (pud_t *)p4d_page_vaddr(p4dp_get(base_p4d + p4d_index(KASAN_SHADOW_END))); memcpy(tmp_pud, ptr, sizeof(pud_t) * PTRS_PER_PUD); set_p4d(&base_p4d[p4d_index(KASAN_SHADOW_END)], diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 9c4427d0b187..cd180f9abb5e 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); #ifdef CONFIG_64BIT pud_t *pud_offset(p4d_t *p4d, unsigned long address) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); return (pud_t *)p4d; @@ -50,7 +50,7 @@ EXPORT_SYMBOL_GPL(pud_offset); p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return pgd_pgtable(pgdp_get(pgd)) + p4d_index(address); return (p4d_t *)pgd; diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index f4b4a9fcbbd8..f765a63e2ca7 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -446,8 +446,8 @@ static int __init ptdump_init(void) kernel_ptd_info.base_addr = KERN_VIRT_START; - pg_level[1].name = pgtable_l5_enabled ? "P4D" : "PGD"; - pg_level[2].name = pgtable_l4_enabled ? "PUD" : "PGD"; + pg_level[1].name = pgtable_l5_enabled() ? "P4D" : "PGD"; + pg_level[2].name = pgtable_l4_enabled() ? "PUD" : "PGD"; for (i = 0; i < ARRAY_SIZE(pg_level); i++) for (j = 0; j < ARRAY_SIZE(pte_bits); j++) diff --git a/drivers/acpi/riscv/cppc.c b/drivers/acpi/riscv/cppc.c index 4ea4ddedd91f..74a5e484f809 100644 --- a/drivers/acpi/riscv/cppc.c +++ b/drivers/acpi/riscv/cppc.c @@ -51,7 +51,7 @@ static void sbi_cppc_read(void *read_data) struct sbi_cppc_data *data = (struct sbi_cppc_data *)read_data; data->ret = sbi_ecall(SBI_EXT_CPPC, SBI_CPPC_READ, - data->reg, 0, 0, 0, 0, 0); + data->reg); } static void sbi_cppc_write(void *write_data) @@ -59,7 +59,7 @@ static void sbi_cppc_write(void *write_data) struct sbi_cppc_data *data = (struct sbi_cppc_data *)write_data; data->ret = sbi_ecall(SBI_EXT_CPPC, SBI_CPPC_WRITE, - data->reg, data->val, 0, 0, 0, 0); + data->reg, data->val); } static void cppc_ffh_csr_read(void *read_data) diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 2991dd92def2..21a2078eab98 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -386,7 +386,7 @@ static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata) edata->event_idx, 0); if (!ret.error) { sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET); return; } } @@ -722,7 +722,7 @@ static int pmu_sbi_snapshot_disable(void) struct sbiret ret; ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, SBI_SHMEM_DISABLE, - SBI_SHMEM_DISABLE, 0, 0, 0, 0); + SBI_SHMEM_DISABLE, 0); if (ret.error) { pr_warn("failed to disable snapshot shared memory\n"); return sbi_err_map_linux_errno(ret.error); @@ -746,10 +746,11 @@ static int pmu_sbi_snapshot_setup(struct riscv_pmu *pmu, int cpu) if (IS_ENABLED(CONFIG_32BIT)) ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, cpu_hw_evt->snapshot_addr_phys, - (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, 0, 0, 0, 0); + (u64)(cpu_hw_evt->snapshot_addr_phys) >> 32, + 0); else ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, - cpu_hw_evt->snapshot_addr_phys, 0, 0, 0, 0, 0); + cpu_hw_evt->snapshot_addr_phys, 0, 0); /* Free up the snapshot area memory and fall back to SBI PMU calls without snapshot */ if (ret.error) { @@ -783,14 +784,14 @@ static u64 pmu_sbi_ctr_read(struct perf_event *event) if (pmu_sbi_is_fw_event(event)) { ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ, - hwc->idx, 0, 0, 0, 0, 0); + hwc->idx); if (ret.error) return 0; val = ret.value; if (IS_ENABLED(CONFIG_32BIT) && sbi_v2_available && info.width >= 32) { ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ_HI, - hwc->idx, 0, 0, 0, 0, 0); + hwc->idx); if (!ret.error) val |= ((u64)ret.value << 32); else @@ -833,10 +834,10 @@ static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival) /* There is no benefit setting SNAPSHOT FLAG for a single counter */ #if defined(CONFIG_32BIT) ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx, - 1, flag, ival, ival >> 32, 0); + 1, flag, ival, ival >> 32); #else ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx, - 1, flag, ival, 0, 0); + 1, flag, ival); #endif if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED)) pr_err("Starting counter idx %d failed with error %d\n", @@ -862,7 +863,8 @@ static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag) if (sbi_pmu_snapshot_available()) flag |= SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, + hwc->idx, 1, flag); if (!ret.error && sbi_pmu_snapshot_available()) { /* * The counter snapshot is based on the index base specified by hwc->idx. @@ -887,7 +889,7 @@ static int pmu_sbi_find_num_ctrs(void) { struct sbiret ret; - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS); if (!ret.error) return ret.value; else @@ -905,7 +907,7 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) return -ENOMEM; for (i = 0; i < nctr; i++) { - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0); + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i); if (ret.error) /* The logical counter ids are not expected to be contiguous */ continue; @@ -938,7 +940,7 @@ static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) continue; sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG, pmu->cmask[i], - SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + SBI_PMU_STOP_FLAG_RESET); } } @@ -960,7 +962,7 @@ static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { /* No need to check the error here as we can't do anything about the error */ ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, i * BITS_PER_LONG, - cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); + cpu_hw_evt->used_hw_ctrs[i], flag); if (!ret.error && sbi_pmu_snapshot_available()) { /* Save the counter values to avoid clobbering */ for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) @@ -1002,7 +1004,7 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, /* Start all the counters that did not overflow in a single shot */ if (ctr_start_mask) { sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG, - ctr_start_mask, 0, 0, 0, 0); + ctr_start_mask, 0); } } @@ -1014,11 +1016,11 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, max_period = riscv_pmu_ctr_get_width_mask(event); init_val = local64_read(&hwc->prev_count) & max_period; #if defined(CONFIG_32BIT) - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, init_val >> 32, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, + 1, flag, init_val, init_val >> 32); #else - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, 0, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, + 1, flag, init_val); #endif perf_event_update_userpage(event); } @@ -1057,8 +1059,9 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG]; /* Start all the counters in a single shot */ - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG, - cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, + idx * BITS_PER_LONG, + cpu_hw_evt->used_hw_ctrs[i], flag); } } diff --git a/tools/testing/selftests/riscv/vector/Makefile b/tools/testing/selftests/riscv/vector/Makefile index 7e0017b3fb8b..0ed09b69dd8a 100644 --- a/tools/testing/selftests/riscv/vector/Makefile +++ b/tools/testing/selftests/riscv/vector/Makefile @@ -2,7 +2,8 @@ # Copyright (C) 2021 ARM Limited # Originally tools/testing/arm64/abi/Makefile -TEST_GEN_PROGS := v_initval vstate_prctl vstate_ptrace validate_v_ptrace +TEST_GEN_PROGS := v_initval vstate_prctl vstate_ptrace validate_v_ptrace \ + v_uaccess_stress TEST_GEN_PROGS_EXTENDED := vstate_exec_nolibc v_exec_initval_nolibc TEST_GEN_LIBS := v_helpers.c sys_hwprobe.c @@ -36,4 +37,7 @@ $(OUTPUT)/vstate_ptrace: vstate_ptrace.c $(OUTPUT)/sys_hwprobe.o $(OUTPUT)/v_hel $(OUTPUT)/validate_v_ptrace: validate_v_ptrace.c $(OUTPUT)/sys_hwprobe.o $(OUTPUT)/v_helpers.o $(CC) -static -o $@ $(CFLAGS) $(LDFLAGS) $^ +$(OUTPUT)/v_uaccess_stress: v_uaccess_stress.c + $(CC) -static -o $@ $(CFLAGS) $(LDFLAGS) $^ + EXTRA_CLEAN += $(TEST_GEN_OBJ) diff --git a/tools/testing/selftests/riscv/vector/v_uaccess_stress.c b/tools/testing/selftests/riscv/vector/v_uaccess_stress.c new file mode 100644 index 000000000000..d70e0387e246 --- /dev/null +++ b/tools/testing/selftests/riscv/vector/v_uaccess_stress.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Stress the kernel's user copy path from several processes at once. + * + * Each child pushes a 1KiB pseudo-random byte stream through a pipe: write() + * copies it from user space into the kernel (copy_from_user()) and read() + * copies it back out (copy_to_user()). The round trip is repeated 512 times + * per child, and every iteration the buffers are shifted so that the copies + * cover all of the source/destination alignment combinations. + * + * On RISC-V those copies may be serviced by kernel-mode vector routines, so + * the payload is generated and verified with plain scalar loads and stores + * only. The comparison goes through volatile pointers, which keeps both GCC + * and clang from turning the loop into a vectorised compare and keeps the + * check independent of the user vector state the kernel is supposed to + * preserve. memcmp() is avoided for the same reason: the libc version is free + * to use vector instructions. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/wait.h> + +#include "kselftest_harness.h" + +#define ITERATIONS 32768 +#define STREAM_SIZE 1024 +#define MAX_SHIFT 16 +#define BUF_SIZE (STREAM_SIZE + MAX_SHIFT) + +#define CONCURRENCY 16 + +/* xorshift64*, so the payload does not depend on any libc or kernel helper. */ +static unsigned long prng_state; + +static unsigned long prng_next(void) +{ + prng_state ^= prng_state >> 12; + prng_state ^= prng_state << 25; + prng_state ^= prng_state >> 27; + + return prng_state * 2685821657736338717UL; +} + +static void fill_random(unsigned char *buf, size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) + buf[i] = (unsigned char)(prng_next() >> 24); +} + +/* + * Scalar, byte at a time comparison. Returns the offset of the first + * mismatching byte, or len when the two streams are identical. + */ +static size_t scalar_diff(const unsigned char *a, const unsigned char *b, + size_t len) +{ + const volatile unsigned char *va = a; + const volatile unsigned char *vb = b; + size_t i; + + for (i = 0; i < len; i++) { + if (va[i] != vb[i]) + break; + } + + return i; +} + +static ssize_t write_all(int fd, const unsigned char *buf, size_t len) +{ + size_t done = 0; + + while (done < len) { + ssize_t ret = write(fd, buf + done, len - done); + + if (ret < 0) { + if (errno == EINTR) + continue; + return -1; + } + done += ret; + } + + return done; +} + +static ssize_t read_all(int fd, unsigned char *buf, size_t len) +{ + size_t done = 0; + + while (done < len) { + ssize_t ret = read(fd, buf + done, len - done); + + if (ret < 0) { + if (errno == EINTR) + continue; + return -1; + } + if (ret == 0) + return -1; + done += ret; + } + + return done; +} + +/* Runs in the child. Returns the exit status to hand back to the parent. */ +static int child_loop(unsigned long seed) +{ + unsigned char src[BUF_SIZE], dst[BUF_SIZE]; + int pipefd[2]; + int i; + + prng_state = seed; + + if (pipe(pipefd)) { + fprintf(stderr, "pid %d: pipe() failed: %s\n", + getpid(), strerror(errno)); + return 1; + } + + for (i = 0; i < ITERATIONS; i++) { + /* + * Walk the source and destination through every relative + * alignment, including the case where both are equally + * misaligned. + */ + unsigned char *in = src + (i % MAX_SHIFT); + unsigned char *out = dst + ((i / MAX_SHIFT) % MAX_SHIFT); + size_t off; + + fill_random(in, STREAM_SIZE); + memset(out, 0, STREAM_SIZE); + + if (write_all(pipefd[1], in, STREAM_SIZE) < 0) { + fprintf(stderr, "pid %d: iteration %d: write() failed: %s\n", + getpid(), i, strerror(errno)); + return 1; + } + + if (read_all(pipefd[0], out, STREAM_SIZE) < 0) { + fprintf(stderr, "pid %d: iteration %d: read() failed: %s\n", + getpid(), i, strerror(errno)); + return 1; + } + + off = scalar_diff(in, out, STREAM_SIZE); + if (off != STREAM_SIZE) { + fprintf(stderr, + "pid %d: iteration %d: mismatch at byte %zu of %d (in %p, out %p): expected 0x%02x, got 0x%02x\n", + getpid(), i, off, STREAM_SIZE, in, out, + in[off], out[off]); + return 1; + } + } + + close(pipefd[0]); + close(pipefd[1]); + + return 0; +} + +static int nr_children(void) +{ + long online = sysconf(_SC_NPROCESSORS_ONLN); + + return online * CONCURRENCY; +} + +TEST(uaccess_round_trip) +{ + int nr = nr_children(); + int failures = 0; + pid_t pids[nr]; + int i; + + ksft_print_msg("%d children, %d iterations of %d bytes each\n", + nr, ITERATIONS, STREAM_SIZE); + + for (i = 0; i < nr; i++) { + pids[i] = fork(); + ASSERT_LE(0, pids[i]) { + TH_LOG("fork() failed: %s", strerror(errno)); + } + + if (pids[i] == 0) { + /* A distinct, reproducible stream per child. */ + _exit(child_loop(0x9e3779b97f4a7c15UL + i)); + } + } + + for (i = 0; i < nr; i++) { + int status; + + if (waitpid(pids[i], &status, 0) < 0) { + TH_LOG("waitpid(%d) failed: %s", pids[i], + strerror(errno)); + failures++; + continue; + } + + if (WIFSIGNALED(status)) { + TH_LOG("child %d killed by signal %d", pids[i], + WTERMSIG(status)); + failures++; + } else if (!WIFEXITED(status) || WEXITSTATUS(status)) { + TH_LOG("child %d failed", pids[i]); + failures++; + } + } + + ASSERT_EQ(0, failures); +} + +TEST_HARNESS_MAIN |