diff options
Diffstat (limited to 'arch/riscv')
378 files changed, 25118 insertions, 8351 deletions
diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild index 2c585f7a0b6e..126fb738fc44 100644 --- a/arch/riscv/Kbuild +++ b/arch/riscv/Kbuild @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y += kernel/ mm/ net/ -obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ obj-$(CONFIG_CRYPTO) += crypto/ obj-y += errata/ obj-$(CONFIG_KVM) += kvm/ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 7612c52e9b1e..c5754942cf85 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -16,19 +16,19 @@ config RISCV select ACPI_MCFG if (ACPI && PCI) select ACPI_PPTT if ACPI select ACPI_REDUCED_HARDWARE_ONLY if ACPI + select ACPI_RIMT if ACPI select ACPI_SPCR_TABLE if ACPI select ARCH_DMA_DEFAULT_COHERENT select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP - select ARCH_ENABLE_MEMORY_HOTREMOVE if MEMORY_HOTPLUG select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_BINFMT_FLAT - select ARCH_HAS_CRC32 if RISCV_ISA_ZBC select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_WX + select ARCH_HAS_ELF_CORE_EFLAGS if BINFMT_ELF && ELF_CORE select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL @@ -43,44 +43,48 @@ config RISCV select ARCH_HAS_PMEM_API select ARCH_HAS_PREEMPT_LAZY select ARCH_HAS_PREPARE_SYNC_CORE_CMD - select ARCH_HAS_PTE_DEVMAP if 64BIT && MMU + select ARCH_HAS_PTDUMP if MMU select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_DIRECT_MAP if MMU select ARCH_HAS_SET_MEMORY if MMU - select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL - select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL + select ARCH_HAS_STRICT_KERNEL_RWX if MMU + select ARCH_HAS_STRICT_MODULE_RWX if MMU select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UBSAN - select ARCH_HAS_VDSO_TIME_DATA + select ARCH_HAS_VDSO_ARCH_DATA if HAVE_GENERIC_VDSO + select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_KEEP_MEMBLOCK if ACPI select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE if 64BIT && MMU select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW - select ARCH_SUPPORTS_CFI_CLANG + # clang >= 17: https://github.com/llvm/llvm-project/commit/62fa708ceb027713b386c7e0efda994f8bdc27e2 + select ARCH_SUPPORTS_CFI if (!CC_IS_CLANG || CLANG_VERSION >= 170000) select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU + select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_HUGETLBFS if MMU - # LLD >= 14: https://github.com/llvm/llvm-project/issues/50505 - select ARCH_SUPPORTS_LTO_CLANG if LLD_VERSION >= 140000 - select ARCH_SUPPORTS_LTO_CLANG_THIN if LLD_VERSION >= 140000 + select ARCH_SUPPORTS_LTO_CLANG if CMODEL_MEDANY + select ARCH_SUPPORTS_LTO_CLANG_THIN + select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU select ARCH_SUPPORTS_PER_VMA_LOCK if MMU select ARCH_SUPPORTS_RT select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK + select ARCH_SUPPORTS_SCHED_MC if SMP select ARCH_USE_CMPXCHG_LOCKREF if 64BIT select ARCH_USE_MEMTEST select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_SYM_ANNOTATIONS - select ARCH_USES_CFI_TRAPS if CFI_CLANG + select ARCH_USES_CFI_TRAPS if CFI select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH if MMU select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT select ARCH_WANT_HUGE_PMD_SHARE if 64BIT - select ARCH_WANT_LD_ORPHAN_WARN if !XIP_KERNEL + select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANTS_NO_INSTR @@ -91,10 +95,13 @@ config RISCV select CLINT_TIMER if RISCV_M_MODE select CLONE_BACKWARDS select COMMON_CLK + select CPU_NO_EFFICIENT_FFS if !RISCV_ISA_ZBB select CPU_PM if CPU_IDLE || HIBERNATION || SUSPEND + select DYNAMIC_FTRACE if FUNCTION_TRACER select EDAC_SUPPORT select FRAME_POINTER if PERF_EVENTS || (FUNCTION_TRACER && !DYNAMIC_FTRACE) select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY if DYNAMIC_FTRACE + select FUNCTION_ALIGNMENT_8B if DYNAMIC_FTRACE_WITH_CALL_OPS select GENERIC_ARCH_TOPOLOGY select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS_BROADCAST if SMP @@ -102,73 +109,78 @@ config RISCV select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_ENTRY - select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO + select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO && 64BIT select GENERIC_IDLE_POLL_SETUP select GENERIC_IOREMAP if MMU + select HAVE_IOREMAP_PROT if MMU select GENERIC_IRQ_IPI if SMP select GENERIC_IRQ_IPI_MUX if SMP select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL select GENERIC_LIB_DEVMEM_IS_ALLOWED + select GENERIC_PENDING_IRQ if SMP select GENERIC_PCI_IOMAP - select GENERIC_PTDUMP if MMU select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD - select GENERIC_TIME_VSYSCALL if MMU && 64BIT - select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO + select GENERIC_TIME_VSYSCALL if GENERIC_GETTIMEOFDAY select HARDIRQS_SW_RESEND select HAS_IOPORT if MMU select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT - select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL - select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT select HAVE_ARCH_KASAN_VMALLOC if MMU && 64BIT select HAVE_ARCH_KFENCE if MMU && 64BIT - select HAVE_ARCH_KGDB if !XIP_KERNEL + select HAVE_ARCH_KSTACK_ERASE + select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET select HAVE_ARCH_SECCOMP_FILTER - select HAVE_ARCH_STACKLEAK + select HAVE_ARCH_SOFT_DIRTY if 64BIT && MMU && RISCV_ISA_SVRSW60T59B select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT && MMU + select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if 64BIT && MMU select HAVE_ARCH_USERFAULTFD_MINOR if 64BIT && USERFAULTFD + select HAVE_ARCH_USERFAULTFD_WP if 64BIT && MMU && USERFAULTFD && RISCV_ISA_SVRSW60T59B select HAVE_ARCH_VMAP_STACK if MMU && 64BIT select HAVE_ASM_MODVERSIONS select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU - select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) - select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE) + select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C + select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS + select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI) select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_GRAPH_FUNC - select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL - select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_GRAPH_TRACER if HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_FUNCTION_GRAPH_FREGS - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION + select HAVE_FUNCTION_TRACER if HAVE_DYNAMIC_FTRACE select HAVE_EBPF_JIT if MMU + select HAVE_GENERIC_TIF_BITS select HAVE_GUP_FAST if MMU select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_ERROR_INJECTION select HAVE_GCC_PLUGINS - select HAVE_GENERIC_VDSO if MMU && 64BIT + select HAVE_GENERIC_VDSO if MMU select HAVE_IRQ_TIME_ACCOUNTING - select HAVE_KERNEL_BZIP2 if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_GZIP if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZ4 if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZMA if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_LZO if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_UNCOMPRESSED if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_ZSTD if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KERNEL_XZ if !XIP_KERNEL && !EFI_ZBOOT - select HAVE_KPROBES if !XIP_KERNEL - select HAVE_KRETPROBES if !XIP_KERNEL + select HAVE_KERNEL_BZIP2 if !EFI_ZBOOT + select HAVE_KERNEL_GZIP if !EFI_ZBOOT + select HAVE_KERNEL_LZ4 if !EFI_ZBOOT + select HAVE_KERNEL_LZMA if !EFI_ZBOOT + select HAVE_KERNEL_LZO if !EFI_ZBOOT + select HAVE_KERNEL_UNCOMPRESSED if !EFI_ZBOOT + select HAVE_KERNEL_ZSTD if !EFI_ZBOOT + select HAVE_KERNEL_XZ if !EFI_ZBOOT + select HAVE_KPROBES + select HAVE_KRETPROBES # https://github.com/ClangBuiltLinux/linux/issues/1881 select HAVE_LD_DEAD_CODE_DATA_ELIMINATION if !LD_IS_LLD select HAVE_MOVE_PMD @@ -179,16 +191,16 @@ config RISCV select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY if !XIP_KERNEL + select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_RETHOOK if !XIP_KERNEL + select HAVE_RETHOOK select HAVE_RSEQ select HAVE_RUST if RUSTC_SUPPORTS_RISCV && CC_IS_CLANG select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS - select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU + select HOTPLUG_PARALLEL if HOTPLUG_CPU select IRQ_DOMAIN select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN @@ -201,7 +213,8 @@ config RISCV select PCI_DOMAINS_GENERIC if PCI select PCI_ECAM if (ACPI && PCI) select PCI_MSI if PCI - select RISCV_ALTERNATIVE if !XIP_KERNEL + select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED + select RISCV_ALTERNATIVE select RISCV_APLIC select RISCV_IMSIC select RISCV_INTC @@ -212,15 +225,13 @@ config RISCV select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select UACCESS_MEMCPY if !MMU + select VDSO_GETRANDOM if HAVE_GENERIC_VDSO && 64BIT select USER_STACKTRACE_SUPPORT select ZONE_DMA32 if 64BIT config RUSTC_SUPPORTS_RISCV def_bool y depends on 64BIT - # Shadow call stack requires rustc version 1.82+ due to use of the - # -Zsanitizer=shadow-call-stack flag. - depends on !SHADOW_CALL_STACK || RUSTC_VERSION >= 108200 config CLANG_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_CLANG @@ -230,21 +241,16 @@ config CLANG_SUPPORTS_DYNAMIC_FTRACE config GCC_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_GCC depends on $(cc-option,-fpatchable-function-entry=8) + depends on CC_HAS_MIN_FUNCTION_ALIGNMENT || !RISCV_ISA_C config HAVE_SHADOW_CALL_STACK def_bool $(cc-option,-fsanitize=shadow-call-stack) # https://github.com/riscv-non-isa/riscv-elf-psabi-doc/commit/a484e843e6eeb51f0cb7b8819e50da6d2444d769 depends on $(ld-option,--no-relax-gp) -config RISCV_USE_LINKER_RELAXATION - def_bool y - # https://github.com/llvm/llvm-project/commit/6611d58f5bbcbec77262d392e2923e1d680f6985 - depends on !LD_IS_LLD || LLD_VERSION >= 150000 - # https://github.com/llvm/llvm-project/commit/bbc0f99f3bc96f1db16f649fc21dd18e5b0918f6 config ARCH_HAS_BROKEN_DWARF5 def_bool y - depends on RISCV_USE_LINKER_RELAXATION # https://github.com/llvm/llvm-project/commit/1df5ea29b43690b6622db2cad7b745607ca4de6a depends on AS_IS_LLVM && AS_VERSION < 180000 # https://github.com/llvm/llvm-project/commit/7ffabb61a5569444b5ac9322e22e5471cc5e4a77 @@ -288,13 +294,6 @@ config MMU Select if you want MMU-based virtualised addressing space support by paged memory management. If unsure, say 'Y'. -config PAGE_OFFSET - hex - default 0x80000000 if !MMU && RISCV_M_MODE - default 0x80200000 if !MMU - default 0xc0000000 if 32BIT - default 0xff60000000000000 if 64BIT - config KASAN_SHADOW_OFFSET hex depends on KASAN_GENERIC @@ -367,7 +366,7 @@ config RISCV_NONSTANDARD_CACHE_OPS systems to handle cache management. config AS_HAS_INSN - def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero) + def_bool $(as-instr,.insn 0x100000f) config AS_HAS_OPTION_ARCH # https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4 @@ -449,14 +448,6 @@ config SMP If you don't know what to do here, say N. -config SCHED_MC - bool "Multi-core scheduler support" - depends on SMP - help - Multi-core scheduler support improves the CPU scheduler's decision - making when dealing with multi-core CPU chips at a cost of slightly - increased overhead in some places. If unsure say N here. - config NR_CPUS int "Maximum number of CPUs (2-512)" depends on SMP @@ -547,7 +538,6 @@ endchoice config RISCV_ALTERNATIVE bool - depends on !XIP_KERNEL help This Kconfig allows the kernel to automatically patch the erratum or cpufeature required by the execution platform at run @@ -566,7 +556,8 @@ config RISCV_ISA_C help Adds "C" to the ISA subsets that the toolchain is allowed to emit when building Linux, which results in compressed instructions in the - Linux binary. + Linux binary. This option produces a kernel that will not run on + systems that do not support compressed instructions. If you don't know what to do here, say Y. @@ -587,8 +578,8 @@ config RISCV_ISA_SVNAPOT depends on RISCV_ALTERNATIVE default y help - Allow kernel to detect the Svnapot ISA-extension dynamically at boot - time and enable its usage. + Enable support for the Svnapot ISA-extension when it is detected + at boot. The Svnapot extension is used to mark contiguous PTEs as a range of contiguous virtual-to-physical translations for a naturally @@ -606,9 +597,8 @@ config RISCV_ISA_SVPBMT depends on RISCV_ALTERNATIVE default y help - Adds support to dynamically detect the presence of the Svpbmt - ISA-extension (Supervisor-mode: page-based memory types) and - enable its usage. + Add support for the Svpbmt ISA-extension (Supervisor-mode: + page-based memory types) in the kernel when it is detected at boot. The memory type for a page contains a combination of attributes that indicate the cacheability, idempotency, and ordering @@ -623,18 +613,19 @@ config TOOLCHAIN_HAS_V default y depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64imv) depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32imv) - depends on LLD_VERSION >= 140000 || LD_VERSION >= 23800 + depends on LD_IS_LLD || LD_VERSION >= 23800 depends on AS_HAS_OPTION_ARCH config RISCV_ISA_V - bool "VECTOR extension support" + bool "Vector extension support" depends on TOOLCHAIN_HAS_V depends on FPU select DYNAMIC_SIGFRAME default y help - Say N here if you want to disable all vector related procedure - in the kernel. + Add support for the Vector extension when it is detected at boot. + When this option is disabled, neither the kernel nor userspace may + use vector procedures. If you don't know what to do here, say Y. @@ -664,12 +655,12 @@ config RISCV_ISA_V_PREEMPTIVE default y help Usually, in-kernel SIMD routines are run with preemption disabled. - Functions which envoke long running SIMD thus must yield core's + Functions which invoke long running SIMD thus must yield the core's vector unit to prevent blocking other tasks for too long. - This config allows kernel to run SIMD without explicitly disable - preemption. Enabling this config will result in higher memory - consumption due to the allocation of per-task's kernel Vector context. + This config allows the kernel to run SIMD without explicitly disabling + preemption. Enabling this config will result in higher memory consumption + due to the allocation of per-task's kernel Vector context. config RISCV_ISA_ZAWRS bool "Zawrs extension support for more efficient busy waiting" @@ -709,7 +700,6 @@ config TOOLCHAIN_HAS_ZACAS config RISCV_ISA_ZACAS bool "Zacas extension support for atomic CAS" - depends on TOOLCHAIN_HAS_ZACAS depends on RISCV_ALTERNATIVE default y help @@ -723,7 +713,7 @@ config TOOLCHAIN_HAS_ZBB default y depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbb) depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbb) - depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on LD_IS_LLD || LD_VERSION >= 23900 depends on AS_HAS_OPTION_ARCH # This symbol indicates that the toolchain supports all v1.0 vector crypto @@ -733,6 +723,14 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb) depends on AS_HAS_OPTION_ARCH +config TOOLCHAIN_HAS_ZBA + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zba) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zba) + depends on LD_IS_LLD || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + config RISCV_ISA_ZBA bool "Zba extension support for bit manipulation instructions" default y @@ -747,12 +745,12 @@ config RISCV_ISA_ZBA config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" - depends on TOOLCHAIN_HAS_ZBB depends on RISCV_ALTERNATIVE default y help - Adds support to dynamically detect the presence of the ZBB - extension (basic bit manipulation) and enable its usage. + Add support for enabling optimisations in the kernel when the + Zbb extension is detected at boot. Some optimisations may + additionally depend on toolchain support for Zbb. The Zbb extension provides instructions to accelerate a number of bit-specific operations (count bit population, sign extending, @@ -765,7 +763,7 @@ config TOOLCHAIN_HAS_ZBC default y depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbc) depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbc) - depends on LLD_VERSION >= 150000 || LD_VERSION >= 23900 + depends on LD_IS_LLD || LD_VERSION >= 23900 depends on AS_HAS_OPTION_ARCH config RISCV_ISA_ZBC @@ -783,6 +781,28 @@ config RISCV_ISA_ZBC If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZBKB + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zbkb) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zbkb) + depends on LD_IS_LLD || LD_VERSION >= 23900 + depends on AS_HAS_OPTION_ARCH + +config RISCV_ISA_ZBKB + bool "Zbkb extension support for bit manipulation instructions" + depends on TOOLCHAIN_HAS_ZBKB + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZBKB + extension (bit manipulation for cryptography) and enable its usage. + + The Zbkb extension provides instructions to accelerate a number + of common cryptography operations (pack, zip, etc). + + If you don't know what to do here, say Y. + config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU @@ -791,9 +811,9 @@ config RISCV_ISA_ZICBOM select RISCV_DMA_NONCOHERENT select DMA_DIRECT_REMAP help - Adds support to dynamically detect the presence of the ZICBOM - extension (Cache Block Management Operations) and enable its - usage. + Add support for the Zicbom extension (Cache Block Management + Operations) and enable its use in the kernel when it is detected + at boot. The Zicbom extension can be used to handle for example non-coherent DMA support on devices that need it. @@ -806,12 +826,41 @@ config RISCV_ISA_ZICBOZ default y help Enable the use of the Zicboz extension (cbo.zero instruction) - when available. + in the kernel when it is detected at boot. The Zicboz extension is used for faster zeroing of memory. If you don't know what to do here, say Y. +config RISCV_ISA_ZICBOP + bool "Zicbop extension support for cache block prefetch" + depends on MMU + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the ZICBOP + extension (Cache Block Prefetch Operations) and enable its + usage. + + The Zicbop extension can be used to prefetch cache blocks for + read/write fetch. + + If you don't know what to do here, say Y. + +config RISCV_ISA_SVRSW60T59B + bool "Svrsw60t59b extension support for using PTE bits 60 and 59" + depends on MMU && 64BIT + depends on RISCV_ALTERNATIVE + default y + help + Adds support to dynamically detect the presence of the Svrsw60t59b + extension and enable its usage. + + The Svrsw60t59b extension allows to free the PTE reserved bits 60 + and 59 for software to use. + + If you don't know what to do here, say Y. + config TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI def_bool y # https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=aed44286efa8ae8717a77d94b51ac3614e2ca6dc @@ -844,8 +893,9 @@ config FPU bool "FPU support" default y help - Say N here if you want to disable all floating-point related procedure - in the kernel. + Add support for floating point operations when an FPU is detected at + boot. When this option is disabled, neither the kernel nor userspace + may use the floating point unit. If you don't know what to do here, say Y. @@ -887,6 +937,28 @@ config RISCV_VECTOR_MISALIGNED help Enable detecting support for vector misaligned loads and stores. +config RISCV_SBI_FWFT_DELEGATE_MISALIGNED + bool "Request firmware delegation of unaligned access exceptions" + depends on RISCV_SBI + depends on NONPORTABLE + help + Use SBI FWFT to request delegation of load address misaligned and + store address misaligned exceptions, if possible, and prefer Linux + kernel emulation of these accesses to firmware emulation. + + Unfortunately, Linux's emulation is still incomplete. Namely, it + currently does not handle vector instructions and KVM guest accesses. + On platforms where these accesses would have been handled by firmware, + enabling this causes unexpected kernel oopses, userspaces crashes and + KVM guest crashes. If you are sure that these are not a problem for + your platform, you can say Y here, which may improve performance. + + Saying N here will not worsen emulation support for unaligned accesses + even in the case where the firmware also has incomplete support. It + simply keeps the firmware's emulation enabled. + + If you don't know what to do here, say N. + choice prompt "Unaligned Accesses Support" default RISCV_PROBE_UNALIGNED_ACCESS @@ -1057,6 +1129,7 @@ config COMPAT config PARAVIRT bool "Enable paravirtualization code" depends on RISCV_SBI + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly @@ -1075,8 +1148,8 @@ config PARAVIRT_TIME_ACCOUNTING config RELOCATABLE bool "Build a relocatable kernel" - depends on MMU && 64BIT && !XIP_KERNEL select MODULE_SECTIONS if MODULES + select ARCH_VMLINUX_NEEDS_RELOCS help This builds a kernel as a Position Independent Executable (PIE), which retains all relocation metadata required to relocate the @@ -1091,7 +1164,7 @@ config RELOCATABLE config RANDOMIZE_BASE bool "Randomize the address of the kernel image" select RELOCATABLE - depends on MMU && 64BIT && !XIP_KERNEL + depends on MMU && 64BIT help Randomizes the virtual address at which the kernel image is loaded, as a security feature that deters exploit attempts @@ -1107,6 +1180,28 @@ config RANDOMIZE_BASE If unsure, say N. +config RISCV_USER_CFI + def_bool y + bool "riscv userspace control flow integrity" + depends on 64BIT && MMU && \ + $(cc-option,-mabi=lp64 -march=rv64ima_zicfiss_zicfilp -fcf-protection=full) + depends on RISCV_ALTERNATIVE + select RISCV_SBI + select ARCH_HAS_USER_SHADOW_STACK + select ARCH_USES_HIGH_VMA_FLAGS + select DYNAMIC_SIGFRAME + help + Provides CPU-assisted control flow integrity to userspace tasks. + Control flow integrity is provided by implementing shadow stack for + backward edge and indirect branch tracking for forward edge. + Shadow stack protection is a hardware feature that detects function + return address corruption. This helps mitigate ROP attacks. + Indirect branch tracking enforces that all indirect branches must land + on a landing pad instruction else CPU will fault. This mitigates against + JOP / COP attacks. Applications must be enabled to use it, and old userspace + does not get protection "for free". + default y. + endmenu # "Kernel features" menu "Boot options" @@ -1139,8 +1234,8 @@ config CMDLINE_FALLBACK config CMDLINE_EXTEND bool "Extend bootloader kernel arguments" help - The command-line arguments provided during boot will be - appended to the built-in command line. This is useful in + The built-in command line will be appended to the command- + line arguments provided during boot. This is useful in cases where the provided arguments are insufficient and you don't want to or cannot modify them. @@ -1159,7 +1254,7 @@ config EFI_STUB config EFI bool "UEFI runtime support" - depends on OF && !XIP_KERNEL + depends on OF depends on MMU default y select ARCH_SUPPORTS_ACPI if 64BIT @@ -1210,44 +1305,6 @@ config PHYS_RAM_BASE explicitly specified to run early relocations of read-write data from flash to RAM. -config XIP_KERNEL - bool "Kernel Execute-In-Place from ROM" - depends on MMU && SPARSEMEM && NONPORTABLE - # This prevents XIP from being enabled by all{yes,mod}config, which - # fail to build since XIP doesn't support large kernels. - depends on !COMPILE_TEST - select PHYS_RAM_BASE_FIXED - help - Execute-In-Place allows the kernel to run from non-volatile storage - directly addressable by the CPU, such as NOR flash. This saves RAM - space since the text section of the kernel is not loaded from flash - to RAM. Read-write sections, such as the data section and stack, - are still copied to RAM. The XIP kernel is not compressed since - it has to run directly from flash, so it will take more space to - store it. The flash address used to link the kernel object files, - and for storing it, is configuration dependent. Therefore, if you - say Y here, you must know the proper physical address where to - store the kernel image depending on your own flash memory usage. - - Also note that the make target becomes "make xipImage" rather than - "make zImage" or "make Image". The final kernel binary to put in - ROM memory will be arch/riscv/boot/xipImage. - - SPARSEMEM is required because the kernel text and rodata that are - flash resident are not backed by memmap, then any attempt to get - a struct page on those regions will trigger a fault. - - If unsure, say N. - -config XIP_PHYS_ADDR - hex "XIP Kernel Physical Location" - depends on XIP_KERNEL - default "0x21000000" - help - This is the physical address in your flash memory the kernel will - be linked for and stored to. This address is dependent on your - own flash usage. - config RISCV_ISA_FALLBACK bool "Permit falling back to parsing riscv,isa for extension support by default" default y @@ -1269,13 +1326,14 @@ config RISCV_ISA_FALLBACK config BUILTIN_DTB bool "Built-in device tree" depends on OF && NONPORTABLE + select GENERIC_BUILTIN_DTB help Build a device tree into the Linux image. This option should be selected if no bootloader is being used. If unsure, say N. -config BUILTIN_DTB_SOURCE +config BUILTIN_DTB_NAME string "Built-in device tree source" depends on BUILTIN_DTB help diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index e318119d570d..3c945d086c7d 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -21,6 +21,29 @@ config ERRATA_ANDES_CMO If you don't know what to do here, say "Y". +config ERRATA_MIPS + bool "MIPS errata" + depends on RISCV_ALTERNATIVE + help + All MIPS errata Kconfig depend on this Kconfig. Disabling + this Kconfig will disable all MIPS errata. Please say "Y" + here if your platform uses MIPS CPU cores. + + Otherwise, please say "N" here to avoid unnecessary overhead. + +config ERRATA_MIPS_P8700_PAUSE_OPCODE + bool "Fix the PAUSE Opcode for MIPS P8700" + depends on ERRATA_MIPS && 64BIT + default n + help + The RISCV MIPS P8700 uses a different opcode for PAUSE. + It is a 'hint' encoding of the SLLI instruction, + with rd=0, rs1=0 and imm=5. It will behave as a NOP + instruction if no additional behavior beyond that of + SLLI is implemented. + + If you are not using the P8700 processor, say n. + config ERRATA_SIFIVE bool "SiFive errata" depends on RISCV_ALTERNATIVE @@ -61,6 +84,7 @@ config ERRATA_STARFIVE_JH7100 select DMA_GLOBAL_POOL select RISCV_DMA_NONCOHERENT select RISCV_NONSTANDARD_CACHE_OPS + select CACHEMAINT_FOR_DMA select SIFIVE_CCACHE default n help diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 1916cf7ba450..c174ac0ec46b 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -1,5 +1,23 @@ menu "SoC selection" +config ARCH_ANDES + bool "Andes SoCs" + depends on MMU + select ERRATA_ANDES + help + This enables support for Andes SoC platform hardware. + +config ARCH_ANLOGIC + bool "Anlogic SoCs" + help + This enables support for Anlogic SoC platform hardware. + +config ARCH_ESWIN + bool "ESWIN SoCs" + help + This enables support for ESWIN SoC platform hardware, + including the ESWIN EIC7700 SoC. + config ARCH_MICROCHIP_POLARFIRE def_bool ARCH_MICROCHIP @@ -15,7 +33,7 @@ config ARCH_RENESAS config ARCH_SIFIVE bool "SiFive SoCs" - select ERRATA_SIFIVE if !XIP_KERNEL + select ERRATA_SIFIVE help This enables support for SiFive SoC platform hardware. @@ -26,6 +44,7 @@ config ARCH_SOPHGO config ARCH_SPACEMIT bool "SpacemiT SoCs" + select PINCTRL help This enables support for SpacemiT SoC platform hardware. @@ -42,23 +61,31 @@ config SOC_STARFIVE config ARCH_SUNXI bool "Allwinner sun20i SoCs" - depends on MMU && !XIP_KERNEL + depends on MMU select ERRATA_THEAD select SUN4I_TIMER help This enables support for Allwinner sun20i platform hardware, including boards based on the D1 and D1s SoCs. +config ARCH_TENSTORRENT + bool "Tenstorrent SoCs" + help + This enables support for Tenstorrent SoC platforms. + Current support is for Blackhole P100 and P150 PCIe cards. + The Blackhole SoC contains four RISC-V CPU tiles each + consisting of 4x SiFive X280 cores. + config ARCH_THEAD bool "T-HEAD RISC-V SoCs" - depends on MMU && !XIP_KERNEL + depends on MMU select ERRATA_THEAD + select PM_GENERIC_DOMAINS if PM help This enables support for the RISC-V based T-HEAD SoCs. config ARCH_VIRT bool "QEMU Virt Machine" - select CLINT_TIMER if RISCV_M_MODE select POWER_RESET select POWER_RESET_SYSCON select POWER_RESET_SYSCON_POWEROFF @@ -78,7 +105,6 @@ config ARCH_CANAAN config SOC_CANAAN_K210 bool "Canaan Kendryte K210 SoC" depends on !MMU && ARCH_CANAAN - select CLINT_TIMER if RISCV_M_MODE select ARCH_HAS_RESET_CONTROLLER select PINCTRL select COMMON_CLK diff --git a/arch/riscv/Kconfig.vendor b/arch/riscv/Kconfig.vendor index b096548fe0ff..3c1f92e406c3 100644 --- a/arch/riscv/Kconfig.vendor +++ b/arch/riscv/Kconfig.vendor @@ -16,6 +16,32 @@ config RISCV_ISA_VENDOR_EXT_ANDES If you don't know what to do here, say Y. endmenu +menu "MIPS" +config RISCV_ISA_VENDOR_EXT_MIPS + bool "MIPS vendor extension support" + select RISCV_ISA_VENDOR_EXT + default y + help + Say N here to disable detection of and support for all MIPS vendor + extensions. Without this option enabled, MIPS vendor extensions will + not be detected at boot and their presence not reported to userspace. + + If you don't know what to do here, say Y. +endmenu + +menu "SiFive" +config RISCV_ISA_VENDOR_EXT_SIFIVE + bool "SiFive vendor extension support" + select RISCV_ISA_VENDOR_EXT + default y + help + Say N here if you want to disable all SiFive vendor extension + support. This will cause any SiFive vendor extensions that are + requested by hardware probing to be ignored. + + If you don't know what to do here, say Y. +endmenu + menu "T-Head" config RISCV_ISA_VENDOR_EXT_THEAD bool "T-Head vendor extension support" diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 13fbc0f94238..ce0cc737f870 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -8,16 +8,16 @@ LDFLAGS_vmlinux := -z norelro ifeq ($(CONFIG_RELOCATABLE),y) - LDFLAGS_vmlinux += -shared -Bsymbolic -z notext --emit-relocs + LDFLAGS_vmlinux += -shared -Bsymbolic -z notext KBUILD_CFLAGS += -fPIE endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux += --no-relax KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY ifeq ($(CONFIG_RISCV_ISA_C),y) - CC_FLAGS_FTRACE := -fpatchable-function-entry=4 + CC_FLAGS_FTRACE := -fpatchable-function-entry=8,4 else - CC_FLAGS_FTRACE := -fpatchable-function-entry=2 + CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2 endif endif @@ -28,7 +28,6 @@ endif export BITS ifeq ($(CONFIG_ARCH_RV64I),y) BITS := 64 - UTS_MACHINE := riscv64 KBUILD_CFLAGS += -mabi=lp64 KBUILD_AFLAGS += -mabi=lp64 @@ -39,24 +38,18 @@ ifeq ($(CONFIG_ARCH_RV64I),y) -Cno-redzone else BITS := 32 - UTS_MACHINE := riscv32 KBUILD_CFLAGS += -mabi=ilp32 KBUILD_AFLAGS += -mabi=ilp32 KBUILD_LDFLAGS += -melf32lriscv endif -ifndef CONFIG_RISCV_USE_LINKER_RELAXATION - KBUILD_CFLAGS += -mno-relax - KBUILD_AFLAGS += -mno-relax -ifndef CONFIG_AS_IS_LLVM - KBUILD_CFLAGS += -Wa,-mno-relax - KBUILD_AFLAGS += -Wa,-mno-relax -endif +UTS_MACHINE := riscv$(BITS) + # LLVM has an issue with target-features and LTO: https://github.com/llvm/llvm-project/issues/59350 # Ensure it is aware of linker relaxation with LTO, otherwise relocations may # be incorrect: https://github.com/llvm/llvm-project/issues/65090 -else ifeq ($(CONFIG_LTO_CLANG),y) +ifeq ($(CONFIG_LTO_CLANG),y) KBUILD_LDFLAGS += -mllvm -mattr=+c -mllvm -mattr=+relax endif @@ -88,9 +81,12 @@ riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas # Check if the toolchain supports Zabha riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha +KBUILD_BASE_ISA = -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') +export KBUILD_BASE_ISA + # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by # matching non-v and non-multi-letter extensions out with the filter ([^v_]*) -KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/') +KBUILD_CFLAGS += $(KBUILD_BASE_ISA) KBUILD_AFLAGS += -march=$(riscv-march-y) @@ -98,7 +94,6 @@ KBUILD_AFLAGS += -march=$(riscv-march-y) CC_FLAGS_FPU := -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)([^v_]*)v?/\1\2/') KBUILD_CFLAGS += -mno-save-restore -KBUILD_CFLAGS += -DCONFIG_PAGE_OFFSET=$(CONFIG_PAGE_OFFSET) ifeq ($(CONFIG_CMODEL_MEDLOW),y) KBUILD_CFLAGS += -mcmodel=medlow @@ -142,21 +137,6 @@ endif CHECKFLAGS += -D__riscv -D__riscv_xlen=$(BITS) # Default target when executing plain make -boot := arch/riscv/boot -ifeq ($(CONFIG_XIP_KERNEL),y) -KBUILD_IMAGE := $(boot)/xipImage -else -ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_CANAAN_K210),yy) -KBUILD_IMAGE := $(boot)/loader.bin -else -ifeq ($(CONFIG_EFI_ZBOOT),) -KBUILD_IMAGE := $(boot)/Image.gz -else -KBUILD_IMAGE := $(boot)/vmlinuz.efi -endif -endif -endif - boot := arch/riscv/boot boot-image-y := Image boot-image-$(CONFIG_KERNEL_BZIP2) := Image.bz2 @@ -167,10 +147,9 @@ boot-image-$(CONFIG_KERNEL_LZO) := Image.lzo boot-image-$(CONFIG_KERNEL_ZSTD) := Image.zst boot-image-$(CONFIG_KERNEL_XZ) := Image.xz ifdef CONFIG_RISCV_M_MODE -boot-image-$(CONFIG_ARCH_CANAAN) := loader.bin +boot-image-$(CONFIG_SOC_CANAAN_K210) := loader.bin endif boot-image-$(CONFIG_EFI_ZBOOT) := vmlinuz.efi -boot-image-$(CONFIG_XIP_KERNEL) := xipImage KBUILD_IMAGE := $(boot)/$(boot-image-y) libs-y += arch/riscv/lib/ @@ -181,6 +160,8 @@ ifeq ($(CONFIG_MMU),y) prepare: vdso_prepare vdso_prepare: prepare0 $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso include/generated/vdso-offsets.h + $(if $(CONFIG_RISCV_USER_CFI),$(Q)$(MAKE) \ + $(build)=arch/riscv/kernel/vdso_cfi include/generated/vdso-cfi-offsets.h) $(if $(CONFIG_COMPAT),$(Q)$(MAKE) \ $(build)=arch/riscv/kernel/compat_vdso include/generated/compat_vdso-offsets.h) @@ -188,6 +169,7 @@ endif endif vdso-install-y += arch/riscv/kernel/vdso/vdso.so.dbg +vdso-install-$(CONFIG_RISCV_USER_CFI) += arch/riscv/kernel/vdso_cfi/vdso-cfi.so.dbg vdso-install-$(CONFIG_COMPAT) += arch/riscv/kernel/compat_vdso/compat_vdso.so.dbg BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin xipImage vmlinuz.efi @@ -235,8 +217,6 @@ define archhelp echo ' Image.xz - Compressed kernel image (arch/riscv/boot/Image.xz)' echo ' vmlinuz.efi - Compressed EFI kernel image (arch/riscv/boot/vmlinuz.efi)' echo ' Default when CONFIG_EFI_ZBOOT=y' - echo ' xipImage - Execute-in-place kernel image (arch/riscv/boot/xipImage)' - echo ' Default when CONFIG_XIP_KERNEL=y' echo ' install - Install kernel using (your) ~/bin/$(INSTALLKERNEL) or' echo ' (distribution) /sbin/$(INSTALLKERNEL) or install to ' echo ' $$(INSTALL_PATH)' diff --git a/arch/riscv/Makefile.postlink b/arch/riscv/Makefile.postlink index 6b0580949b6a..0e4cf8ad2f14 100644 --- a/arch/riscv/Makefile.postlink +++ b/arch/riscv/Makefile.postlink @@ -10,26 +10,17 @@ __archpost: -include include/config/auto.conf include $(srctree)/scripts/Kbuild.include -include $(srctree)/scripts/Makefile.lib quiet_cmd_relocs_check = CHKREL $@ cmd_relocs_check = \ $(CONFIG_SHELL) $(srctree)/arch/riscv/tools/relocs_check.sh "$(OBJDUMP)" "$(NM)" "$@" -ifdef CONFIG_RELOCATABLE -quiet_cmd_cp_vmlinux_relocs = CPREL vmlinux.relocs -cmd_cp_vmlinux_relocs = cp vmlinux vmlinux.relocs - -endif - # `@true` prevents complaint when there is nothing to be done -vmlinux: FORCE +vmlinux vmlinux.unstripped: FORCE @true ifdef CONFIG_RELOCATABLE $(call if_changed,relocs_check) - $(call if_changed,cp_vmlinux_relocs) - $(call if_changed,strip_relocs) endif clean: diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index b25d524ce5eb..fcfbe3f814d6 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -20,25 +20,7 @@ OBJCOPYFLAGS_xipImage :=-O binary -R .note -R .note.gnu.build-id -R .comment -S targets := Image Image.* loader loader.o loader.lds loader.bin xipImage -ifeq ($(CONFIG_XIP_KERNEL),y) - -quiet_cmd_mkxip = $(quiet_cmd_objcopy) -cmd_mkxip = $(cmd_objcopy) - -$(obj)/xipImage: vmlinux FORCE - $(call if_changed,mkxip) - @$(kecho) ' Physical Address of xipImage: $(CONFIG_XIP_PHYS_ADDR)' - -endif - -ifdef CONFIG_RELOCATABLE -vmlinux.relocs: vmlinux - @ (! [ -f vmlinux.relocs ] && echo "vmlinux.relocs can't be found, please remove vmlinux and try again") || true - -$(obj)/Image: vmlinux.relocs FORCE -else $(obj)/Image: vmlinux FORCE -endif $(call if_changed,objcopy) $(obj)/Image.gz: $(obj)/Image FORCE diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index bff887d38abe..69d8751fb17c 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 subdir-y += allwinner +subdir-y += andes +subdir-y += anlogic subdir-y += canaan +subdir-y += eswin subdir-y += microchip subdir-y += renesas subdir-y += sifive subdir-y += sophgo subdir-y += spacemit subdir-y += starfive +subdir-y += tenstorrent subdir-y += thead - -obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE)) diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts index bc5c84f22762..5f2e5cc3e3d5 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-devterm-v3.14.dts @@ -17,7 +17,7 @@ #cooling-cells = <2>; }; - i2c-gpio-0 { + i2c-0 { compatible = "i2c-gpio"; sda-gpios = <&pio 3 14 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>; /* PD14/GPIO44 */ scl-gpios = <&pio 3 15 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>; /* PD15/GPIO45 */ diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts index 08cf716328a0..feaa75d5aead 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-lichee-rv-dock.dts @@ -59,6 +59,18 @@ status = "okay"; }; +&ledc { + pinctrl-0 = <&ledc_pc0_pin>; + pinctrl-names = "default"; + status = "okay"; + + multi-led@0 { + reg = <0x0>; + color = <LED_COLOR_ID_RGB>; + function = LED_FUNCTION_STATUS; + }; +}; + &mmc1 { bus-width = <4>; mmc-pwrseq = <&wifi_pwrseq>; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts b/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts index 8dbe717c79ce..73840ea300f0 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1-nezha.dts @@ -22,6 +22,7 @@ #include <dt-bindings/gpio/gpio.h> #include <dt-bindings/input/input.h> +#include <dt-bindings/leds/common.h> /dts-v1/; @@ -121,6 +122,18 @@ }; }; +&ledc { + pinctrl-0 = <&ledc_pc0_pin>; + pinctrl-names = "default"; + status = "okay"; + + multi-led@0 { + reg = <0x0>; + color = <LED_COLOR_ID_RGB>; + function = LED_FUNCTION_STATUS; + }; +}; + &mdio { ext_rgmii_phy: ethernet-phy@1 { compatible = "ethernet-phy-ieee802.3-c22"; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi b/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi index b18f368e06e0..b0fb0ea377bc 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1.dtsi @@ -59,6 +59,12 @@ }; /omit-if-no-ref/ + ledc_pc0_pin: ledc-pc0-pin { + pins = "PC0"; + function = "ledc"; + }; + + /omit-if-no-ref/ uart0_pb8_pins: uart0-pb8-pins { pins = "PB8", "PB9"; function = "uart0"; diff --git a/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi b/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi index 6367112e614a..3f4ee820ef56 100644 --- a/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi +++ b/arch/riscv/boot/dts/allwinner/sun20i-d1s.dtsi @@ -1,6 +1,8 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) // Copyright (C) 2021-2022 Samuel Holland <samuel@sholland.org> +#include <dt-bindings/thermal/thermal.h> + #define SOC_PERIPHERAL_IRQ(nr) (nr + 16) #include "sunxi-d1s-t113.dtsi" @@ -28,7 +30,7 @@ riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", "zifencei", "zihpm", "xtheadvector"; - thead,vlenb = <128>; + thead,vlenb = <16>; #cooling-cells = <2>; cpu0_intc: interrupt-controller { @@ -115,4 +117,33 @@ <0x00000000 0x0000000e 0xffffffff 0xffffffff 0x00010000>, <0x00000000 0x0000000f 0xffffffff 0xffffffff 0x00020000>; }; + + thermal-zones { + cpu-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&ths>; + + cooling-maps { + map0 { + trip = <&cpu_alert>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + + trips { + cpu_alert: cpu-alert { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu-crit { + temperature = <100000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; }; diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi index e4175adb028d..82cc85acccb1 100644 --- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi +++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi @@ -79,6 +79,36 @@ }; /omit-if-no-ref/ + i2c2_pd_pins: i2c2-pd-pins { + pins = "PD20", "PD21"; + function = "i2c2"; + }; + + /omit-if-no-ref/ + i2c3_pg_pins: i2c3-pg-pins { + pins = "PG10", "PG11"; + function = "i2c3"; + }; + + /omit-if-no-ref/ + i2s1_pins: i2s1-pins { + pins = "PG12", "PG13"; + function = "i2s1"; + }; + + /omit-if-no-ref/ + i2s1_din0_pin: i2s1-din0-pin { + pins = "PG14"; + function = "i2s1_din"; + }; + + /omit-if-no-ref/ + i2s1_dout0_pin: i2s1-dout0-pin { + pins = "PG15"; + function = "i2s1_dout"; + }; + + /omit-if-no-ref/ lcd_rgb666_pins: lcd-rgb666-pins { pins = "PD0", "PD1", "PD2", "PD3", "PD4", "PD5", "PD6", "PD7", "PD8", "PD9", "PD10", "PD11", @@ -127,6 +157,24 @@ }; /omit-if-no-ref/ + spi1_pins: spi1-pins { + pins = "PD10", "PD11", "PD12", "PD13"; + function = "spi1"; + }; + + /omit-if-no-ref/ + spi1_hold_pin: spi1-hold-pin { + pins = "PD14"; + function = "spi1"; + }; + + /omit-if-no-ref/ + spi1_wp_pin: spi1-wp-pin { + pins = "PD15"; + function = "spi1"; + }; + + /omit-if-no-ref/ uart1_pg6_pins: uart1-pg6-pins { pins = "PG6", "PG7"; function = "uart1"; @@ -156,6 +204,21 @@ #reset-cells = <1>; }; + ledc: led-controller@2008000 { + compatible = "allwinner,sun20i-d1-ledc", + "allwinner,sun50i-a100-ledc"; + reg = <0x2008000 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(20) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_LEDC>, <&ccu CLK_LEDC>; + clock-names = "bus", "mod"; + resets = <&ccu RST_BUS_LEDC>; + dmas = <&dma 42>; + dma-names = "tx"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + gpadc: adc@2009000 { compatible = "allwinner,sun20i-d1-gpadc"; reg = <0x2009000 0x400>; @@ -166,6 +229,18 @@ #io-channel-cells = <1>; }; + ths: thermal-sensor@2009400 { + compatible = "allwinner,sun20i-d1-ths"; + reg = <0x2009400 0x400>; + interrupts = <SOC_PERIPHERAL_IRQ(58) IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_THS>; + clock-names = "bus"; + resets = <&ccu RST_BUS_THS>; + nvmem-cells = <&ths_calibration>; + nvmem-cell-names = "calibration"; + #thermal-sensor-cells = <0>; + }; + dmic: dmic@2031000 { compatible = "allwinner,sun20i-d1-dmic", "allwinner,sun50i-h6-dmic"; @@ -426,6 +501,10 @@ reg = <0x3006000 0x1000>; #address-cells = <1>; #size-cells = <1>; + + ths_calibration: thermal-sensor-calibration@14 { + reg = <0x14 0x8>; + }; }; crypto: crypto@3040000 { diff --git a/arch/riscv/boot/dts/andes/Makefile b/arch/riscv/boot/dts/andes/Makefile new file mode 100644 index 000000000000..c545c668ef70 --- /dev/null +++ b/arch/riscv/boot/dts/andes/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_ANDES) += qilai-voyager.dtb diff --git a/arch/riscv/boot/dts/andes/qilai-voyager.dts b/arch/riscv/boot/dts/andes/qilai-voyager.dts new file mode 100644 index 000000000000..fa7d2b32a9b4 --- /dev/null +++ b/arch/riscv/boot/dts/andes/qilai-voyager.dts @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Andes Technology Corporation. All rights reserved. + */ + +#include "qilai.dtsi" + +/ { + model = "Voyager"; + compatible = "andestech,voyager", "andestech,qilai"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@400000000 { + device_type = "memory"; + reg = <0x4 0x00000000 0x4 0x00000000>; + }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/andes/qilai.dtsi b/arch/riscv/boot/dts/andes/qilai.dtsi new file mode 100644 index 000000000000..de3de32f8c39 --- /dev/null +++ b/arch/riscv/boot/dts/andes/qilai.dtsi @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Andes Technology Corporation. All rights reserved. + */ + +/dts-v1/; + +#include <dt-bindings/interrupt-controller/irq.h> + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <62500000>; + + cpu0: cpu@0 { + compatible = "andestech,ax45mp", "riscv"; + device_type = "cpu"; + reg = <0>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "zicntr", "zicsr", "zifencei", + "zihpm", "xandespmu"; + mmu-type = "riscv,sv39"; + clock-frequency = <100000000>; + i-cache-size = <0x8000>; + i-cache-sets = <256>; + i-cache-line-size = <64>; + d-cache-size = <0x8000>; + d-cache-sets = <128>; + d-cache-line-size = <64>; + next-level-cache = <&l2_cache>; + + cpu0_intc: interrupt-controller { + compatible = "andestech,cpu-intc", "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu1: cpu@1 { + compatible = "andestech,ax45mp", "riscv"; + device_type = "cpu"; + reg = <1>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "zicntr", "zicsr", "zifencei", + "zihpm", "xandespmu"; + mmu-type = "riscv,sv39"; + clock-frequency = <100000000>; + i-cache-size = <0x8000>; + i-cache-sets = <256>; + i-cache-line-size = <64>; + d-cache-size = <0x8000>; + d-cache-sets = <128>; + d-cache-line-size = <64>; + next-level-cache = <&l2_cache>; + + cpu1_intc: interrupt-controller { + compatible = "andestech,cpu-intc", + "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu2: cpu@2 { + compatible = "andestech,ax45mp", "riscv"; + device_type = "cpu"; + reg = <2>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "zicntr", "zicsr", "zifencei", + "zihpm", "xandespmu"; + mmu-type = "riscv,sv39"; + clock-frequency = <100000000>; + i-cache-size = <0x8000>; + i-cache-sets = <256>; + i-cache-line-size = <64>; + d-cache-size = <0x8000>; + d-cache-sets = <128>; + d-cache-line-size = <64>; + next-level-cache = <&l2_cache>; + + cpu2_intc: interrupt-controller { + compatible = "andestech,cpu-intc", + "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu3: cpu@3 { + compatible = "andestech,ax45mp", "riscv"; + device_type = "cpu"; + reg = <3>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "zicntr", "zicsr", "zifencei", + "zihpm", "xandespmu"; + mmu-type = "riscv,sv39"; + clock-frequency = <100000000>; + i-cache-size = <0x8000>; + i-cache-sets = <256>; + i-cache-line-size = <64>; + d-cache-size = <0x8000>; + d-cache-sets = <128>; + d-cache-line-size = <64>; + next-level-cache = <&l2_cache>; + + cpu3_intc: interrupt-controller { + compatible = "andestech,cpu-intc", + "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; + + soc { + compatible = "simple-bus"; + ranges; + interrupt-parent = <&plic>; + #address-cells = <2>; + #size-cells = <2>; + + plmt: timer@100000 { + compatible = "andestech,qilai-plmt", "andestech,plmt0"; + reg = <0x0 0x00100000 0x0 0x100000>; + interrupts-extended = <&cpu0_intc 7>, + <&cpu1_intc 7>, + <&cpu2_intc 7>, + <&cpu3_intc 7>; + }; + + l2_cache: cache-controller@200000 { + compatible = "andestech,qilai-ax45mp-cache", + "andestech,ax45mp-cache", "cache"; + reg = <0x0 0x00200000 0x0 0x100000>; + interrupts = <16 IRQ_TYPE_LEVEL_HIGH>; + cache-line-size = <64>; + cache-level = <2>; + cache-sets = <2048>; + cache-size = <0x200000>; + cache-unified; + }; + + plic_sw: interrupt-controller@400000 { + compatible = "andestech,qilai-plicsw", + "andestech,plicsw"; + reg = <0x0 0x00400000 0x0 0x400000>; + interrupts-extended = <&cpu0_intc 3>, + <&cpu1_intc 3>, + <&cpu2_intc 3>, + <&cpu3_intc 3>; + }; + + plic: interrupt-controller@2000000 { + compatible = "andestech,qilai-plic", + "andestech,nceplic100"; + reg = <0x0 0x02000000 0x0 0x2000000>; + #address-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>; + riscv,ndev = <71>; + }; + + uart0: serial@30300000 { + compatible = "andestech,uart16550", "ns16550a"; + reg = <0x0 0x30300000 0x0 0x100000>; + interrupts = <9 IRQ_TYPE_LEVEL_HIGH>; + clock-frequency = <50000000>; + reg-offset = <32>; + reg-shift = <2>; + reg-io-width = <4>; + no-loopback-test; + }; + }; +}; diff --git a/arch/riscv/boot/dts/anlogic/Makefile b/arch/riscv/boot/dts/anlogic/Makefile new file mode 100644 index 000000000000..87f3b2f418cf --- /dev/null +++ b/arch/riscv/boot/dts/anlogic/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_ANLOGIC) += dr1v90-mlkpai-fs01.dtb diff --git a/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts b/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts new file mode 100644 index 000000000000..597407655efd --- /dev/null +++ b/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Junhui Liu <junhui.liu@pigmoral.tech> + */ + +#include "dr1v90.dtsi" + +/ { + model = "Milianke MLKPAI-FS01"; + compatible = "milianke,mlkpai-fs01", "anlogic,dr1v90"; + + aliases { + serial0 = &uart1; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x20000000>; + }; +}; + +&uart1 { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/anlogic/dr1v90.dtsi b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi new file mode 100644 index 000000000000..9fe183f5f5c8 --- /dev/null +++ b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Junhui Liu <junhui.liu@pigmoral.tech> + */ + +/dts-v1/; +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Anlogic DR1V90"; + compatible = "anlogic,dr1v90"; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <800000000>; + + cpu@0 { + compatible = "nuclei,ux900", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <256>; + d-cache-size = <32768>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <256>; + i-cache-size = <32768>; + mmu-type = "riscv,sv39"; + reg = <0>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", + "zba", "zbb", "zbc", "zbkc", "zbs", + "zicntr", "zicsr", "zifencei", + "zihintpause", "zihpm"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; + + soc { + compatible = "simple-bus"; + interrupt-parent = <&plic>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + aclint_mswi: interrupt-controller@68031000 { + compatible = "anlogic,dr1v90-aclint-mswi", "nuclei,ux900-aclint-mswi"; + reg = <0x0 0x68031000 0x0 0x4000>; + interrupts-extended = <&cpu0_intc 3>; + }; + + aclint_mtimer: timer@68035000 { + compatible = "anlogic,dr1v90-aclint-mtimer", "nuclei,ux900-aclint-mtimer"; + reg = <0x0 0x68035000 0x0 0x8000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu0_intc 7>; + }; + + aclint_sswi: interrupt-controller@6803d000 { + compatible = "anlogic,dr1v90-aclint-sswi", "nuclei,ux900-aclint-sswi"; + reg = <0x0 0x6803d000 0x0 0x3000>; + #interrupt-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 1>; + }; + + plic: interrupt-controller@6c000000 { + compatible = "anlogic,dr1v90-plic", "sifive,plic-1.0.0"; + reg = <0x0 0x6c000000 0x0 0x4000000>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + riscv,ndev = <150>; + }; + + uart0: serial@f8400000 { + compatible = "anlogic,dr1v90-uart", "snps,dw-apb-uart"; + reg = <0x0 0xf8400000 0x0 0x1000>; + clock-frequency = <50000000>; + interrupts = <71>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + uart1: serial@f8401000 { + compatible = "anlogic,dr1v90-uart", "snps,dw-apb-uart"; + reg = <0x0 0xf8401000 0x0 0x1000>; + clock-frequency = <50000000>; + interrupts = <72>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/eswin/Makefile b/arch/riscv/boot/dts/eswin/Makefile new file mode 100644 index 000000000000..224101ae471e --- /dev/null +++ b/arch/riscv/boot/dts/eswin/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_ESWIN) += eic7700-hifive-premier-p550.dtb diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts new file mode 100644 index 000000000000..131ed1fc6b2e --- /dev/null +++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2024, Beijing ESWIN Computing Technology Co., Ltd. + */ + +/dts-v1/; + +#include "eic7700.dtsi" + +/ { + compatible = "sifive,hifive-premier-p550", "eswin,eic7700"; + model = "SiFive HiFive Premier P550"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; +}; + +&uart0 { + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/eswin/eic7700.dtsi b/arch/riscv/boot/dts/eswin/eic7700.dtsi new file mode 100644 index 000000000000..c3ed93008bca --- /dev/null +++ b/arch/riscv/boot/dts/eswin/eic7700.dtsi @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2024 Beijing ESWIN Computing Technology Co., Ltd. + */ + +/dts-v1/; + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <1000000>; + + cpu0: cpu@0 { + compatible = "sifive,p550", "riscv"; + device_type = "cpu"; + d-cache-block-size = <64>; + d-cache-sets = <128>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache_0>; + reg = <0x0>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "sscofpmf", + "zba", "zbb", "zicsr", "zifencei"; + tlb-split; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu1: cpu@1 { + compatible = "sifive,p550", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <128>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache_1>; + reg = <0x1>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "sscofpmf", + "zba", "zbb", "zicsr", "zifencei"; + tlb-split; + + cpu1_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu2: cpu@2 { + compatible = "sifive,p550", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <128>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache_2>; + reg = <0x2>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "sscofpmf", + "zba", "zbb", "zicsr", "zifencei"; + tlb-split; + + cpu2_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu3: cpu@3 { + compatible = "sifive,p550", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <128>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache_3>; + reg = <0x3>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "h", "sscofpmf", + "zba", "zbb", "zicsr", "zifencei"; + tlb-split; + + cpu3_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + l2_cache_0: l2-cache0 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <512>; + cache-size = <262144>; + cache-unified; + next-level-cache = <&ccache>; + }; + + l2_cache_1: l2-cache1 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <512>; + cache-size = <262144>; + cache-unified; + next-level-cache = <&ccache>; + }; + + l2_cache_2: l2-cache2 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <512>; + cache-size = <262144>; + cache-unified; + next-level-cache = <&ccache>; + }; + + l2_cache_3: l2-cache3 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <512>; + cache-size = <262144>; + cache-unified; + next-level-cache = <&ccache>; + }; + }; + + pmu { + compatible = "riscv,pmu"; + riscv,event-to-mhpmcounters = + <0x00001 0x00001 0x00000001>, + <0x00002 0x00002 0x00000004>, + <0x00004 0x00006 0x00000078>, + <0x10009 0x10009 0x00000078>, + <0x10019 0x10019 0x00000078>, + <0x10021 0x10021 0x00000078>; + riscv,event-to-mhpmevent = + <0x00004 0x00000000 0x00000202>, + <0x00005 0x00000000 0x00004000>, + <0x00006 0x00000000 0x00002001>, + <0x10009 0x00000000 0x00000102>, + <0x10019 0x00000000 0x00001002>, + <0x10021 0x00000000 0x00000802>; + riscv,raw-event-to-mhpmcounters = + <0x00000000 0x00000000 0xffffffff 0xfc0000ff 0x00000078>, + <0x00000000 0x00000001 0xffffffff 0xfffe07ff 0x00000078>, + <0x00000000 0x00000002 0xffffffff 0xfffe00ff 0x00000078>, + <0x00000000 0x00000003 0xfffffffc 0x000000ff 0x00000078>, + <0x00000000 0x00000004 0xffffffc0 0x000000ff 0x00000078>, + <0x00000000 0x00000005 0xffffffff 0xfffffdff 0x00000078>, + <0x00000000 0x00000006 0xfffffe00 0x110204ff 0x00000078>, + <0x00000000 0x00000007 0xffffffff 0xf00000ff 0x00000078>, + <0x00000000 0x00000008 0xfffffe04 0x000000ff 0x00000078>, + <0x00000000 0x00000009 0xffffffff 0xffffc0ff 0x00000078>, + <0x00000000 0x0000000a 0xffffffff 0xf00000ff 0x00000078>, + <0x00000000 0x0000000b 0xffffffff 0xfffffcff 0x00000078>, + <0x00000000 0x0000000c 0xfffffff0 0x000000ff 0x00000078>, + <0x00000000 0x0000000d 0xffffffff 0x800000ff 0x00000078>, + <0x00000000 0x0000000e 0xffffffff 0xf80000ff 0x00000078>, + <0x00000000 0x0000000f 0xfffffffc 0x000000ff 0x00000078>; + }; + + soc { + compatible = "simple-bus"; + ranges; + interrupt-parent = <&plic>; + #address-cells = <2>; + #size-cells = <2>; + dma-noncoherent; + + clint: timer@2000000 { + compatible = "eswin,eic7700-clint", "sifive,clint0"; + reg = <0x0 0x02000000 0x0 0x10000>; + interrupts-extended = + <&cpu0_intc 3>, <&cpu0_intc 7>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>; + }; + + ccache: cache-controller@2010000 { + compatible = "eswin,eic7700-l3-cache", "sifive,ccache0", "cache"; + reg = <0x0 0x2010000 0x0 0x4000>; + interrupts = <1>, <3>, <4>, <2>; + cache-block-size = <64>; + cache-level = <3>; + cache-sets = <4096>; + cache-size = <4194304>; + cache-unified; + }; + + plic: interrupt-controller@c000000 { + compatible = "eswin,eic7700-plic", "sifive,plic-1.0.0"; + reg = <0x0 0xc000000 0x0 0x4000000>; + interrupt-controller; + interrupts-extended = + <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>; + riscv,ndev = <520>; + #address-cells = <0>; + #interrupt-cells = <1>; + }; + + uart0: serial@50900000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x50900000 0x0 0x10000>; + interrupts = <100>; + clock-frequency = <200000000>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + uart1: serial@50910000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x50910000 0x0 0x10000>; + interrupts = <101>; + clock-frequency = <200000000>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + uart2: serial@50920000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x50920000 0x0 0x10000>; + interrupts = <102>; + clock-frequency = <200000000>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + uart3: serial@50930000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x50930000 0x0 0x10000>; + interrupts = <103>; + clock-frequency = <200000000>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + uart4: serial@50940000 { + compatible = "snps,dw-apb-uart"; + reg = <0x0 0x50940000 0x0 0x10000>; + interrupts = <104>; + clock-frequency = <200000000>; + reg-io-width = <4>; + reg-shift = <2>; + status = "disabled"; + }; + + gpio@51600000 { + compatible = "snps,dw-apb-gpio"; + reg = <0x0 0x51600000 0x0 0x80>; + #address-cells = <1>; + #size-cells = <0>; + + gpioA: gpio-port@0 { + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = + <303>, <304>, <305>, <306>, <307>, <308>, <309>, + <310>, <311>, <312>, <313>, <314>, <315>, <316>, + <317>, <318>, <319>, <320>, <321>, <322>, <323>, + <324>, <325>, <326>, <327>, <328>, <329>, <330>, + <331>, <332>, <333>, <334>; + gpio-controller; + ngpios = <32>; + #gpio-cells = <2>; + }; + + gpioB: gpio-port@1 { + compatible = "snps,dw-apb-gpio-port"; + reg = <1>; + gpio-controller; + ngpios = <32>; + #gpio-cells = <2>; + }; + + gpioC: gpio-port@2 { + compatible = "snps,dw-apb-gpio-port"; + reg = <2>; + gpio-controller; + ngpios = <32>; + #gpio-cells = <2>; + }; + + gpioD: gpio-port@3 { + compatible = "snps,dw-apb-gpio-port"; + reg = <3>; + gpio-controller; + ngpios = <16>; + #gpio-cells = <2>; + }; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile index f51aeeb9fd3b..43f91cbf338b 100644 --- a/arch/riscv/boot/dts/microchip/Makefile +++ b/arch/riscv/boot/dts/microchip/Makefile @@ -1,7 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-beaglev-fire.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb -dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-tysom-m.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-beaglev-fire.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-disco-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-icicle-kit-prod.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-m100pfsevp.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-polarberry.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-sev-kit.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += mpfs-tysom-m.dtb +dtb-$(CONFIG_ARCH_MICROCHIP) += pic64gx-curiosity-kit.dtb diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts index 47cf693beb68..0e1b0b8d394b 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts @@ -79,6 +79,26 @@ }; +&gpio0 { + interrupts = <13>, <14>, <15>, <16>, + <17>, <18>, <19>, <20>, + <21>, <22>, <23>, <24>, + <25>, <26>; + ngpios = <14>; + status = "okay"; +}; + +&gpio1 { + interrupts = <27>, <28>, <29>, <30>, + <31>, <32>, <33>, <34>, + <35>, <36>, <37>, <38>, + <39>, <40>, <41>, <42>, + <43>, <44>, <45>, <46>, + <47>, <48>, <49>, <50>; + ngpios = <24>; + status = "okay"; +}; + &gpio2 { interrupts = <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, @@ -88,7 +108,7 @@ <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>, <53>; - ngpios=<32>; + ngpios = <32>; gpio-line-names = "P8_PIN3_USER_LED_0", "P8_PIN4_USER_LED_1", "P8_PIN5_USER_LED_2", "P8_PIN6_USER_LED_3", "P8_PIN7_USER_LED_4", "P8_PIN8_USER_LED_5", "P8_PIN9_USER_LED_6", "P8_PIN10_USER_LED_7", "P8_PIN11_USER_LED_8", @@ -144,6 +164,35 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &mac0 { status = "okay"; phy-mode = "sgmii"; @@ -199,6 +248,82 @@ status = "okay"; }; +&qspi { + status = "okay"; + cs-gpios = <&gpio1 17 GPIO_ACTIVE_LOW>, <&gpio0 12 GPIO_ACTIVE_LOW>; + num-cs = <2>; + + adc@0 { + compatible = "microchip,mcp3464r"; + reg = <0>; /* CE0 */ + spi-cpol; + spi-cpha; + spi-max-frequency = <5000000>; + microchip,hw-device-address = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + /* CH0 to AGND */ + reg = <0>; + label = "CH0"; + }; + + channel@1 { + /* CH1 to AGND */ + reg = <1>; + label = "CH1"; + }; + + channel@2 { + /* CH2 to AGND */ + reg = <2>; + label = "CH2"; + }; + + channel@3 { + /* CH3 to AGND */ + reg = <3>; + label = "CH3"; + }; + + channel@4 { + /* CH4 to AGND */ + reg = <4>; + label = "CH4"; + }; + + channel@5 { + /* CH5 to AGND */ + reg = <5>; + label = "CH5"; + }; + + channel@6 { + /* CH6 to AGND */ + reg = <6>; + label = "CH6"; + }; + + channel@7 { + /* CH7 is connected to AGND */ + reg = <7>; + label = "CH7"; + }; + }; + + mmc@1 { + compatible = "mmc-spi-slot"; + reg = <1>; + gpios = <&gpio2 31 1>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <5000000>; + disable-wp; + }; +}; + + &syscontroller { microchip,bitstream-flash = <&sys_ctrl_flash>; status = "okay"; diff --git a/arch/riscv/boot/dts/microchip/mpfs-disco-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-disco-kit-fabric.dtsi new file mode 100644 index 000000000000..ae8be7d6f392 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-disco-kit-fabric.dtsi @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020-2025 Microchip Technology Inc */ + +/ { + core_pwm0: pwm@40000000 { + compatible = "microchip,corepwm-rtl-v4"; + reg = <0x0 0x40000000 0x0 0xF0>; + microchip,sync-update-mask = /bits/ 32 <0>; + #pwm-cells = <3>; + clocks = <&ccc_sw CLK_CCC_PLL0_OUT3>; + status = "disabled"; + }; + + i2c2: i2c@40000200 { + compatible = "microchip,corei2c-rtl-v7"; + reg = <0x0 0x40000200 0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&ccc_sw CLK_CCC_PLL0_OUT3>; + interrupt-parent = <&plic>; + interrupts = <122>; + clock-frequency = <100000>; + status = "disabled"; + }; + + ihc: mailbox { + compatible = "microchip,sbi-ipc"; + interrupt-parent = <&plic>; + interrupts = <180>, <179>, <178>, <177>; + interrupt-names = "hart-1", "hart-2", "hart-3", "hart-4"; + #mbox-cells = <1>; + status = "disabled"; + }; + + mailbox@50000000 { + compatible = "microchip,miv-ihc-rtl-v2"; + reg = <0x0 0x50000000 0x0 0x1c000>; + interrupt-parent = <&plic>; + interrupts = <180>, <179>, <178>, <177>; + interrupt-names = "hart-1", "hart-2", "hart-3", "hart-4"; + #mbox-cells = <1>; + microchip,ihc-chan-disabled-mask = /bits/ 16 <0>; + status = "disabled"; + }; + + refclk_ccc: clock-cccref { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; +}; + +&ccc_sw { + clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, + <&refclk_ccc>, <&refclk_ccc>; + clock-names = "pll0_ref0", "pll0_ref1", "pll1_ref0", "pll1_ref1", + "dll0_ref", "dll1_ref"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts new file mode 100644 index 000000000000..f769c9d5d7b4 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020-2025 Microchip Technology Inc */ + +/dts-v1/; + +#include "mpfs.dtsi" +#include "mpfs-disco-kit-fabric.dtsi" +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/leds/common.h> + +/ { + model = "Microchip PolarFire-SoC Discovery Kit"; + compatible = "microchip,mpfs-disco-kit-reference-rtl-v2507", + "microchip,mpfs-disco-kit", + "microchip,mpfs"; + + aliases { + ethernet0 = &mac0; + serial4 = &mmuart4; + }; + + chosen { + stdout-path = "serial4:115200n8"; + }; + + leds { + compatible = "gpio-leds"; + + led-1 { + gpios = <&gpio2 17 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led1"; + }; + + led-2 { + gpios = <&gpio2 18 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led2"; + }; + + led-3 { + gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led3"; + }; + + led-4 { + gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led4"; + }; + + led-5 { + gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led5"; + }; + + led-6 { + gpios = <&gpio2 22 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led6"; + }; + + led-7 { + gpios = <&gpio2 23 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led7"; + }; + + led-8 { + gpios = <&gpio1 9 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led8"; + }; + }; + + ddrc_cache_lo: memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss_payload: region@bfc00000 { + reg = <0x0 0xbfc00000 0x0 0x400000>; + no-map; + }; + }; +}; + +&core_pwm0 { + status = "okay"; +}; + +&gpio1 { + status = "okay"; +}; + +&gpio2 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c2 { + status = "okay"; +}; + +&ihc { + status = "okay"; +}; + +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + +&mac0 { + phy-mode = "sgmii"; + phy-handle = <&phy0>; + status = "okay"; + + phy0: ethernet-phy@b { + reg = <0xb>; + }; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + no-1-8-v; + status = "okay"; +}; + +&mmuart1 { + status = "okay"; +}; + +&mmuart4 { + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&refclk_ccc { + clock-frequency = <50000000>; +}; + +&rtc { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&spi1 { + status = "okay"; +}; + +&syscontroller { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi new file mode 100644 index 000000000000..7816408343a3 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2025 Microchip Technology Inc */ + +/dts-v1/; + +#include "mpfs-icicle-kit-fabric.dtsi" +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/leds/common.h> + +/ { + aliases { + ethernet0 = &mac1; + serial0 = &mmuart0; + serial1 = &mmuart1; + serial2 = &mmuart2; + serial3 = &mmuart3; + serial4 = &mmuart4; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; + + leds { + compatible = "gpio-leds"; + + led-1 { + gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led1"; + }; + + led-2 { + gpios = <&gpio2 17 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_RED>; + label = "led2"; + }; + + led-3 { + gpios = <&gpio2 18 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led3"; + }; + + led-4 { + gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>; + color = <LED_COLOR_ID_AMBER>; + label = "led4"; + }; + }; + + ddrc_cache_lo: memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + + ddrc_cache_hi: memory@1040000000 { + device_type = "memory"; + reg = <0x10 0x40000000 0x0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss_payload: region@bfc00000 { + reg = <0x0 0xbfc00000 0x0 0x400000>; + no-map; + }; + }; +}; + +&core_pwm0 { + status = "okay"; +}; + +&gpio2 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + power-monitor@10 { + compatible = "microchip,pac1934"; + reg = <0x10>; + + #address-cells = <1>; + #size-cells = <0>; + + channel@1 { + reg = <0x1>; + shunt-resistor-micro-ohms = <10000>; + label = "VDDREG"; + }; + + channel@2 { + reg = <0x2>; + shunt-resistor-micro-ohms = <10000>; + label = "VDDA25"; + }; + + channel@3 { + reg = <0x3>; + shunt-resistor-micro-ohms = <10000>; + label = "VDD25"; + }; + + channel@4 { + reg = <0x4>; + shunt-resistor-micro-ohms = <10000>; + label = "VDDA_REG"; + }; + }; +}; + +&i2c2 { + status = "okay"; +}; + +&ihc { + status = "okay"; +}; + +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + +&mac0 { + phy-mode = "sgmii"; + phy-handle = <&phy0>; + status = "okay"; +}; + +&mac1 { + phy-mode = "sgmii"; + phy-handle = <&phy1>; + status = "okay"; + + phy1: ethernet-phy@9 { + reg = <9>; + }; + + phy0: ethernet-phy@8 { + reg = <8>; + }; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + status = "okay"; +}; + +&mmuart1 { + status = "okay"; +}; + +&mmuart2 { + status = "okay"; +}; + +&mmuart3 { + status = "okay"; +}; + +&mmuart4 { + status = "okay"; +}; + +&pcie { + status = "okay"; +}; + +&qspi { + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&refclk_ccc { + clock-frequency = <50000000>; +}; + +&rtc { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&spi1 { + status = "okay"; +}; + +&syscontroller { + status = "okay"; +}; + +&syscontroller_qspi { + /* + * The flash *is* there, but Icicle kits that have engineering sample + * silicon (write?) access to this flash to non-functional. The system + * controller itself can actually access it, but the MSS cannot write + * an image there. Instantiating a coreQSPI in the fabric & connecting + * it to the flash instead should work though. Pre-production or later + * silicon does not have this issue. + */ + status = "disabled"; + + sys_ctrl_flash: flash@0 { // MT25QL01GBBB8ESF-0SIT + compatible = "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <20000000>; + spi-rx-bus-width = <1>; + reg = <0>; + }; +}; + +&usb { + status = "okay"; + dr_mode = "host"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi index 1069134f2e12..9078e5b1e49c 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi @@ -1,10 +1,10 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Copyright (c) 2020-2021 Microchip Technology Inc */ -/ { - compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit", - "microchip,mpfs"; +#include "mpfs.dtsi" +#include "mpfs-pinctrl.dtsi" +/ { core_pwm0: pwm@40000000 { compatible = "microchip,corepwm-rtl-v4"; reg = <0x0 0x40000000 0x0 0xF0>; @@ -26,14 +26,35 @@ status = "disabled"; }; + ihc: mailbox { + compatible = "microchip,sbi-ipc"; + interrupt-parent = <&plic>; + interrupts = <180>, <179>, <178>, <177>; + interrupt-names = "hart-1", "hart-2", "hart-3", "hart-4"; + #mbox-cells = <1>; + status = "disabled"; + }; + + mailbox@50000000 { + compatible = "microchip,miv-ihc-rtl-v2"; + reg = <0x0 0x50000000 0x0 0x1c000>; + interrupt-parent = <&plic>; + interrupts = <180>, <179>, <178>, <177>; + interrupt-names = "hart-1", "hart-2", "hart-3", "hart-4"; + #mbox-cells = <1>; + microchip,ihc-chan-disabled-mask = /bits/ 16 <0>; + status = "disabled"; + }; + pcie: pcie@3000000000 { compatible = "microchip,pcie-host-1.0"; #address-cells = <0x3>; #interrupt-cells = <0x1>; #size-cells = <0x2>; device_type = "pci"; - reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; - reg-names = "cfg", "apb"; + reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43008000 0x0 0x2000>, + <0x0 0x4300a000 0x0 0x2000>; + reg-names = "cfg", "bridge", "ctrl"; bus-range = <0x0 0x7f>; interrupt-parent = <&plic>; interrupts = <119>; @@ -56,12 +77,22 @@ }; }; - refclk_ccc: cccrefclk { + refclk_ccc: clock-cccref { compatible = "fixed-clock"; #clock-cells = <0>; }; }; +&can0 { + pinctrl-names = "default"; + pinctrl-0 = <&can0_fabric>; +}; + +&can1 { + pinctrl-names = "default"; + pinctrl-0 = <&ikrd_can1_cfg>; +}; + &ccc_nw { clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>; @@ -69,3 +100,43 @@ "dll0_ref", "dll1_ref"; status = "okay"; }; + +&mmuart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_fabric>; +}; + +&mmuart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_fabric>; +}; + +&mmuart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_fabric>; +}; + +&mmuart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_fabric>; +}; + +&mssio { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_mssio>, <&can1_mssio>, <&mdio0_mssio>, <&mdio1_mssio>; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_fabric>; +}; + +&spi0 { + pinctrl-names = "default"; + pinctrl-0 = <&spi0_fabric>; +}; + +&spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&ikrd_spi1_cfg>; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts new file mode 100644 index 000000000000..636493f6584d --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2025 Microchip Technology Inc */ + +/dts-v1/; + +#include "mpfs-icicle-kit-common.dtsi" + +/ { + model = "Microchip PolarFire-SoC Icicle Kit (Production Silicon)"; + compatible = "microchip,mpfs-icicle-prod-reference-rtl-v2507", + "microchip,mpfs-icicle-kit-prod", + "microchip,mpfs-icicle-kit", + "microchip,mpfs-prod", + "microchip,mpfs"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_fabric>; +}; + +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_mssio>; +}; + +&syscontroller { + microchip,bitstream-flash = <&sys_ctrl_flash>; +}; + +&syscontroller_qspi { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts index f80df225f72b..6fadce815c9a 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts @@ -3,249 +3,30 @@ /dts-v1/; -#include "mpfs.dtsi" -#include "mpfs-icicle-kit-fabric.dtsi" -#include <dt-bindings/gpio/gpio.h> -#include <dt-bindings/leds/common.h> +#include "mpfs-icicle-kit-common.dtsi" / { model = "Microchip PolarFire-SoC Icicle Kit"; - compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit", + compatible = "microchip,mpfs-icicle-es-reference-rtl-v2507", + "microchip,mpfs-icicle-kit", "microchip,mpfs"; - - aliases { - ethernet0 = &mac1; - serial0 = &mmuart0; - serial1 = &mmuart1; - serial2 = &mmuart2; - serial3 = &mmuart3; - serial4 = &mmuart4; - }; - - chosen { - stdout-path = "serial1:115200n8"; - }; - - leds { - compatible = "gpio-leds"; - - led-1 { - gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>; - color = <LED_COLOR_ID_RED>; - label = "led1"; - }; - - led-2 { - gpios = <&gpio2 17 GPIO_ACTIVE_HIGH>; - color = <LED_COLOR_ID_RED>; - label = "led2"; - }; - - led-3 { - gpios = <&gpio2 18 GPIO_ACTIVE_HIGH>; - color = <LED_COLOR_ID_AMBER>; - label = "led3"; - }; - - led-4 { - gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>; - color = <LED_COLOR_ID_AMBER>; - label = "led4"; - }; - }; - - ddrc_cache_lo: memory@80000000 { - device_type = "memory"; - reg = <0x0 0x80000000 0x0 0x40000000>; - status = "okay"; - }; - - ddrc_cache_hi: memory@1040000000 { - device_type = "memory"; - reg = <0x10 0x40000000 0x0 0x40000000>; - status = "okay"; - }; - - reserved-memory { - #address-cells = <2>; - #size-cells = <2>; - ranges; - - hss_payload: region@BFC00000 { - reg = <0x0 0xBFC00000 0x0 0x400000>; - no-map; - }; - }; -}; - -&core_pwm0 { - status = "okay"; -}; - -&gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; - status = "okay"; }; &i2c0 { - status = "okay"; -}; - + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_fabric>; +}; + +/* + * Due to silicon errata, routing via MSS IOs doesn't work on ES devices. + * Instead, i2c1, appearing on B1/C1, which are normally MSS IOs, is routed + * via the fabric and back to B1/C1 via "fabric-test" functionality. + * This is done silently by Libero, so the iomux0 setting for i2c1 has to + * be fabric IO, despite tooling etc saying that MSS IOs are used. + * + * See Section 3.3 of https://ww1.microchip.com/downloads/aemDocuments/documents/FPGA/ProductDocuments/Errata/polarfiresoc/microsemi_polarfire_soc_fpga_egineering_samples_errata_er0219_v1.pdf + */ &i2c1 { - status = "okay"; - - power-monitor@10 { - compatible = "microchip,pac1934"; - reg = <0x10>; - - #address-cells = <1>; - #size-cells = <0>; - - channel@1 { - reg = <0x1>; - shunt-resistor-micro-ohms = <10000>; - label = "VDDREG"; - }; - - channel@2 { - reg = <0x2>; - shunt-resistor-micro-ohms = <10000>; - label = "VDDA25"; - }; - - channel@3 { - reg = <0x3>; - shunt-resistor-micro-ohms = <10000>; - label = "VDD25"; - }; - - channel@4 { - reg = <0x4>; - shunt-resistor-micro-ohms = <10000>; - label = "VDDA_REG"; - }; - }; -}; - -&i2c2 { - status = "okay"; -}; - -&mac0 { - phy-mode = "sgmii"; - phy-handle = <&phy0>; - status = "okay"; -}; - -&mac1 { - phy-mode = "sgmii"; - phy-handle = <&phy1>; - status = "okay"; - - phy1: ethernet-phy@9 { - reg = <9>; - }; - - phy0: ethernet-phy@8 { - reg = <8>; - }; -}; - -&mbox { - status = "okay"; -}; - -&mmc { - bus-width = <4>; - disable-wp; - cap-sd-highspeed; - cap-mmc-highspeed; - mmc-ddr-1_8v; - mmc-hs200-1_8v; - sd-uhs-sdr12; - sd-uhs-sdr25; - sd-uhs-sdr50; - sd-uhs-sdr104; - status = "okay"; -}; - -&mmuart1 { - status = "okay"; -}; - -&mmuart2 { - status = "okay"; -}; - -&mmuart3 { - status = "okay"; -}; - -&mmuart4 { - status = "okay"; -}; - -&pcie { - status = "okay"; -}; - -&qspi { - status = "okay"; -}; - -&refclk { - clock-frequency = <125000000>; -}; - -&refclk_ccc { - clock-frequency = <50000000>; -}; - -&rtc { - status = "okay"; -}; - -&spi0 { - status = "okay"; -}; - -&spi1 { - status = "okay"; -}; - -&syscontroller { - status = "okay"; -}; - -&syscontroller_qspi { - /* - * The flash *is* there, but Icicle kits that have engineering sample - * silicon (write?) access to this flash to non-functional. The system - * controller itself can actually access it, but the MSS cannot write - * an image there. Instantiating a coreQSPI in the fabric & connecting - * it to the flash instead should work though. Pre-production or later - * silicon does not have this issue. - */ - status = "disabled"; - - sys_ctrl_flash: flash@0 { // MT25QL01GBBB8ESF-0SIT - compatible = "jedec,spi-nor"; - #address-cells = <1>; - #size-cells = <1>; - spi-max-frequency = <20000000>; - spi-rx-bus-width = <1>; - reg = <0>; - }; -}; - -&usb { - status = "okay"; - dr_mode = "host"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_fabric>; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi index 8230f06ddf48..36a9860f31da 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi @@ -20,8 +20,9 @@ #interrupt-cells = <0x1>; #size-cells = <0x2>; device_type = "pci"; - reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; - reg-names = "cfg", "apb"; + reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43008000 0x0 0x2000>, + <0x0 0x4300a000 0x0 0x2000>; + reg-names = "cfg", "bridge", "ctrl"; bus-range = <0x0 0x7f>; interrupt-parent = <&plic>; interrupts = <119>; diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts index a8d623ee9fa4..86234968df48 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts @@ -52,11 +52,36 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio0 { - interrupts = <13>, <14>, <15>, <16>, - <17>, <18>, <19>, <20>, - <21>, <22>, <23>, <24>, - <25>, <26>; ngpios = <14>; status = "okay"; @@ -75,14 +100,6 @@ }; &gpio2 { - interrupts = <13>, <14>, <15>, <16>, - <17>, <18>, <19>, <20>, - <21>, <22>, <23>, <24>, - <25>, <26>, <27>, <28>, - <29>, <30>, <31>, <32>, - <33>, <34>, <35>, <36>, - <37>, <38>, <39>, <40>, - <41>, <42>, <43>, <44>; status = "okay"; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi new file mode 100644 index 000000000000..d37cabc6438a --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-pinctrl.dtsi @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +&iomux0 { + spi0_fabric: mux-spi0-fabric { + function = "spi0"; + groups = "spi0_fabric"; + }; + + spi0_mssio: mux-spi0-mssio { + function = "spi0"; + groups = "spi0_mssio"; + }; + + spi1_fabric: mux-spi1-fabric { + function = "spi1"; + groups = "spi1_fabric"; + }; + + spi1_mssio: mux-spi1-mssio { + function = "spi1"; + groups = "spi1_mssio"; + }; + + i2c0_fabric: mux-i2c0-fabric { + function = "i2c0"; + groups = "i2c0_fabric"; + }; + + i2c0_mssio: mux-i2c0-mssio { + function = "i2c0"; + groups = "i2c0_mssio"; + }; + + i2c1_fabric: mux-i2c1-fabric { + function = "i2c1"; + groups = "i2c1_fabric"; + }; + + i2c1_mssio: mux-i2c1-mssio { + function = "i2c1"; + groups = "i2c1_mssio"; + }; + + can0_fabric: mux-can0-fabric { + function = "can0"; + groups = "can0_fabric"; + }; + + can0_mssio: mux-can0-mssio { + function = "can0"; + groups = "can0_mssio"; + }; + + can1_fabric: mux-can1-fabric { + function = "can1"; + groups = "can1_fabric"; + }; + + can1_mssio: mux-can1-mssio { + function = "can1"; + groups = "can1_mssio"; + }; + + qspi_fabric: mux-qspi-fabric { + function = "qspi"; + groups = "qspi_fabric"; + }; + + qspi_mssio: mux-qspi-mssio { + function = "qspi"; + groups = "qspi_mssio"; + }; + + uart0_fabric: mux-uart0-fabric { + function = "uart0"; + groups = "uart0_fabric"; + }; + + uart0_mssio: mux-uart0-mssio { + function = "uart0"; + groups = "uart0_mssio"; + }; + + uart1_fabric: mux-uart1-fabric { + function = "uart1"; + groups = "uart1_fabric"; + }; + + uart1_mssio: mux-uart1-mssio { + function = "uart1"; + groups = "uart1_mssio"; + }; + + uart2_fabric: mux-uart2-fabric { + function = "uart2"; + groups = "uart2_fabric"; + }; + + uart2_mssio: mux-uart2-mssio { + function = "uart2"; + groups = "uart2_mssio"; + }; + + uart3_fabric: mux-uart3-fabric { + function = "uart3"; + groups = "uart3_fabric"; + }; + + uart3_mssio: mux-uart3-mssio { + function = "uart3"; + groups = "uart3_mssio"; + }; + + uart4_fabric: mux-uart4-fabric { + function = "uart4"; + groups = "uart4_fabric"; + }; + + uart4_mssio: mux-uart4-mssio { + function = "uart4"; + groups = "uart4_mssio"; + }; + + mdio0_fabric: mux-mdio0-fabric { + function = "mdio0"; + groups = "mdio0_fabric"; + }; + + mdio0_mssio: mux-mdio0-mssio { + function = "mdio0"; + groups = "mdio0_mssio"; + }; + + mdio1_fabric: mux-mdio1-fabric { + function = "mdio1"; + groups = "mdio1_fabric"; + }; + + mdio1_mssio: mux-mdio1-mssio { + function = "mdio1"; + groups = "mdio1_mssio"; + }; +}; + +&mssio { + ikrd_can1_cfg: ikrd-can1-cfg { + can1-pins { + pins = <34>, <35>, <36>; + function = "can"; + bias-pull-up; + drive-strength = <8>; + power-source = <3300000>; + microchip,ibufmd = <0x1>; + }; + }; + + ikrd_spi1_cfg: ikrd-spi1-cfg { + spi1-pins { + pins = <30>, <31>, <32>, <33>; + function = "spi"; + bias-pull-up; + drive-strength = <8>; + power-source = <3300000>; + microchip,ibufmd = <0x1>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi index 9a56de7b91d6..a57dca891965 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi @@ -20,8 +20,9 @@ #interrupt-cells = <0x1>; #size-cells = <0x2>; device_type = "pci"; - reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; - reg-names = "cfg", "apb"; + reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43008000 0x0 0x2000>, + <0x0 0x4300a000 0x0 0x2000>; + reg-names = "cfg", "bridge", "ctrl"; bus-range = <0x0 0x7f>; interrupt-parent = <&plic>; interrupts = <119>; diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts index ea0808ab1042..510d59153cd0 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts @@ -30,6 +30,35 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + /* * phy0 is connected to mac0, but the port itself is on the (optional) carrier * board. diff --git a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts index f9a890579438..8f1908a10567 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts @@ -56,15 +56,36 @@ status = "okay"; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio2 { - interrupts = <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>, - <53>, <53>, <53>, <53>; status = "okay"; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts index d1120f5f2c01..bc15530a2979 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts @@ -69,13 +69,36 @@ }; }; +&irqmux { + interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>, + <3 &plic 16>, <4 &plic 17>, <5 &plic 18>, + <6 &plic 19>, <7 &plic 20>, <8 &plic 21>, + <9 &plic 22>, <10 &plic 23>, <11 &plic 24>, + <12 &plic 25>, <13 &plic 26>, + + <32 &plic 27>, <33 &plic 28>, <34 &plic 29>, + <35 &plic 30>, <36 &plic 31>, <37 &plic 32>, + <38 &plic 33>, <39 &plic 34>, <40 &plic 35>, + <41 &plic 36>, <42 &plic 37>, <43 &plic 38>, + <44 &plic 39>, <45 &plic 40>, <46 &plic 41>, + <47 &plic 42>, <48 &plic 43>, <49 &plic 44>, + <50 &plic 45>, <51 &plic 46>, <52 &plic 47>, + <53 &plic 48>, <54 &plic 49>, <55 &plic 50>, + + <64 &plic 53>, <65 &plic 53>, <66 &plic 53>, + <67 &plic 53>, <68 &plic 53>, <69 &plic 53>, + <70 &plic 53>, <71 &plic 53>, <72 &plic 53>, + <73 &plic 53>, <74 &plic 53>, <75 &plic 53>, + <76 &plic 53>, <77 &plic 53>, <78 &plic 53>, + <79 &plic 53>, <80 &plic 53>, <81 &plic 53>, + <82 &plic 53>, <83 &plic 53>, <84 &plic 53>, + <85 &plic 53>, <86 &plic 53>, <87 &plic 53>, + <88 &plic 53>, <89 &plic 53>, <90 &plic 53>, + <91 &plic 53>, <92 &plic 53>, <93 &plic 53>, + <94 &plic 53>, <95 &plic 53>; +}; + &gpio1 { - interrupts = <27>, <28>, <29>, <30>, - <31>, <32>, <33>, <47>, - <35>, <36>, <37>, <38>, - <39>, <40>, <41>, <42>, - <43>, <44>, <45>, <46>, - <47>, <48>, <49>, <50>; status = "okay"; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 9883ca3554c5..d535d4c72763 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -251,12 +251,39 @@ #dma-cells = <1>; }; - clkcfg: clkcfg@20002000 { - compatible = "microchip,mpfs-clkcfg"; - reg = <0x0 0x20002000 0x0 0x1000>, <0x0 0x3E001000 0x0 0x1000>; - clocks = <&refclk>; - #clock-cells = <1>; + mss_top_sysreg: syscon@20002000 { + compatible = "microchip,mpfs-mss-top-sysreg", "syscon", "simple-mfd"; + reg = <0x0 0x20002000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; #reset-cells = <1>; + + irqmux: interrupt-controller@54 { + compatible = "microchip,mpfs-irqmux"; + reg = <0x54 0x4>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-map-mask = <0x7f>; + }; + + iomux0: pinctrl@200 { + compatible = "microchip,mpfs-pinctrl-iomux0"; + reg = <0x200 0x4>; + pinctrl-use-default; + + }; + + mssio: pinctrl@204 { + compatible = "microchip,mpfs-pinctrl-mssio"; + reg = <0x204 0x7c>; + /* on icicle ref design at least */ + pinctrl-use-default; + }; + }; + + sysreg_scb: syscon@20003000 { + compatible = "microchip,mpfs-sysreg-scb", "syscon"; + reg = <0x0 0x20003000 0x0 0x1000>; }; ccc_se: clock-controller@38010000 { @@ -425,6 +452,7 @@ clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <56>; + resets = <&mss_top_sysreg CLK_CAN0>; status = "disabled"; }; @@ -434,6 +462,7 @@ clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <57>; + resets = <&mss_top_sysreg CLK_CAN1>; status = "disabled"; }; @@ -445,9 +474,9 @@ interrupt-parent = <&plic>; interrupts = <64>, <65>, <66>, <67>, <68>, <69>; local-mac-address = [00 00 00 00 00 00]; - clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>; - clock-names = "pclk", "hclk"; - resets = <&clkcfg CLK_MAC0>; + clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC0>; status = "disabled"; }; @@ -459,45 +488,66 @@ interrupt-parent = <&plic>; interrupts = <70>, <71>, <72>, <73>, <74>, <75>; local-mac-address = [00 00 00 00 00 00]; - clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>; - clock-names = "pclk", "hclk"; - resets = <&clkcfg CLK_MAC1>; + clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>, <&refclk>; + clock-names = "pclk", "hclk", "tsu_clk"; + resets = <&mss_top_sysreg CLK_MAC1>; status = "disabled"; }; gpio0: gpio@20120000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20120000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; #interrupt-cells = <1>; + interrupts = <0>, <1>, <2>, <3>, + <4>, <5>, <6>, <7>, + <8>, <9>, <10>, <11>, + <12>, <13>; clocks = <&clkcfg CLK_GPIO0>; gpio-controller; #gpio-cells = <2>; + ngpios = <14>; status = "disabled"; }; gpio1: gpio@20121000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20121000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; #interrupt-cells = <1>; + interrupts = <32>, <33>, <34>, <35>, + <36>, <37>, <38>, <39>, + <40>, <41>, <42>, <43>, + <44>, <45>, <46>, <47>, + <48>, <49>, <50>, <51>, + <52>, <53>, <54>, <55>; clocks = <&clkcfg CLK_GPIO1>; gpio-controller; #gpio-cells = <2>; + ngpios = <24>; status = "disabled"; }; gpio2: gpio@20122000 { compatible = "microchip,mpfs-gpio"; reg = <0x0 0x20122000 0x0 0x1000>; - interrupt-parent = <&plic>; + interrupt-parent = <&irqmux>; interrupt-controller; #interrupt-cells = <1>; + interrupts = <64>, <65>, <66>, <67>, + <68>, <69>, <70>, <71>, + <72>, <73>, <74>, <75>, + <76>, <77>, <78>, <79>, + <80>, <81>, <82>, <83>, + <84>, <85>, <86>, <87>, + <88>, <89>, <90>, <91>, + <92>, <93>, <94>, <95>; clocks = <&clkcfg CLK_GPIO2>; gpio-controller; #gpio-cells = <2>; + ngpios = <32>; status = "disabled"; }; @@ -521,10 +571,14 @@ status = "disabled"; }; - mbox: mailbox@37020000 { + control_scb: syscon@37020000 { + compatible = "microchip,mpfs-control-scb", "syscon"; + reg = <0x0 0x37020000 0x0 0x100>; + }; + + mbox: mailbox@37020800 { compatible = "microchip,mpfs-mailbox"; - reg = <0x0 0x37020000 0x0 0x58>, <0x0 0x2000318C 0x0 0x40>, - <0x0 0x37020800 0x0 0x100>; + reg = <0x0 0x37020800 0x0 0x1000>; interrupt-parent = <&plic>; interrupts = <96>; #mbox-cells = <1>; @@ -541,5 +595,12 @@ clocks = <&scbclk>; status = "disabled"; }; + + clkcfg: clkcfg@3e001000 { + compatible = "microchip,mpfs-clkcfg"; + reg = <0x0 0x3e001000 0x0 0x1000>; + clocks = <&refclk>; + #clock-cells = <1>; + }; }; }; diff --git a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts new file mode 100644 index 000000000000..2f2ccd77af30 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree Source for the PIC64GX Curiosity Kit + * + * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries + * + * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com> + * + * The Curiosity-GX10000 (PIC64GX SoC Curiosity Kit) is a compact SoC + * prototyping board featuring a Microchip PIC64GX SoC + * PIC64GX-1000. Features include: + * - 1 GB DDR4 SDRAM + * - Gigabit Ethernet + * - microSD-card slot + * + * https://www.microchip.com/en-us/development-tool/curiosity-pic64gx1000-kit-es + */ + +/dts-v1/; + +#include "pic64gx.dtsi" +#include "pic64gx-pinctrl.dtsi" + +/* Clock frequency (in Hz) of the rtcclk */ +#define RTCCLK_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Microchip PIC64GX Curiosity Kit"; + compatible = "microchip,pic64gx-curiosity-kit", "microchip,pic64gx"; + + aliases { + ethernet0 = &mac0; + serial1 = &mmuart1; + serial2 = &mmuart2; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; + + cpus { + timebase-frequency = <RTCCLK_FREQ>; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss: hss-buffer@bfc00000 { + compatible = "shared-dma-pool"; + reg = <0x0 0xbfc00000 0x0 0x400000>; + no-map; + }; + }; +}; + +&gpio0 { + interrupts = <13>, <14>, <15>, <16>, + <17>, <18>, <19>, <20>, + <21>, <22>, <23>, <24>, + <25>, <26>; + status ="okay"; + gpio-line-names = + "", "", "", "", "", "", "", "", + "", "", "", "", "MIPI_CAM_RESET", "MIPI_CAM_STANDBY"; +}; + +&gpio1 { + interrupts = <27>, <28>, <29>, <30>, + <31>, <32>, <33>, <34>, + <35>, <36>, <37>, <38>, + <39>, <40>, <41>, <42>, + <43>, <44>, <45>, <46>, + <47>, <48>, <49>, <50>; + status ="okay"; + gpio-line-names = + "", "", "LED1", "LED2", "LED3", "LED4", "LED5", "LED6", + "LED7", "LED8", "", "", "", "", "", "", + "", "", "", "", "HDMI_HPD", "", "", "GPIO_1_23"; +}; + +&gpio2 { + interrupts = <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>; + pinctrl-names = "default"; + pinctrl-0 = <&mdio1_gpio>, <&spi0_gpio>, <&can0_gpio>, <&pcie_gpio>, + <&qspi_gpio>, <&uart3_gpio>, <&uart4_gpio>, <&can1_gpio>; + status ="okay"; + gpio-line-names = + "", "", "", "", "", "", "SWITCH2", "USR_IO12", + "DIP1", "DIP2", "", "DIP3", "USR_IO1", "USR_IO2", "USR_IO7", "USR_IO8", + "USR_IO3", "USR_IO4", "USR_IO5", "USR_IO6", "", "", "USR_IO9", "USR_IO10", + "DIP4", "USR_IO11", "", "", "SWITCH1", "", "", ""; +}; + +&mac0 { + status = "okay"; + phy-mode = "sgmii"; + phy-handle = <&phy0>; + pinctrl-names = "default"; + pinctrl-0 = <&mdio0_default>; + + phy0: ethernet-phy@b { + reg = <0xb>; + }; +}; + +&mbox { + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; +}; + +&mmc { + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + sdhci-caps-mask = <0x00000007 0x00000000>; + status = "okay"; +}; + +&mmuart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_fio>; + status = "okay"; +}; + +&mmuart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_default>; + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&rtc { + status = "okay"; +}; + +&syscontroller { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi new file mode 100644 index 000000000000..cf73b0556633 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx-pinctrl.dtsi @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) + +&iomux0 { + spi0_fio: mux-spi0-fabric { + function = "spi0"; + groups = "spi0_fabric"; + }; + + spi1_mssio: mux-spi1-mssio { + function = "spi1"; + groups = "spi1_mssio"; + }; + + i2c0_mssio: mux-i2c0-mssio { + function = "i2c0"; + groups = "i2c0_mssio"; + }; + + i2c1_mssio: mux-i2c1-mssio { + function = "i2c1"; + groups = "i2c1_mssio"; + }; + + can0_fio: mux-can0-fabric { + function = "can0"; + groups = "can0_fabric"; + }; + + can1_fio: mux-can1-fabric { + function = "can1"; + groups = "can1_fabric"; + }; + + qspi_fio: mux-qspi-fabric { + function = "qspi"; + groups = "qspi_fabric"; + }; + + uart0_mssio: mux-uart0-mssio { + function = "uart0"; + groups = "uart0_mssio"; + }; + + uart1_fio: mux-uart1-fabric { + function = "uart1"; + groups = "uart1_fabric"; + }; + + uart2_fio: mux-uart2-fabric { + function = "uart2"; + groups = "uart2_fabric"; + }; + + uart3_fio: mux-uart3-fabric { + function = "uart3"; + groups = "uart3_fabric"; + }; + + uart4_fio: mux-uart4-fabric { + function = "uart4"; + groups = "uart4_fabric"; + }; + + mdio0_fio: mux-mdio0-fabric { + function = "mdio0"; + groups = "mdio0_fabric"; + }; + + mdio1_fio: mux-mdio1-fabric { + function = "mdio1"; + groups = "mdio1_fabric"; + }; +}; + +&gpio2_pinctrl { + //TODO rethink the labels, since a bunch of these are not defaults or + //just outright remove the non-default groups + mdio0_default: mux-mac0 { + function = "mdio0"; + groups = "mdio0"; + }; + + mdio0_gpio: mux-mac0-gpio2 { + function = "gpio"; + groups = "gpio_mdio0"; + }; + + mdio1_default: mux-mac1 { + function = "mdio1"; + groups = "mdio1"; + }; + + mdio1_gpio: mux-mac1-gpio2 { + function = "gpio"; + groups = "gpio_mdio1"; + }; + + spi0_default: mux-spi0 { + function = "spi0"; + groups = "spi0"; + }; + + spi0_gpio: mux-spi0-gpio2 { + function = "gpio"; + groups = "gpio_spi0"; + }; + + can0_default: mux-can0 { + function = "can0"; + groups = "can0"; + }; + + can0_gpio: mux-can0-gpio2 { + function = "gpio"; + groups = "gpio_can0"; + }; + + pcie_default: mux-pcie { + function = "pcie"; + groups = "pcie"; + }; + + pcie_gpio: mux-pcie-gpio2 { + function = "gpio"; + groups = "gpio_pcie"; + }; + + qspi_default: mux-qspi { + function = "qspi"; + groups = "qspi"; + }; + + qspi_gpio: mux-qspi-gpio2 { + function = "gpio"; + groups = "gpio_qspi"; + }; + + uart3_default: mux-uart3 { + function = "uart3"; + groups = "uart3"; + }; + + uart3_gpio: mux-uart3-gpio2 { + function = "gpio"; + groups = "gpio_uart3"; + }; + + uart4_default: mux-uart4 { + function = "uart4"; + groups = "uart4"; + }; + + uart4_gpio: mux-uart4-gpio2 { + function = "gpio"; + groups = "gpio_uart4"; + }; + + can1_default: mux-can1 { + function = "can1"; + groups = "can1"; + }; + + can1_gpio: mux-can1-gpio2 { + function = "gpio"; + groups = "gpio_can1"; + }; + + uart2_default: mux-uart2 { + function = "uart2"; + groups = "uart2"; + }; + + uart2_gpio: mux-uart2-gpio2 { + function = "gpio"; + groups = "gpio_uart2"; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/pic64gx.dtsi b/arch/riscv/boot/dts/microchip/pic64gx.dtsi new file mode 100644 index 000000000000..c164d7bc270a --- /dev/null +++ b/arch/riscv/boot/dts/microchip/pic64gx.dtsi @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree Source for the PIC64GX SoCs + * + * Copyright (C) 2024 Microchip Technology Inc. and its subsidiaries + * + * Author: Pierre-Henry Moussay <pierre-henry.moussay@microchip.com> + * + * PIC64GX is a series RISC-V multicore SoCs: + * https://www.microchip.com/en-us/products/microprocessors/64-bit-mpus/pic64gx + */ + +/dts-v1/; +#include "dt-bindings/clock/microchip,mpfs-clock.h" + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Microchip PIC64GX SoC"; + compatible = "microchip,pic64gx"; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "sifive,e51", "sifive,rocket0", "riscv"; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <16384>; + reg = <0>; + riscv,isa = "rv64imac"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "c", "zicntr", "zicsr", + "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + status = "disabled"; + + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu1: cpu@1 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <1>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu2: cpu@2 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <2>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu2_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu3: cpu@3 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <3>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + + cpu3_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu4: cpu@4 { + compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <32>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <64>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <32>; + mmu-type = "riscv,sv39"; + reg = <4>; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", + "zicsr", "zifencei", "zihpm"; + clocks = <&clkcfg CLK_CPU>; + tlb-split; + next-level-cache = <&cctrllr>; + status = "okay"; + cpu4_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + + core4 { + cpu = <&cpu4>; + }; + }; + }; + }; + + scbclk: clock-80000000 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <80000000>; + }; + + refclk: mssrefclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; + + axiclk: axi-aclk0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + videoclk: video-aclk0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + syscontroller: syscontroller { + compatible = "microchip,pic64gx-sys-controller"; + mboxes = <&mbox 0>; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + + clint: clint@2000000 { + compatible = "microchip,pic64gx-clint", "sifive,clint0"; + reg = <0x0 0x2000000 0x0 0xC000>; + interrupts-extended = <&cpu0_intc 0xffffffff>, <&cpu0_intc 0xffffffff>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>, + <&cpu4_intc 3>, <&cpu4_intc 7>; + }; + + cctrllr: cache-controller@2010000 { + compatible = "microchip,pic64gx-ccache", "microchip,mpfs-ccache", + "sifive,fu540-c000-ccache", "cache"; + reg = <0x0 0x2010000 0x0 0x1000>; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <1024>; + cache-size = <2097152>; + cache-unified; + interrupt-parent = <&plic>; + interrupts = <1>, <3>, <4>, <2>; + }; + + pdma: dma-controller@3000000 { + compatible = "microchip,pic64gx-pdma", "microchip,mpfs-pdma", + "sifive,pdma0"; + reg = <0x0 0x3000000 0x0 0x8000>; + interrupt-parent = <&plic>; + interrupts = <5 6>, <7 8>, <9 10>, <11 12>; + #dma-cells = <1>; + }; + + plic: interrupt-controller@c000000 { + compatible = "microchip,pic64gx-plic", "sifive,plic-1.0.0"; + reg = <0x0 0xc000000 0x0 0x4000000>; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 0xffffffff>, + <&cpu1_intc 0xffffffff>, <&cpu1_intc 9>, + <&cpu2_intc 0xffffffff>, <&cpu2_intc 9>, + <&cpu3_intc 0xffffffff>, <&cpu3_intc 9>, + <&cpu4_intc 0xffffffff>, <&cpu4_intc 9>; + riscv,ndev = <186>; + }; + + mmuart0: serial@20000000 { + compatible = "ns16550a"; + reg = <0x0 0x20000000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <90>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART0>; + pinctrl-names = "default"; + pinctrl-0 = <&uart0_mssio>; + status = "disabled"; /* Reserved for the HSS */ + }; + + mss_top_sysreg: syscon@20002000 { + compatible = "microchip,pic64gx-mss-top-sysreg", + "microchip,mpfs-mss-top-sysreg", + "syscon", "simple-mfd"; + reg = <0x0 0x20002000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + #reset-cells = <1>; + + iomux0: pinctrl@200 { + compatible = "microchip,pic64gx-pinctrl-iomux0", + "microchip,mpfs-pinctrl-iomux0"; + reg = <0x200 0x4>; + pinctrl-use-default; + }; + + mssio: pinctrl@204 { + compatible = "microchip,pic64gx-pinctrl-mssio", + "microchip,mpfs-pinctrl-mssio"; + reg = <0x204 0x7c>; + /* on icicle ref design at least */ + pinctrl-use-default; + }; + }; + + sysreg_scb: syscon@20003000 { + compatible = "microchip,pic64gx-sysreg-scb", + "microchip,mpfs-sysreg-scb", + "syscon"; + reg = <0x0 0x20003000 0x0 0x1000>; + }; + + /* Common node entry for emmc/sd */ + mmc: mmc@20008000 { + compatible = "microchip,pic64gx-sd4hc", "cdns,sd4hc"; + reg = <0x0 0x20008000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <88>; + clocks = <&clkcfg CLK_MMC>; + max-frequency = <200000000>; + status = "disabled"; + }; + + mmuart1: serial@20100000 { + compatible = "ns16550a"; + reg = <0x0 0x20100000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <91>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART1>; + status = "disabled"; + }; + + mmuart2: serial@20102000 { + compatible = "ns16550a"; + reg = <0x0 0x20102000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <92>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART2>; + status = "disabled"; + }; + + mmuart3: serial@20104000 { + compatible = "ns16550a"; + reg = <0x0 0x20104000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <93>; + current-speed = <115200>; + clocks = <&clkcfg CLK_MMUART3>; + status = "disabled"; + }; + + mmuart4: serial@20106000 { + compatible = "ns16550a"; + reg = <0x0 0x20106000 0x0 0x400>; + reg-io-width = <4>; + reg-shift = <2>; + interrupt-parent = <&plic>; + interrupts = <94>; + clocks = <&clkcfg CLK_MMUART4>; + current-speed = <115200>; + status = "disabled"; + }; + + spi0: spi@20108000 { + compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x20108000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <54>; + clocks = <&clkcfg CLK_SPI0>; + status = "disabled"; + }; + + spi1: spi@20109000 { + compatible = "microchip,pic64gx-spi", "microchip,mpfs-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x20109000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <55>; + clocks = <&clkcfg CLK_SPI1>; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_mssio>; + status = "disabled"; + }; + + i2c0: i2c@2010a000 { + compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7"; + reg = <0x0 0x2010a000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <58>; + clocks = <&clkcfg CLK_I2C0>; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_mssio>; + status = "disabled"; + }; + + i2c1: i2c@2010b000 { + compatible = "microchip,pic64gx-i2c", "microchip,corei2c-rtl-v7"; + reg = <0x0 0x2010b000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <61>; + clocks = <&clkcfg CLK_I2C1>; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_mssio>; + status = "disabled"; + }; + + can0: can@2010c000 { + compatible = "microchip,pic64gx-can", "microchip,mpfs-can"; + reg = <0x0 0x2010c000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>; + interrupt-parent = <&plic>; + interrupts = <56>; + resets = <&mss_top_sysreg CLK_CAN0>; + status = "disabled"; + }; + + can1: can@2010d000 { + compatible = "microchip,pic64gx-can", "microchip,mpfs-can"; + reg = <0x0 0x2010d000 0x0 0x1000>; + clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>; + interrupt-parent = <&plic>; + interrupts = <57>; + resets = <&mss_top_sysreg CLK_CAN1>; + status = "disabled"; + }; + + mac0: ethernet@20110000 { + compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb", + "cdns,macb"; + reg = <0x0 0x20110000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <64>, <65>, <66>, <67>, <68>, <69>; + /* Filled in by a bootloader */ + local-mac-address = [00 00 00 00 00 00]; + clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>; + clock-names = "pclk", "hclk"; + resets = <&mss_top_sysreg CLK_MAC0>; + status = "disabled"; + }; + + mac1: ethernet@20112000 { + compatible = "microchip,pic64gx-macb", "microchip,mpfs-macb", + "cdns,macb"; + reg = <0x0 0x20112000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <0>; + interrupt-parent = <&plic>; + interrupts = <70>, <71>, <72>, <73>, <74>, <75>; + /* Filled in by a bootloader */ + local-mac-address = [00 00 00 00 00 00]; + clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>; + clock-names = "pclk", "hclk"; + resets = <&mss_top_sysreg CLK_MAC1>; + status = "disabled"; + }; + + gpio0: gpio@20120000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20120000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupt-controller; + #interrupt-cells = <1>; + clocks = <&clkcfg CLK_GPIO0>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <14>; + status = "disabled"; + }; + + gpio1: gpio@20121000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20121000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupt-controller; + #interrupt-cells = <1>; + clocks = <&clkcfg CLK_GPIO1>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <24>; + status = "disabled"; + }; + + gpio2: gpio@20122000 { + compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio"; + reg = <0x0 0x20122000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupt-controller; + #interrupt-cells = <1>; + clocks = <&clkcfg CLK_GPIO2>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + status = "disabled"; + }; + + rtc: rtc@20124000 { + compatible = "microchip,pic64gx-rtc", "microchip,mpfs-rtc"; + reg = <0x0 0x20124000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <80>, <81>; + clocks = <&clkcfg CLK_RTC>, <&clkcfg CLK_RTCREF>; + clock-names = "rtc", "rtcref"; + status = "disabled"; + }; + + usb: usb@20201000 { + compatible = "microchip,pic64gx-musb", "microchip,mpfs-musb"; + reg = <0x0 0x20201000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <86>, <87>; + clocks = <&clkcfg CLK_USB>; + interrupt-names = "dma", "mc"; + status = "disabled"; + }; + + qspi: spi@21000000 { + compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x21000000 0x0 0x1000>; + interrupt-parent = <&plic>; + interrupts = <85>; + clocks = <&clkcfg CLK_QSPI>; + status = "disabled"; + }; + + control_scb: syscon@37020000 { + compatible = "microchip,pic64gx-control-scb", + "microchip,mpfs-control-scb", + "syscon"; + reg = <0x0 0x37020000 0x0 0x100>; + }; + + syscontroller_qspi: spi@37020100 { + compatible = "microchip,pic64gx-qspi", "microchip,coreqspi-rtl-v2"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0x37020100 0x0 0x100>; + interrupt-parent = <&plic>; + interrupts = <110>; + clocks = <&scbclk>; + status = "disabled"; + }; + + mbox: mailbox@37020800 { + compatible = "microchip,pic64gx-mailbox", "microchip,mpfs-mailbox"; + reg = <0x0 0x37020800 0x0 0x100>; + interrupt-parent = <&plic>; + interrupts = <96>; + #mbox-cells = <1>; + status = "disabled"; + }; + + ccc_se: clock-controller@38010000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38010000 0x0 0x1000>, <0x0 0x38020000 0x0 0x1000>, + <0x0 0x39010000 0x0 0x1000>, <0x0 0x39020000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_ne: clock-controller@38040000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38040000 0x0 0x1000>, <0x0 0x38080000 0x0 0x1000>, + <0x0 0x39040000 0x0 0x1000>, <0x0 0x39080000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_nw: clock-controller@38100000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38100000 0x0 0x1000>, <0x0 0x38200000 0x0 0x1000>, + <0x0 0x39100000 0x0 0x1000>, <0x0 0x39200000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + ccc_sw: clock-controller@38400000 { + compatible = "microchip,pic64gx-ccc", "microchip,mpfs-ccc"; + reg = <0x0 0x38400000 0x0 0x1000>, <0x0 0x38800000 0x0 0x1000>, + <0x0 0x39400000 0x0 0x1000>, <0x0 0x39800000 0x0 0x1000>; + #clock-cells = <1>; + status = "disabled"; + }; + + clkcfg: clkcfg@3e001000 { + compatible = "microchip,pic64gx-clkcfg", "microchip,mpfs-clkcfg"; + reg = <0x0 0x3e001000 0x0 0x1000>; + clocks = <&refclk>; + #clock-cells = <1>; + }; + + gpio2_pinctrl: pinctrl@41000000 { + compatible = "microchip,pic64gx-pinctrl-gpio2"; + reg = <0x0 0x41000000 0x0 0x4>; + pinctrl-use-default; + pinctrl-names = "default"; + pinctrl-0 = <&mdio0_fio>, <&mdio1_fio>, <&spi0_fio>, <&qspi_fio>, + <&uart3_fio>, <&uart4_fio>, <&can1_fio>, <&can0_fio>, + <&uart2_fio>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi index e0ddf8f602c7..571de3cafa82 100644 --- a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi +++ b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi @@ -12,6 +12,8 @@ #include <arm64/renesas/r9a07g043.dtsi> / { + interrupt-parent = <&plic>; + cpus { #address-cells = <1>; #size-cells = <0>; @@ -52,7 +54,6 @@ &soc { dma-noncoherent; - interrupt-parent = <&plic>; irqc: interrupt-controller@110a0000 { compatible = "renesas,r9a07g043f-irqc"; @@ -143,7 +144,8 @@ }; l2cache: cache-controller@13400000 { - compatible = "andestech,ax45mp-cache", "cache"; + compatible = "renesas,r9a07g043f-ax45mp-cache", "andestech,ax45mp-cache", + "cache"; reg = <0x0 0x13400000 0x0 0x100000>; interrupts = <SOC_PERIPHERAL_IRQ(476) IRQ_TYPE_LEVEL_HIGH>; cache-size = <0x40000>; diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts index 900a50526d77..06731b8c7bc3 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts @@ -49,32 +49,28 @@ compatible = "pwm-leds"; led-d1 { - pwms = <&pwm0 0 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 0 7812500 0>; color = <LED_COLOR_ID_GREEN>; max-brightness = <255>; label = "d1"; }; led-d2 { - pwms = <&pwm0 1 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 1 7812500 0>; color = <LED_COLOR_ID_GREEN>; max-brightness = <255>; label = "d2"; }; led-d3 { - pwms = <&pwm0 2 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 2 7812500 0>; color = <LED_COLOR_ID_GREEN>; max-brightness = <255>; label = "d3"; }; led-d4 { - pwms = <&pwm0 3 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 3 7812500 0>; color = <LED_COLOR_ID_GREEN>; max-brightness = <255>; label = "d4"; diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts index 72b87b08ab44..850fa1d25be7 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts @@ -47,12 +47,21 @@ gpios = <&gpio 2 GPIO_ACTIVE_LOW>; }; + fan1 { + compatible = "pwm-fan"; + pwms = <&pwm1 2 7812500 0>; + }; + + fan2 { + compatible = "pwm-fan"; + pwms = <&pwm1 3 7812500 0>; + }; + led-controller-1 { compatible = "pwm-leds"; led-d12 { - pwms = <&pwm0 0 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 0 7812500 0>; color = <LED_COLOR_ID_GREEN>; max-brightness = <255>; label = "d12"; @@ -68,20 +77,17 @@ label = "d2"; led-red { - pwms = <&pwm0 2 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 2 7812500 0>; color = <LED_COLOR_ID_RED>; }; led-green { - pwms = <&pwm0 1 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 1 7812500 0>; color = <LED_COLOR_ID_GREEN>; }; led-blue { - pwms = <&pwm0 3 7812500 PWM_POLARITY_INVERTED>; - active-low; + pwms = <&pwm0 3 7812500 0>; color = <LED_COLOR_ID_BLUE>; }; }; diff --git a/arch/riscv/boot/dts/sophgo/Makefile b/arch/riscv/boot/dts/sophgo/Makefile index 47d4243a8f35..6f65526d4193 100644 --- a/arch/riscv/boot/dts/sophgo/Makefile +++ b/arch/riscv/boot/dts/sophgo/Makefile @@ -3,3 +3,6 @@ dtb-$(CONFIG_ARCH_SOPHGO) += cv1800b-milkv-duo.dtb dtb-$(CONFIG_ARCH_SOPHGO) += cv1812h-huashan-pi.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2002-licheerv-nano-b.dtb dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-milkv-pioneer.dtb +dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-evb-v1.dtb +dtb-$(CONFIG_ARCH_SOPHGO) += sg2042-evb-v2.dtb +dtb-$(CONFIG_ARCH_SOPHGO) += sg2044-sophgo-srd3-10.dtb diff --git a/arch/riscv/boot/dts/sophgo/cv1800b-milkv-duo.dts b/arch/riscv/boot/dts/sophgo/cv1800b-milkv-duo.dts index 9feb520eaec4..0e6d79e6e3a4 100644 --- a/arch/riscv/boot/dts/sophgo/cv1800b-milkv-duo.dts +++ b/arch/riscv/boot/dts/sophgo/cv1800b-milkv-duo.dts @@ -100,3 +100,8 @@ pinctrl-names = "default"; status = "okay"; }; + +&usb { + dr_mode = "host"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/cv1800b.dtsi b/arch/riscv/boot/dts/sophgo/cv1800b.dtsi index aa1f5df100f0..90de978f69c1 100644 --- a/arch/riscv/boot/dts/sophgo/cv1800b.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv1800b.dtsi @@ -3,8 +3,11 @@ * Copyright (C) 2023 Jisheng Zhang <jszhang@kernel.org> */ +#define SOC_PERIPHERAL_IRQ(nr) ((nr) + 16) + #include <dt-bindings/pinctrl/pinctrl-cv1800b.h> -#include "cv18xx.dtsi" +#include "cv180x-cpus.dtsi" +#include "cv180x.dtsi" / { compatible = "sophgo,cv1800b"; @@ -15,23 +18,37 @@ }; soc { + interrupt-parent = <&plic>; + dma-noncoherent; + pinctrl: pinctrl@3001000 { compatible = "sophgo,cv1800b-pinctrl"; reg = <0x03001000 0x1000>, <0x05027000 0x1000>; reg-names = "sys", "rtc"; }; - }; -}; -&plic { - compatible = "sophgo,cv1800b-plic", "thead,c900-plic"; -}; + clk: clock-controller@3002000 { + compatible = "sophgo,cv1800b-clk"; + reg = <0x03002000 0x1000>; + clocks = <&osc>; + #clock-cells = <1>; + }; -&clint { - compatible = "sophgo,cv1800b-clint", "thead,c900-clint"; -}; + plic: interrupt-controller@70000000 { + compatible = "sophgo,cv1800b-plic", "thead,c900-plic"; + reg = <0x70000000 0x4000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <2>; + riscv,ndev = <101>; + }; -&clk { - compatible = "sophgo,cv1800-clk"; + clint: timer@74000000 { + compatible = "sophgo,cv1800b-clint", "thead,c900-clint"; + reg = <0x74000000 0x10000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + }; + }; }; diff --git a/arch/riscv/boot/dts/sophgo/cv180x-cpus.dtsi b/arch/riscv/boot/dts/sophgo/cv180x-cpus.dtsi new file mode 100644 index 000000000000..93fd9e47a195 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/cv180x-cpus.dtsi @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2023 Jisheng Zhang <jszhang@kernel.org> + * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> + */ + +/ { + cpus: cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <25000000>; + + cpu0: cpu@0 { + compatible = "thead,c906", "riscv"; + device_type = "cpu"; + reg = <0>; + d-cache-block-size = <64>; + d-cache-sets = <512>; + d-cache-size = <65536>; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + mmu-type = "riscv,sv39"; + riscv,isa = "rv64imafdc"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", + "zifencei", "zihpm"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + }; +}; diff --git a/arch/riscv/boot/dts/sophgo/cv18xx.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index c18822ec849f..06b0ce5a2db7 100644 --- a/arch/riscv/boot/dts/sophgo/cv18xx.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -7,40 +7,12 @@ #include <dt-bindings/clock/sophgo,cv1800.h> #include <dt-bindings/gpio/gpio.h> #include <dt-bindings/interrupt-controller/irq.h> +#include "cv18xx-reset.h" / { #address-cells = <1>; #size-cells = <1>; - cpus: cpus { - #address-cells = <1>; - #size-cells = <0>; - timebase-frequency = <25000000>; - - cpu0: cpu@0 { - compatible = "thead,c906", "riscv"; - device_type = "cpu"; - reg = <0>; - d-cache-block-size = <64>; - d-cache-sets = <512>; - d-cache-size = <65536>; - i-cache-block-size = <64>; - i-cache-sets = <128>; - i-cache-size = <32768>; - mmu-type = "riscv,sv39"; - riscv,isa = "rv64imafdc"; - riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", - "zifencei", "zihpm"; - - cpu0_intc: interrupt-controller { - compatible = "riscv,cpu-intc"; - interrupt-controller; - #interrupt-cells = <1>; - }; - }; - }; - osc: oscillator { compatible = "fixed-clock"; clock-output-names = "osc_25m"; @@ -49,16 +21,67 @@ soc { compatible = "simple-bus"; - interrupt-parent = <&plic>; #address-cells = <1>; #size-cells = <1>; - dma-noncoherent; ranges; - clk: clock-controller@3002000 { - reg = <0x03002000 0x1000>; - clocks = <&osc>; - #clock-cells = <1>; + syscon: syscon@3000000 { + compatible = "sophgo,cv1800b-top-syscon", + "syscon", "simple-mfd"; + reg = <0x03000000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + usbphy: phy@48 { + compatible = "sophgo,cv1800b-usb2-phy"; + reg = <0x48 0x4>; + #phy-cells = <0>; + clocks = <&clk CLK_USB_125M>, + <&clk CLK_USB_33K>, + <&clk CLK_USB_12M>; + clock-names = "app", "stb", "lpm"; + resets = <&rst RST_COMBO_PHY0>; + }; + + dmamux: dma-router@154 { + compatible = "sophgo,cv1800b-dmamux"; + reg = <0x154 0x8>, <0x298 0x4>; + #dma-cells = <2>; + dma-masters = <&dmac>; + }; + }; + + rst: reset-controller@3003000 { + compatible = "sophgo,cv1800b-reset"; + reg = <0x3003000 0x1000>; + #reset-cells = <1>; + }; + + mdio: mdio-mux@3009800 { + compatible = "mdio-mux-mmioreg", "mdio-mux"; + reg = <0x3009800 0x4>; + #address-cells = <1>; + #size-cells = <0>; + mdio-parent-bus = <&gmac0_mdio>; + mux-mask = <0x80>; + status = "disabled"; + + internal_mdio: mdio@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + internal_ephy: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + }; + }; + + external_mdio: mdio@80 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x80>; + }; }; gpio0: gpio@3020000 { @@ -66,6 +89,7 @@ reg = <0x3020000 0x1000>; #address-cells = <1>; #size-cells = <0>; + resets = <&rst RST_GPIO0>; porta: gpio-controller@0 { compatible = "snps,dw-apb-gpio-port"; @@ -75,7 +99,7 @@ reg = <0>; interrupt-controller; #interrupt-cells = <2>; - interrupts = <60 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(44) IRQ_TYPE_LEVEL_HIGH>; }; }; @@ -84,6 +108,7 @@ reg = <0x3021000 0x1000>; #address-cells = <1>; #size-cells = <0>; + resets = <&rst RST_GPIO1>; portb: gpio-controller@0 { compatible = "snps,dw-apb-gpio-port"; @@ -93,7 +118,7 @@ reg = <0>; interrupt-controller; #interrupt-cells = <2>; - interrupts = <61 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(45) IRQ_TYPE_LEVEL_HIGH>; }; }; @@ -102,6 +127,7 @@ reg = <0x3022000 0x1000>; #address-cells = <1>; #size-cells = <0>; + resets = <&rst RST_GPIO2>; portc: gpio-controller@0 { compatible = "snps,dw-apb-gpio-port"; @@ -111,7 +137,7 @@ reg = <0>; interrupt-controller; #interrupt-cells = <2>; - interrupts = <62 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(46) IRQ_TYPE_LEVEL_HIGH>; }; }; @@ -120,6 +146,7 @@ reg = <0x3023000 0x1000>; #address-cells = <1>; #size-cells = <0>; + resets = <&rst RST_GPIO3>; portd: gpio-controller@0 { compatible = "snps,dw-apb-gpio-port"; @@ -129,7 +156,7 @@ reg = <0>; interrupt-controller; #interrupt-cells = <2>; - interrupts = <63 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(47) IRQ_TYPE_LEVEL_HIGH>; }; }; @@ -137,7 +164,7 @@ compatible = "sophgo,cv1800b-saradc"; reg = <0x030f0000 0x1000>; clocks = <&clk CLK_SARADC>; - interrupts = <100 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(84) IRQ_TYPE_LEVEL_HIGH>; #address-cells = <1>; #size-cells = <0>; status = "disabled"; @@ -162,7 +189,8 @@ #size-cells = <0>; clocks = <&clk CLK_I2C>, <&clk CLK_APB_I2C0>; clock-names = "ref", "pclk"; - interrupts = <49 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(33) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C0>; status = "disabled"; }; @@ -173,7 +201,8 @@ #size-cells = <0>; clocks = <&clk CLK_I2C>, <&clk CLK_APB_I2C1>; clock-names = "ref", "pclk"; - interrupts = <50 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(34) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C1>; status = "disabled"; }; @@ -184,7 +213,8 @@ #size-cells = <0>; clocks = <&clk CLK_I2C>, <&clk CLK_APB_I2C2>; clock-names = "ref", "pclk"; - interrupts = <51 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(35) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C2>; status = "disabled"; }; @@ -195,7 +225,8 @@ #size-cells = <0>; clocks = <&clk CLK_I2C>, <&clk CLK_APB_I2C3>; clock-names = "ref", "pclk"; - interrupts = <52 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(36) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C3>; status = "disabled"; }; @@ -206,51 +237,102 @@ #size-cells = <0>; clocks = <&clk CLK_I2C>, <&clk CLK_APB_I2C4>; clock-names = "ref", "pclk"; - interrupts = <53 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(37) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C4>; status = "disabled"; }; + gmac0: ethernet@4070000 { + compatible = "sophgo,cv1800b-dwmac", "snps,dwmac-3.70a"; + reg = <0x04070000 0x10000>; + clocks = <&clk CLK_AXI4_ETH0>, <&clk CLK_ETH0_500M>; + clock-names = "stmmaceth", "ptp_ref"; + interrupts = <SOC_PERIPHERAL_IRQ(15) IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq"; + phy-handle = <&internal_ephy>; + phy-mode = "internal"; + resets = <&rst RST_ETH0>; + reset-names = "stmmaceth"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + snps,multicast-filter-bins = <0>; + snps,perfect-filter-entries = <1>; + snps,aal; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; + snps,axi-config = <&gmac0_stmmac_axi_setup>; + status = "disabled"; + + gmac0_mdio: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac0_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + queue0 {}; + }; + + gmac0_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + queue0 {}; + }; + + gmac0_stmmac_axi_setup: stmmac-axi-config { + snps,blen = <16 8 4 0 0 0 0>; + snps,rd_osr_lmt = <2>; + snps,wr_osr_lmt = <1>; + }; + }; + uart0: serial@4140000 { compatible = "snps,dw-apb-uart"; reg = <0x04140000 0x100>; - interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(28) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_UART0>, <&clk CLK_APB_UART0>; clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; + resets = <&rst RST_UART0>; status = "disabled"; }; uart1: serial@4150000 { compatible = "snps,dw-apb-uart"; reg = <0x04150000 0x100>; - interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(29) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_UART1>, <&clk CLK_APB_UART1>; clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; + resets = <&rst RST_UART1>; status = "disabled"; }; uart2: serial@4160000 { compatible = "snps,dw-apb-uart"; reg = <0x04160000 0x100>; - interrupts = <46 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(30) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_UART2>, <&clk CLK_APB_UART2>; clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; + resets = <&rst RST_UART2>; status = "disabled"; }; uart3: serial@4170000 { compatible = "snps,dw-apb-uart"; reg = <0x04170000 0x100>; - interrupts = <47 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(31) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_UART3>, <&clk CLK_APB_UART3>; clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; + resets = <&rst RST_UART3>; status = "disabled"; }; @@ -261,7 +343,8 @@ #size-cells = <0>; clocks = <&clk CLK_SPI>, <&clk CLK_APB_SPI0>; clock-names = "ssi_clk", "pclk"; - interrupts = <54 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(38) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPI0>; status = "disabled"; }; @@ -272,7 +355,8 @@ #size-cells = <0>; clocks = <&clk CLK_SPI>, <&clk CLK_APB_SPI1>; clock-names = "ssi_clk", "pclk"; - interrupts = <55 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(39) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPI1>; status = "disabled"; }; @@ -283,7 +367,8 @@ #size-cells = <0>; clocks = <&clk CLK_SPI>, <&clk CLK_APB_SPI2>; clock-names = "ssi_clk", "pclk"; - interrupts = <56 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(40) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPI2>; status = "disabled"; }; @@ -294,25 +379,27 @@ #size-cells = <0>; clocks = <&clk CLK_SPI>, <&clk CLK_APB_SPI3>; clock-names = "ssi_clk", "pclk"; - interrupts = <57 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(41) IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPI3>; status = "disabled"; }; uart4: serial@41c0000 { compatible = "snps,dw-apb-uart"; reg = <0x041c0000 0x100>; - interrupts = <48 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(32) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_UART4>, <&clk CLK_APB_UART4>; clock-names = "baudclk", "apb_pclk"; reg-shift = <2>; reg-io-width = <4>; + resets = <&rst RST_UART4>; status = "disabled"; }; sdhci0: mmc@4310000 { compatible = "sophgo,cv1800b-dwcmshc"; reg = <0x4310000 0x1000>; - interrupts = <36 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(20) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_AXI4_SD0>, <&clk CLK_SD0>; clock-names = "core", "bus"; @@ -322,7 +409,7 @@ sdhci1: mmc@4320000 { compatible = "sophgo,cv1800b-dwcmshc"; reg = <0x4320000 0x1000>; - interrupts = <38 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(22) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_AXI4_SD1>, <&clk CLK_SD1>; clock-names = "core", "bus"; @@ -332,7 +419,7 @@ dmac: dma-controller@4330000 { compatible = "snps,axi-dma-1.01a"; reg = <0x04330000 0x1000>; - interrupts = <29 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(13) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>; clock-names = "core-clk", "cfgr-clk"; #dma-cells = <1>; @@ -341,22 +428,36 @@ 1024 1024 1024 1024>; snps,priority = <0 1 2 3 4 5 6 7>; snps,dma-masters = <2>; - snps,data-width = <4>; + snps,data-width = <2>; status = "disabled"; }; - plic: interrupt-controller@70000000 { - reg = <0x70000000 0x4000000>; - interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; - interrupt-controller; - #address-cells = <0>; - #interrupt-cells = <2>; - riscv,ndev = <101>; + usb: usb@4340000 { + compatible = "sophgo,cv1800b-usb"; + reg = <0x04340000 0x10000>; + clocks = <&clk CLK_AXI4_USB>, <&clk CLK_APB_USB>; + clock-names = "otg", "utmi"; + g-np-tx-fifo-size = <32>; + g-rx-fifo-size = <1536>; + g-tx-fifo-size = <128 128 64 64 64 64 32 32>; + interrupts = <SOC_PERIPHERAL_IRQ(14) IRQ_TYPE_LEVEL_HIGH>; + phys = <&usbphy>; + phy-names = "usb2-phy"; + resets = <&rst RST_USB>; + reset-names = "dwc2"; + status = "disabled"; }; - clint: timer@74000000 { - reg = <0x74000000 0x10000>; - interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + rtc@5025000 { + compatible = "sophgo,cv1800b-rtc", "syscon"; + reg = <0x5025000 0x2000>; + interrupts = <SOC_PERIPHERAL_IRQ(1) IRQ_TYPE_LEVEL_HIGH>, + <SOC_PERIPHERAL_IRQ(2) IRQ_TYPE_LEVEL_HIGH>, + <SOC_PERIPHERAL_IRQ(3) IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "alarm", "longpress", "vbat"; + clocks = <&clk CLK_RTC_25M>, + <&clk CLK_SRC_RTC_SYS_0>; + clock-names = "rtc", "mcu"; }; }; }; diff --git a/arch/riscv/boot/dts/sophgo/cv1812h-huashan-pi.dts b/arch/riscv/boot/dts/sophgo/cv1812h-huashan-pi.dts index 26b57e15adc1..aedf79f47407 100644 --- a/arch/riscv/boot/dts/sophgo/cv1812h-huashan-pi.dts +++ b/arch/riscv/boot/dts/sophgo/cv1812h-huashan-pi.dts @@ -55,6 +55,14 @@ non-removable; }; +&gmac0 { + status = "okay"; +}; + +&mdio { + status = "okay"; +}; + &sdhci0 { status = "okay"; bus-width = <4>; @@ -78,3 +86,8 @@ &uart0 { status = "okay"; }; + +&usb { + dr_mode = "host"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/cv1812h.dtsi b/arch/riscv/boot/dts/sophgo/cv1812h.dtsi index 8a1b95c5116b..9a2a314d3347 100644 --- a/arch/riscv/boot/dts/sophgo/cv1812h.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv1812h.dtsi @@ -3,9 +3,12 @@ * Copyright (C) 2023 Inochi Amaoto <inochiama@outlook.com> */ +#define SOC_PERIPHERAL_IRQ(nr) ((nr) + 16) + #include <dt-bindings/interrupt-controller/irq.h> #include <dt-bindings/pinctrl/pinctrl-cv1812h.h> -#include "cv18xx.dtsi" +#include "cv180x-cpus.dtsi" +#include "cv180x.dtsi" #include "cv181x.dtsi" / { @@ -17,23 +20,37 @@ }; soc { + interrupt-parent = <&plic>; + dma-noncoherent; + pinctrl: pinctrl@3001000 { compatible = "sophgo,cv1812h-pinctrl"; reg = <0x03001000 0x1000>, <0x05027000 0x1000>; reg-names = "sys", "rtc"; }; - }; -}; -&plic { - compatible = "sophgo,cv1812h-plic", "thead,c900-plic"; -}; + clk: clock-controller@3002000 { + compatible = "sophgo,cv1812h-clk"; + reg = <0x03002000 0x1000>; + clocks = <&osc>; + #clock-cells = <1>; + }; -&clint { - compatible = "sophgo,cv1812h-clint", "thead,c900-clint"; -}; + plic: interrupt-controller@70000000 { + compatible = "sophgo,cv1812h-plic", "thead,c900-plic"; + reg = <0x70000000 0x4000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <2>; + riscv,ndev = <101>; + }; -&clk { - compatible = "sophgo,cv1810-clk"; + clint: timer@74000000 { + compatible = "sophgo,cv1812h-clint", "thead,c900-clint"; + reg = <0x74000000 0x10000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + }; + }; }; diff --git a/arch/riscv/boot/dts/sophgo/cv181x.dtsi b/arch/riscv/boot/dts/sophgo/cv181x.dtsi index 5fd14dd1b14f..bbdb30653e9a 100644 --- a/arch/riscv/boot/dts/sophgo/cv181x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv181x.dtsi @@ -11,7 +11,7 @@ emmc: mmc@4300000 { compatible = "sophgo,cv1800b-dwcmshc"; reg = <0x4300000 0x1000>; - interrupts = <34 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <SOC_PERIPHERAL_IRQ(18) IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk CLK_AXI4_EMMC>, <&clk CLK_EMMC>; clock-names = "core", "bus"; diff --git a/arch/riscv/boot/dts/sophgo/cv18xx-reset.h b/arch/riscv/boot/dts/sophgo/cv18xx-reset.h new file mode 100644 index 000000000000..7e7c5ca2dbbd --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/cv18xx-reset.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@outlook.com> + */ + +#ifndef _SOPHGO_CV18XX_RESET +#define _SOPHGO_CV18XX_RESET + +#define RST_DDR 2 +#define RST_H264C 3 +#define RST_JPEG 4 +#define RST_H265C 5 +#define RST_VIPSYS 6 +#define RST_TDMA 7 +#define RST_TPU 8 +#define RST_TPUSYS 9 +#define RST_USB 11 +#define RST_ETH0 12 +#define RST_ETH1 13 +#define RST_NAND 14 +#define RST_EMMC 15 +#define RST_SD0 16 +#define RST_SDMA 18 +#define RST_I2S0 19 +#define RST_I2S1 20 +#define RST_I2S2 21 +#define RST_I2S3 22 +#define RST_UART0 23 +#define RST_UART1 24 +#define RST_UART2 25 +#define RST_UART3 26 +#define RST_I2C0 27 +#define RST_I2C1 28 +#define RST_I2C2 29 +#define RST_I2C3 30 +#define RST_I2C4 31 +#define RST_PWM0 32 +#define RST_PWM1 33 +#define RST_PWM2 34 +#define RST_PWM3 35 +#define RST_SPI0 40 +#define RST_SPI1 41 +#define RST_SPI2 42 +#define RST_SPI3 43 +#define RST_GPIO0 44 +#define RST_GPIO1 45 +#define RST_GPIO2 46 +#define RST_EFUSE 47 +#define RST_WDT 48 +#define RST_AHB_ROM 49 +#define RST_SPIC 50 +#define RST_TEMPSEN 51 +#define RST_SARADC 52 +#define RST_COMBO_PHY0 58 +#define RST_SPI_NAND 61 +#define RST_SE 62 +#define RST_UART4 74 +#define RST_GPIO3 75 +#define RST_SYSTEM 76 +#define RST_TIMER 77 +#define RST_TIMER0 78 +#define RST_TIMER1 79 +#define RST_TIMER2 80 +#define RST_TIMER3 81 +#define RST_TIMER4 82 +#define RST_TIMER5 83 +#define RST_TIMER6 84 +#define RST_TIMER7 85 +#define RST_WGN0 86 +#define RST_WGN1 87 +#define RST_WGN2 88 +#define RST_KEYSCAN 89 +#define RST_AUDDAC 91 +#define RST_AUDDAC_APB 92 +#define RST_AUDADC 93 +#define RST_VCSYS 95 +#define RST_ETHPHY 96 +#define RST_ETHPHY_APB 97 +#define RST_AUDSRC 98 +#define RST_VIP_CAM0 99 +#define RST_WDT1 100 +#define RST_WDT2 101 +#define RST_AUTOCLEAR_CPUCORE0 256 +#define RST_AUTOCLEAR_CPUCORE1 257 +#define RST_AUTOCLEAR_CPUCORE2 258 +#define RST_AUTOCLEAR_CPUCORE3 259 +#define RST_AUTOCLEAR_CPUSYS0 260 +#define RST_AUTOCLEAR_CPUSYS1 261 +#define RST_AUTOCLEAR_CPUSYS2 262 +#define RST_CPUCORE0 288 +#define RST_CPUCORE1 289 +#define RST_CPUCORE2 290 +#define RST_CPUCORE3 291 +#define RST_CPUSYS0 292 +#define RST_CPUSYS1 293 +#define RST_CPUSYS2 294 + +#endif /* _SOPHGO_CV18XX_RESET */ diff --git a/arch/riscv/boot/dts/sophgo/sg2002-licheerv-nano-b.dts b/arch/riscv/boot/dts/sophgo/sg2002-licheerv-nano-b.dts index 86a712b953a5..b1853770d017 100644 --- a/arch/riscv/boot/dts/sophgo/sg2002-licheerv-nano-b.dts +++ b/arch/riscv/boot/dts/sophgo/sg2002-licheerv-nano-b.dts @@ -93,3 +93,8 @@ pinctrl-names = "default"; status = "okay"; }; + +&usb { + dr_mode = "host"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2002.dtsi b/arch/riscv/boot/dts/sophgo/sg2002.dtsi index 7f79de33163c..98001cce238e 100644 --- a/arch/riscv/boot/dts/sophgo/sg2002.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2002.dtsi @@ -3,9 +3,12 @@ * Copyright (C) 2024 Thomas Bonnefille <thomas.bonnefille@bootlin.com> */ +#define SOC_PERIPHERAL_IRQ(nr) ((nr) + 16) + #include <dt-bindings/interrupt-controller/irq.h> #include <dt-bindings/pinctrl/pinctrl-sg2002.h> -#include "cv18xx.dtsi" +#include "cv180x-cpus.dtsi" +#include "cv180x.dtsi" #include "cv181x.dtsi" / { @@ -17,25 +20,39 @@ }; soc { + interrupt-parent = <&plic>; + dma-noncoherent; + pinctrl: pinctrl@3001000 { compatible = "sophgo,sg2002-pinctrl"; reg = <0x03001000 0x1000>, <0x05027000 0x1000>; reg-names = "sys", "rtc"; }; - }; -}; -&plic { - compatible = "sophgo,sg2002-plic", "thead,c900-plic"; -}; + clk: clock-controller@3002000 { + compatible = "sophgo,sg2002-clk", "sophgo,sg2000-clk"; + reg = <0x03002000 0x1000>; + clocks = <&osc>; + #clock-cells = <1>; + }; -&clint { - compatible = "sophgo,sg2002-clint", "thead,c900-clint"; -}; + plic: interrupt-controller@70000000 { + compatible = "sophgo,sg2002-plic", "thead,c900-plic"; + reg = <0x70000000 0x4000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>; + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <2>; + riscv,ndev = <101>; + }; -&clk { - compatible = "sophgo,sg2000-clk"; + clint: timer@74000000 { + compatible = "sophgo,sg2002-clint", "thead,c900-clint"; + reg = <0x74000000 0x10000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>; + }; + }; }; &sdhci0 { diff --git a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi index b136b6c4128c..509488eee432 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042-cpus.dtsi @@ -259,8 +259,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <0>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -270,6 +272,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache0>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu0_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -284,8 +287,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <1>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -295,6 +300,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache0>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu1_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -309,8 +315,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <2>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -320,6 +328,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache0>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu2_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -334,8 +343,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <3>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -345,6 +356,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache0>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu3_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -359,8 +371,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <4>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -370,6 +384,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache1>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu4_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -384,8 +399,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <5>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -395,6 +412,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache1>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu5_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -409,8 +427,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <6>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -420,6 +440,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache1>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu6_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -434,8 +455,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <7>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -445,6 +468,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache1>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu7_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -459,8 +483,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <8>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -470,6 +496,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache4>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu8_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -484,8 +511,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <9>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -495,6 +524,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache4>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu9_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -509,8 +539,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <10>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -520,6 +552,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache4>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu10_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -534,8 +567,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <11>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -545,6 +580,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache4>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu11_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -559,8 +595,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <12>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -570,6 +608,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache5>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu12_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -584,8 +623,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <13>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -595,6 +636,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache5>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu13_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -609,8 +651,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <14>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -620,6 +664,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache5>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu14_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -634,8 +679,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <15>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -645,6 +692,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache5>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu15_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -659,8 +707,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <16>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -670,6 +720,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache2>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu16_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -684,8 +735,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <17>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -695,6 +748,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache2>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu17_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -709,8 +763,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <18>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -720,6 +776,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache2>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu18_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -734,8 +791,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <19>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -745,6 +804,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache2>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu19_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -759,8 +819,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <20>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -770,6 +832,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache3>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu20_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -784,8 +847,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <21>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -795,6 +860,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache3>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu21_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -809,8 +875,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <22>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -820,6 +888,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache3>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu22_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -834,8 +903,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <23>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -845,6 +916,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache3>; mmu-type = "riscv,sv39"; + numa-node-id = <0>; cpu23_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -859,8 +931,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <24>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -870,6 +944,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache6>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu24_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -884,8 +959,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <25>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -895,6 +972,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache6>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu25_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -909,8 +987,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <26>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -920,6 +1000,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache6>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu26_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -934,8 +1015,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <27>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -945,6 +1028,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache6>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu27_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -959,8 +1043,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <28>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -970,6 +1056,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache7>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu28_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -984,8 +1071,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <29>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -995,6 +1084,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache7>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu29_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1009,8 +1099,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <30>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1020,6 +1112,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache7>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu30_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1034,8 +1127,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <31>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1045,6 +1140,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache7>; mmu-type = "riscv,sv39"; + numa-node-id = <1>; cpu31_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1059,8 +1155,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <32>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1070,6 +1168,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache8>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu32_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1084,8 +1183,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <33>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1095,6 +1196,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache8>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu33_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1109,8 +1211,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <34>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1120,6 +1224,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache8>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu34_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1134,8 +1239,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <35>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1145,6 +1252,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache8>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu35_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1159,8 +1267,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <36>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1170,6 +1280,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache9>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu36_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1184,8 +1295,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <37>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1195,6 +1308,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache9>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu37_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1209,8 +1323,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <38>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1220,6 +1336,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache9>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu38_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1234,8 +1351,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <39>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1245,6 +1364,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache9>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu39_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1259,8 +1379,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <40>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1270,6 +1392,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache12>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu40_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1284,8 +1407,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <41>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1295,6 +1420,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache12>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu41_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1309,8 +1435,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <42>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1320,6 +1448,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache12>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu42_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1334,8 +1463,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <43>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1345,6 +1476,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache12>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu43_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1359,8 +1491,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <44>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1370,6 +1504,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache13>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu44_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1384,8 +1519,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <45>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1395,6 +1532,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache13>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu45_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1409,8 +1547,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <46>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1420,6 +1560,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache13>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu46_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1434,8 +1575,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <47>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1445,6 +1588,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache13>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu47_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1459,8 +1603,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <48>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1470,6 +1616,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache10>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu48_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1484,8 +1631,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <49>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1495,6 +1644,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache10>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu49_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1509,8 +1659,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <50>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1520,6 +1672,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache10>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu50_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1534,8 +1687,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <51>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1545,6 +1700,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache10>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu51_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1559,8 +1715,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <52>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1570,6 +1728,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache11>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu52_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1584,8 +1743,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <53>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1595,6 +1756,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache11>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu53_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1609,8 +1771,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <54>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1620,6 +1784,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache11>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu54_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1634,8 +1799,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <55>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1645,6 +1812,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache11>; mmu-type = "riscv,sv39"; + numa-node-id = <2>; cpu55_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1659,8 +1827,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <56>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1670,6 +1840,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache14>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu56_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1684,8 +1855,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <57>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1695,6 +1868,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache14>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu57_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1709,8 +1883,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <58>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1720,6 +1896,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache14>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu58_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1734,8 +1911,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <59>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1745,6 +1924,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache14>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu59_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1759,8 +1939,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <60>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1770,6 +1952,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache15>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu60_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1784,8 +1967,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <61>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1795,6 +1980,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache15>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu61_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1809,8 +1995,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <62>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1820,6 +2008,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache15>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu62_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1834,8 +2023,10 @@ riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; riscv,isa-extensions = "i", "m", "a", "f", "d", "c", - "zicntr", "zicsr", "zifencei", - "zihpm"; + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <63>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -1845,6 +2036,7 @@ d-cache-sets = <512>; next-level-cache = <&l2_cache15>; mmu-type = "riscv,sv39"; + numa-node-id = <3>; cpu63_intc: interrupt-controller { compatible = "riscv,cpu-intc"; @@ -1997,4 +2189,309 @@ cache-unified; }; }; + + soc { + intc: interrupt-controller@7090000000 { + compatible = "sophgo,sg2042-plic", "thead,c900-plic"; + #address-cells = <0>; + #interrupt-cells = <2>; + reg = <0x00000070 0x90000000 0x00000000 0x04000000>; + interrupt-controller; + interrupts-extended = + <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>, + <&cpu4_intc 11>, <&cpu4_intc 9>, + <&cpu5_intc 11>, <&cpu5_intc 9>, + <&cpu6_intc 11>, <&cpu6_intc 9>, + <&cpu7_intc 11>, <&cpu7_intc 9>, + <&cpu8_intc 11>, <&cpu8_intc 9>, + <&cpu9_intc 11>, <&cpu9_intc 9>, + <&cpu10_intc 11>, <&cpu10_intc 9>, + <&cpu11_intc 11>, <&cpu11_intc 9>, + <&cpu12_intc 11>, <&cpu12_intc 9>, + <&cpu13_intc 11>, <&cpu13_intc 9>, + <&cpu14_intc 11>, <&cpu14_intc 9>, + <&cpu15_intc 11>, <&cpu15_intc 9>, + <&cpu16_intc 11>, <&cpu16_intc 9>, + <&cpu17_intc 11>, <&cpu17_intc 9>, + <&cpu18_intc 11>, <&cpu18_intc 9>, + <&cpu19_intc 11>, <&cpu19_intc 9>, + <&cpu20_intc 11>, <&cpu20_intc 9>, + <&cpu21_intc 11>, <&cpu21_intc 9>, + <&cpu22_intc 11>, <&cpu22_intc 9>, + <&cpu23_intc 11>, <&cpu23_intc 9>, + <&cpu24_intc 11>, <&cpu24_intc 9>, + <&cpu25_intc 11>, <&cpu25_intc 9>, + <&cpu26_intc 11>, <&cpu26_intc 9>, + <&cpu27_intc 11>, <&cpu27_intc 9>, + <&cpu28_intc 11>, <&cpu28_intc 9>, + <&cpu29_intc 11>, <&cpu29_intc 9>, + <&cpu30_intc 11>, <&cpu30_intc 9>, + <&cpu31_intc 11>, <&cpu31_intc 9>, + <&cpu32_intc 11>, <&cpu32_intc 9>, + <&cpu33_intc 11>, <&cpu33_intc 9>, + <&cpu34_intc 11>, <&cpu34_intc 9>, + <&cpu35_intc 11>, <&cpu35_intc 9>, + <&cpu36_intc 11>, <&cpu36_intc 9>, + <&cpu37_intc 11>, <&cpu37_intc 9>, + <&cpu38_intc 11>, <&cpu38_intc 9>, + <&cpu39_intc 11>, <&cpu39_intc 9>, + <&cpu40_intc 11>, <&cpu40_intc 9>, + <&cpu41_intc 11>, <&cpu41_intc 9>, + <&cpu42_intc 11>, <&cpu42_intc 9>, + <&cpu43_intc 11>, <&cpu43_intc 9>, + <&cpu44_intc 11>, <&cpu44_intc 9>, + <&cpu45_intc 11>, <&cpu45_intc 9>, + <&cpu46_intc 11>, <&cpu46_intc 9>, + <&cpu47_intc 11>, <&cpu47_intc 9>, + <&cpu48_intc 11>, <&cpu48_intc 9>, + <&cpu49_intc 11>, <&cpu49_intc 9>, + <&cpu50_intc 11>, <&cpu50_intc 9>, + <&cpu51_intc 11>, <&cpu51_intc 9>, + <&cpu52_intc 11>, <&cpu52_intc 9>, + <&cpu53_intc 11>, <&cpu53_intc 9>, + <&cpu54_intc 11>, <&cpu54_intc 9>, + <&cpu55_intc 11>, <&cpu55_intc 9>, + <&cpu56_intc 11>, <&cpu56_intc 9>, + <&cpu57_intc 11>, <&cpu57_intc 9>, + <&cpu58_intc 11>, <&cpu58_intc 9>, + <&cpu59_intc 11>, <&cpu59_intc 9>, + <&cpu60_intc 11>, <&cpu60_intc 9>, + <&cpu61_intc 11>, <&cpu61_intc 9>, + <&cpu62_intc 11>, <&cpu62_intc 9>, + <&cpu63_intc 11>, <&cpu63_intc 9>; + riscv,ndev = <224>; + }; + + clint_mswi: interrupt-controller@7094000000 { + compatible = "sophgo,sg2042-aclint-mswi", "thead,c900-aclint-mswi"; + reg = <0x00000070 0x94000000 0x00000000 0x00004000>; + interrupts-extended = <&cpu0_intc 3>, + <&cpu1_intc 3>, + <&cpu2_intc 3>, + <&cpu3_intc 3>, + <&cpu4_intc 3>, + <&cpu5_intc 3>, + <&cpu6_intc 3>, + <&cpu7_intc 3>, + <&cpu8_intc 3>, + <&cpu9_intc 3>, + <&cpu10_intc 3>, + <&cpu11_intc 3>, + <&cpu12_intc 3>, + <&cpu13_intc 3>, + <&cpu14_intc 3>, + <&cpu15_intc 3>, + <&cpu16_intc 3>, + <&cpu17_intc 3>, + <&cpu18_intc 3>, + <&cpu19_intc 3>, + <&cpu20_intc 3>, + <&cpu21_intc 3>, + <&cpu22_intc 3>, + <&cpu23_intc 3>, + <&cpu24_intc 3>, + <&cpu25_intc 3>, + <&cpu26_intc 3>, + <&cpu27_intc 3>, + <&cpu28_intc 3>, + <&cpu29_intc 3>, + <&cpu30_intc 3>, + <&cpu31_intc 3>, + <&cpu32_intc 3>, + <&cpu33_intc 3>, + <&cpu34_intc 3>, + <&cpu35_intc 3>, + <&cpu36_intc 3>, + <&cpu37_intc 3>, + <&cpu38_intc 3>, + <&cpu39_intc 3>, + <&cpu40_intc 3>, + <&cpu41_intc 3>, + <&cpu42_intc 3>, + <&cpu43_intc 3>, + <&cpu44_intc 3>, + <&cpu45_intc 3>, + <&cpu46_intc 3>, + <&cpu47_intc 3>, + <&cpu48_intc 3>, + <&cpu49_intc 3>, + <&cpu50_intc 3>, + <&cpu51_intc 3>, + <&cpu52_intc 3>, + <&cpu53_intc 3>, + <&cpu54_intc 3>, + <&cpu55_intc 3>, + <&cpu56_intc 3>, + <&cpu57_intc 3>, + <&cpu58_intc 3>, + <&cpu59_intc 3>, + <&cpu60_intc 3>, + <&cpu61_intc 3>, + <&cpu62_intc 3>, + <&cpu63_intc 3>; + }; + + clint_mtimer0: timer@70ac004000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac004000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu0_intc 7>, + <&cpu1_intc 7>, + <&cpu2_intc 7>, + <&cpu3_intc 7>; + }; + + clint_mtimer1: timer@70ac014000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac014000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu4_intc 7>, + <&cpu5_intc 7>, + <&cpu6_intc 7>, + <&cpu7_intc 7>; + }; + + clint_mtimer2: timer@70ac024000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac024000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu8_intc 7>, + <&cpu9_intc 7>, + <&cpu10_intc 7>, + <&cpu11_intc 7>; + }; + + clint_mtimer3: timer@70ac034000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac034000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu12_intc 7>, + <&cpu13_intc 7>, + <&cpu14_intc 7>, + <&cpu15_intc 7>; + }; + + clint_mtimer4: timer@70ac044000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac044000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu16_intc 7>, + <&cpu17_intc 7>, + <&cpu18_intc 7>, + <&cpu19_intc 7>; + }; + + clint_mtimer5: timer@70ac054000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac054000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu20_intc 7>, + <&cpu21_intc 7>, + <&cpu22_intc 7>, + <&cpu23_intc 7>; + }; + + clint_mtimer6: timer@70ac064000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac064000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu24_intc 7>, + <&cpu25_intc 7>, + <&cpu26_intc 7>, + <&cpu27_intc 7>; + }; + + clint_mtimer7: timer@70ac074000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac074000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu28_intc 7>, + <&cpu29_intc 7>, + <&cpu30_intc 7>, + <&cpu31_intc 7>; + }; + + clint_mtimer8: timer@70ac084000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac084000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu32_intc 7>, + <&cpu33_intc 7>, + <&cpu34_intc 7>, + <&cpu35_intc 7>; + }; + + clint_mtimer9: timer@70ac094000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac094000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu36_intc 7>, + <&cpu37_intc 7>, + <&cpu38_intc 7>, + <&cpu39_intc 7>; + }; + + clint_mtimer10: timer@70ac0a4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0a4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu40_intc 7>, + <&cpu41_intc 7>, + <&cpu42_intc 7>, + <&cpu43_intc 7>; + }; + + clint_mtimer11: timer@70ac0b4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0b4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu44_intc 7>, + <&cpu45_intc 7>, + <&cpu46_intc 7>, + <&cpu47_intc 7>; + }; + + clint_mtimer12: timer@70ac0c4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0c4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu48_intc 7>, + <&cpu49_intc 7>, + <&cpu50_intc 7>, + <&cpu51_intc 7>; + }; + + clint_mtimer13: timer@70ac0d4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0d4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu52_intc 7>, + <&cpu53_intc 7>, + <&cpu54_intc 7>, + <&cpu55_intc 7>; + }; + + clint_mtimer14: timer@70ac0e4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0e4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu56_intc 7>, + <&cpu57_intc 7>, + <&cpu58_intc 7>, + <&cpu59_intc 7>; + }; + + clint_mtimer15: timer@70ac0f4000 { + compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x00000070 0xac0f4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu60_intc 7>, + <&cpu61_intc 7>, + <&cpu62_intc 7>, + <&cpu63_intc 7>; + }; + }; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-evb-v1.dts b/arch/riscv/boot/dts/sophgo/sg2042-evb-v1.dts new file mode 100644 index 000000000000..b116dfa904cd --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2042-evb-v1.dts @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Sophgo Technology Inc. All rights reserved. + */ + +#include "sg2042.dtsi" + +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/input/input.h> + +/ { + model = "Sophgo SG2042 EVB V1.X"; + compatible = "sophgo,sg2042-evb-v1", "sophgo,sg2042"; + + chosen { + stdout-path = "serial0"; + }; + + gpio-power { + compatible = "gpio-keys"; + + key-power { + label = "Power Key"; + linux,code = <KEY_POWER>; + gpios = <&port0a 22 GPIO_ACTIVE_HIGH>; + linux,input-type = <EV_KEY>; + debounce-interval = <100>; + }; + }; + + pwmfan: pwm-fan { + compatible = "pwm-fan"; + cooling-levels = <103 128 179 230 255>; + pwms = <&pwm 0 40000 0>; + #cooling-cells = <2>; + }; + + thermal-zones { + soc-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + thermal-sensors = <&mcu 0>; + + trips { + soc_active1: soc-active1 { + temperature = <30000>; + hysteresis = <8000>; + type = "active"; + }; + + soc_active2: soc-active2 { + temperature = <58000>; + hysteresis = <12000>; + type = "active"; + }; + + soc_active3: soc-active3 { + temperature = <70000>; + hysteresis = <10000>; + type = "active"; + }; + + soc_hot: soc-hot { + temperature = <80000>; + hysteresis = <5000>; + type = "hot"; + }; + }; + + cooling-maps { + map0 { + trip = <&soc_active1>; + cooling-device = <&pwmfan 0 1>; + }; + + map1 { + trip = <&soc_active2>; + cooling-device = <&pwmfan 1 2>; + }; + + map2 { + trip = <&soc_active3>; + cooling-device = <&pwmfan 2 3>; + }; + + map3 { + trip = <&soc_hot>; + cooling-device = <&pwmfan 3 4>; + }; + }; + }; + + board-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + thermal-sensors = <&mcu 1>; + + trips { + board_active: board-active { + temperature = <75000>; + hysteresis = <8000>; + type = "active"; + }; + }; + + cooling-maps { + map4 { + trip = <&board_active>; + cooling-device = <&pwmfan 3 4>; + }; + }; + }; + }; +}; + +&cgi_main { + clock-frequency = <25000000>; +}; + +&cgi_dpll0 { + clock-frequency = <25000000>; +}; + +&cgi_dpll1 { + clock-frequency = <25000000>; +}; + +&emmc { + pinctrl-0 = <&emmc_cfg>; + pinctrl-names = "default"; + bus-width = <4>; + no-sdio; + no-sd; + non-removable; + wp-inverted; + status = "okay"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_cfg>; + pinctrl-names = "default"; + status = "okay"; + + mcu: syscon@17 { + compatible = "sophgo,sg2042-hwmon-mcu"; + reg = <0x17>; + #thermal-sensor-cells = <1>; + }; +}; + +&gmac0 { + phy-handle = <&phy0>; + phy-mode = "rgmii-id"; + status = "okay"; + + mdio { + phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + reset-gpios = <&port0a 27 GPIO_ACTIVE_LOW>; + reset-assert-us = <100000>; + reset-deassert-us = <100000>; + }; + }; +}; + +&pcie_rc0 { + status = "okay"; +}; + +&pcie_rc1 { + status = "okay"; +}; + +&pcie_rc2 { + status = "okay"; +}; + +&pinctrl { + emmc_cfg: sdhci-emmc-cfg { + sdhci-emmc-wp-pins { + pinmux = <PINMUX(PIN_EMMC_WP, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + + sdhci-emmc-cd-pins { + pinmux = <PINMUX(PIN_EMMC_CD, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-emmc-rst-pwr-pins { + pinmux = <PINMUX(PIN_EMMC_RST, 0)>, + <PINMUX(PIN_EMMC_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + i2c1_cfg: i2c1-cfg { + i2c1-pins { + pinmux = <PINMUX(PIN_IIC1_SDA, 0)>, + <PINMUX(PIN_IIC1_SCL, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; + + sd_cfg: sdhci-sd-cfg { + sdhci-sd-cd-wp-pins { + pinmux = <PINMUX(PIN_SDIO_CD, 0)>, + <PINMUX(PIN_SDIO_WP, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-sd-rst-pwr-pins { + pinmux = <PINMUX(PIN_SDIO_RST, 0)>, + <PINMUX(PIN_SDIO_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + uart0_cfg: uart0-cfg { + uart0-rx-pins { + pinmux = <PINMUX(PIN_UART0_TX, 0)>, + <PINMUX(PIN_UART0_RX, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; +}; + +&sd { + pinctrl-0 = <&sd_cfg>; + pinctrl-names = "default"; + bus-width = <4>; + no-sdio; + no-mmc; + wp-inverted; + status = "okay"; +}; + +&spifmc0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <1>; + }; +}; + +&spifmc1 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <1>; + }; +}; + +&uart0 { + pinctrl-0 = <&uart0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-evb-v2.dts b/arch/riscv/boot/dts/sophgo/sg2042-evb-v2.dts new file mode 100644 index 000000000000..b2ceae2d8829 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2042-evb-v2.dts @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Sophgo Technology Inc. All rights reserved. + */ + +#include "sg2042.dtsi" + +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/input/input.h> + +/ { + model = "Sophgo SG2042 EVB V2.0"; + compatible = "sophgo,sg2042-evb-v2", "sophgo,sg2042"; + + chosen { + stdout-path = "serial0"; + }; + + pwmfan: pwm-fan { + compatible = "pwm-fan"; + cooling-levels = <103 128 179 230 255>; + pwms = <&pwm 0 40000 0>; + #cooling-cells = <2>; + }; + + thermal-zones { + soc-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + thermal-sensors = <&mcu 0>; + + trips { + soc_active1: soc-active1 { + temperature = <30000>; + hysteresis = <8000>; + type = "active"; + }; + + soc_active2: soc-active2 { + temperature = <58000>; + hysteresis = <12000>; + type = "active"; + }; + + soc_active3: soc-active3 { + temperature = <70000>; + hysteresis = <10000>; + type = "active"; + }; + + soc_hot: soc-hot { + temperature = <80000>; + hysteresis = <5000>; + type = "hot"; + }; + }; + + cooling-maps { + map0 { + trip = <&soc_active1>; + cooling-device = <&pwmfan 0 1>; + }; + + map1 { + trip = <&soc_active2>; + cooling-device = <&pwmfan 1 2>; + }; + + map2 { + trip = <&soc_active3>; + cooling-device = <&pwmfan 2 3>; + }; + + map3 { + trip = <&soc_hot>; + cooling-device = <&pwmfan 3 4>; + }; + }; + }; + + board-thermal { + polling-delay-passive = <1000>; + polling-delay = <1000>; + thermal-sensors = <&mcu 1>; + + trips { + board_active: board-active { + temperature = <75000>; + hysteresis = <8000>; + type = "active"; + }; + }; + + cooling-maps { + map4 { + trip = <&board_active>; + cooling-device = <&pwmfan 3 4>; + }; + }; + }; + }; +}; + +&cgi_main { + clock-frequency = <25000000>; +}; + +&cgi_dpll0 { + clock-frequency = <25000000>; +}; + +&cgi_dpll1 { + clock-frequency = <25000000>; +}; + +&emmc { + pinctrl-0 = <&emmc_cfg>; + pinctrl-names = "default"; + bus-width = <4>; + no-sdio; + no-sd; + non-removable; + wp-inverted; + status = "okay"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_cfg>; + pinctrl-names = "default"; + status = "okay"; + + mcu: syscon@17 { + compatible = "sophgo,sg2042-hwmon-mcu"; + reg = <0x17>; + #thermal-sensor-cells = <1>; + }; +}; + +&gmac0 { + phy-handle = <&phy0>; + phy-mode = "rgmii-id"; + status = "okay"; + + mdio { + phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + reset-gpios = <&port0a 27 GPIO_ACTIVE_LOW>; + reset-assert-us = <100000>; + reset-deassert-us = <100000>; + }; + }; +}; + +&pcie_rc0 { + status = "okay"; +}; + +&pcie_rc1 { + status = "okay"; +}; + +&pcie_rc2 { + status = "okay"; +}; + +&pinctrl { + emmc_cfg: sdhci-emmc-cfg { + sdhci-emmc-wp-pins { + pinmux = <PINMUX(PIN_EMMC_WP, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + + sdhci-emmc-cd-pins { + pinmux = <PINMUX(PIN_EMMC_CD, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-emmc-rst-pwr-pins { + pinmux = <PINMUX(PIN_EMMC_RST, 0)>, + <PINMUX(PIN_EMMC_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + i2c1_cfg: i2c1-cfg { + i2c1-pins { + pinmux = <PINMUX(PIN_IIC1_SDA, 0)>, + <PINMUX(PIN_IIC1_SCL, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; + + sd_cfg: sdhci-sd-cfg { + sdhci-sd-cd-wp-pins { + pinmux = <PINMUX(PIN_SDIO_CD, 0)>, + <PINMUX(PIN_SDIO_WP, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-sd-rst-pwr-pins { + pinmux = <PINMUX(PIN_SDIO_RST, 0)>, + <PINMUX(PIN_SDIO_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + uart0_cfg: uart0-cfg { + uart0-rx-pins { + pinmux = <PINMUX(PIN_UART0_TX, 0)>, + <PINMUX(PIN_UART0_RX, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; +}; + +&sd { + pinctrl-0 = <&sd_cfg>; + pinctrl-names = "default"; + bus-width = <4>; + no-sdio; + no-mmc; + wp-inverted; + status = "okay"; +}; + +&spifmc1 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <1>; + }; +}; + +&uart0 { + pinctrl-0 = <&uart0_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts index be596d01ff8d..ecf8c1e29079 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts +++ b/arch/riscv/boot/dts/sophgo/sg2042-milkv-pioneer.dts @@ -42,6 +42,8 @@ }; &emmc { + pinctrl-0 = <&emmc_cfg>; + pinctrl-names = "default"; bus-width = <4>; no-sdio; no-sd; @@ -50,7 +52,20 @@ status = "okay"; }; +&i2c0 { + pinctrl-0 = <&i2c0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + rtc: rtc@68 { + compatible = "dallas,ds1307"; + reg = <0x68>; + }; +}; + &i2c1 { + pinctrl-0 = <&i2c1_cfg>; + pinctrl-names = "default"; status = "okay"; mcu: syscon@17 { @@ -60,7 +75,95 @@ }; }; +&pinctrl { + emmc_cfg: sdhci-emmc-cfg { + sdhci-emmc-wp-pins { + pinmux = <PINMUX(PIN_EMMC_WP, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + + sdhci-emmc-cd-pins { + pinmux = <PINMUX(PIN_EMMC_CD, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-emmc-rst-pwr-pins { + pinmux = <PINMUX(PIN_EMMC_RST, 0)>, + <PINMUX(PIN_EMMC_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + i2c0_cfg: i2c0-cfg { + i2c0-pins { + pinmux = <PINMUX(PIN_IIC0_SDA, 0)>, + <PINMUX(PIN_IIC0_SCL, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; + + i2c1_cfg: i2c1-cfg { + i2c1-pins { + pinmux = <PINMUX(PIN_IIC1_SDA, 0)>, + <PINMUX(PIN_IIC1_SCL, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; + + sd_cfg: sdhci-sd-cfg { + sdhci-sd-cd-wp-pins { + pinmux = <PINMUX(PIN_SDIO_CD, 0)>, + <PINMUX(PIN_SDIO_WP, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + + sdhci-sd-rst-pwr-pins { + pinmux = <PINMUX(PIN_SDIO_RST, 0)>, + <PINMUX(PIN_SDIO_PWR_EN, 0)>; + bias-disable; + drive-strength-microamp = <26800>; + input-schmitt-disable; + }; + }; + + uart0_cfg: uart0-cfg { + uart0-rx-pins { + pinmux = <PINMUX(PIN_UART0_TX, 0)>, + <PINMUX(PIN_UART0_RX, 0)>; + bias-pull-up; + drive-strength-microamp = <26800>; + input-schmitt-enable; + }; + }; +}; + +&pcie_rc0 { + status = "okay"; +}; + +&pcie_rc2 { + status = "okay"; +}; + +&pcie_rc3 { + status = "okay"; +}; + &sd { + pinctrl-0 = <&sd_cfg>; + pinctrl-names = "default"; bus-width = <4>; no-sdio; no-mmc; @@ -68,11 +171,44 @@ status = "okay"; }; +&spifmc0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <1>; + }; +}; + +&spifmc1 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-tx-bus-width = <1>; + spi-rx-bus-width = <1>; + }; +}; + &uart0 { + pinctrl-0 = <&uart0_cfg>; + pinctrl-names = "default"; status = "okay"; }; / { + pwmfan: pwm-fan { + compatible = "pwm-fan"; + cooling-levels = <103 128 179 230 255>; + pwms = <&pwm 0 40000 0>; + #cooling-cells = <2>; + }; + thermal-zones { soc-thermal { polling-delay-passive = <1000>; @@ -104,6 +240,28 @@ type = "hot"; }; }; + + cooling-maps { + map0 { + trip = <&soc_active1>; + cooling-device = <&pwmfan 0 1>; + }; + + map1 { + trip = <&soc_active2>; + cooling-device = <&pwmfan 1 2>; + }; + + map2 { + trip = <&soc_active3>; + cooling-device = <&pwmfan 2 3>; + }; + + map3 { + trip = <&soc_hot>; + cooling-device = <&pwmfan 3 4>; + }; + }; }; board-thermal { @@ -118,6 +276,13 @@ type = "active"; }; }; + + cooling-maps { + map4 { + trip = <&board_active>; + cooling-device = <&pwmfan 3 4>; + }; + }; }; }; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2042.dtsi b/arch/riscv/boot/dts/sophgo/sg2042.dtsi index e62ac51ac55a..9fddf3f0b3b9 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042.dtsi @@ -8,6 +8,7 @@ #include <dt-bindings/clock/sophgo,sg2042-pll.h> #include <dt-bindings/clock/sophgo,sg2042-rpgate.h> #include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/pinctrl/pinctrl-sg2042.h> #include <dt-bindings/reset/sophgo,sg2042-reset.h> #include "sg2042-cpus.dtsi" @@ -18,6 +19,26 @@ #size-cells = <2>; dma-noncoherent; + distance-map { + compatible = "numa-distance-map-v1"; + distance-matrix = <0 0 10>, + <0 1 15>, + <0 2 25>, + <0 3 30>, + <1 0 15>, + <1 1 10>, + <1 2 30>, + <1 3 25>, + <2 0 25>, + <2 1 30>, + <2 2 10>, + <2 3 15>, + <3 0 30>, + <3 1 25>, + <3 2 15>, + <3 3 10>; + }; + aliases { serial0 = &uart0; }; @@ -47,6 +68,30 @@ interrupt-parent = <&intc>; ranges; + spifmc0: spi@7000180000 { + compatible = "sophgo,sg2042-spifmc-nor"; + reg = <0x70 0x00180000 0x0 0x1000000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkgen GATE_CLK_AHB_SF>; + interrupt-parent = <&intc>; + interrupts = <108 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rstgen RST_SF0>; + status = "disabled"; + }; + + spifmc1: spi@7002180000 { + compatible = "sophgo,sg2042-spifmc-nor"; + reg = <0x70 0x02180000 0x0 0x1000000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkgen GATE_CLK_AHB_SF>; + interrupt-parent = <&intc>; + interrupts = <109 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rstgen RST_SF1>; + status = "disabled"; + }; + i2c0: i2c@7030005000 { compatible = "snps,designware-i2c"; reg = <0x70 0x30005000 0x0 0x1000>; @@ -165,6 +210,15 @@ }; }; + pwm: pwm@703000c000 { + compatible = "sophgo,sg2042-pwm"; + reg = <0x70 0x3000c000 0x0 0x20>; + #pwm-cells = <3>; + clocks = <&clkgen GATE_CLK_APB_PWM>; + clock-names = "apb"; + resets = <&rstgen RST_PWM>; + }; + pllclk: clock-controller@70300100c0 { compatible = "sophgo,sg2042-pll"; reg = <0x70 0x300100c0 0x0 0x40>; @@ -173,6 +227,16 @@ #clock-cells = <1>; }; + msi: msi-controller@7030010304 { + compatible = "sophgo,sg2042-msi"; + reg = <0x70 0x30010304 0x0 0x4>, + <0x70 0x30010300 0x0 0x4>; + reg-names = "clr", "doorbell"; + msi-controller; + #msi-cells = <0>; + msi-ranges = <&intc 64 IRQ_TYPE_EDGE_RISING 32>; + }; + rpgate: clock-controller@7030010368 { compatible = "sophgo,sg2042-rpgate"; reg = <0x70 0x30010368 0x0 0x98>; @@ -181,6 +245,11 @@ #clock-cells = <1>; }; + pinctrl: pinctrl@7030011000 { + compatible = "sophgo,sg2042-pinctrl"; + reg = <0x70 0x30011000 0x0 0x1000>; + }; + clkgen: clock-controller@7030012000 { compatible = "sophgo,sg2042-clkgen"; reg = <0x70 0x30012000 0x0 0x1000>; @@ -195,309 +264,6 @@ #clock-cells = <1>; }; - clint_mswi: interrupt-controller@7094000000 { - compatible = "sophgo,sg2042-aclint-mswi", "thead,c900-aclint-mswi"; - reg = <0x00000070 0x94000000 0x00000000 0x00004000>; - interrupts-extended = <&cpu0_intc 3>, - <&cpu1_intc 3>, - <&cpu2_intc 3>, - <&cpu3_intc 3>, - <&cpu4_intc 3>, - <&cpu5_intc 3>, - <&cpu6_intc 3>, - <&cpu7_intc 3>, - <&cpu8_intc 3>, - <&cpu9_intc 3>, - <&cpu10_intc 3>, - <&cpu11_intc 3>, - <&cpu12_intc 3>, - <&cpu13_intc 3>, - <&cpu14_intc 3>, - <&cpu15_intc 3>, - <&cpu16_intc 3>, - <&cpu17_intc 3>, - <&cpu18_intc 3>, - <&cpu19_intc 3>, - <&cpu20_intc 3>, - <&cpu21_intc 3>, - <&cpu22_intc 3>, - <&cpu23_intc 3>, - <&cpu24_intc 3>, - <&cpu25_intc 3>, - <&cpu26_intc 3>, - <&cpu27_intc 3>, - <&cpu28_intc 3>, - <&cpu29_intc 3>, - <&cpu30_intc 3>, - <&cpu31_intc 3>, - <&cpu32_intc 3>, - <&cpu33_intc 3>, - <&cpu34_intc 3>, - <&cpu35_intc 3>, - <&cpu36_intc 3>, - <&cpu37_intc 3>, - <&cpu38_intc 3>, - <&cpu39_intc 3>, - <&cpu40_intc 3>, - <&cpu41_intc 3>, - <&cpu42_intc 3>, - <&cpu43_intc 3>, - <&cpu44_intc 3>, - <&cpu45_intc 3>, - <&cpu46_intc 3>, - <&cpu47_intc 3>, - <&cpu48_intc 3>, - <&cpu49_intc 3>, - <&cpu50_intc 3>, - <&cpu51_intc 3>, - <&cpu52_intc 3>, - <&cpu53_intc 3>, - <&cpu54_intc 3>, - <&cpu55_intc 3>, - <&cpu56_intc 3>, - <&cpu57_intc 3>, - <&cpu58_intc 3>, - <&cpu59_intc 3>, - <&cpu60_intc 3>, - <&cpu61_intc 3>, - <&cpu62_intc 3>, - <&cpu63_intc 3>; - }; - - clint_mtimer0: timer@70ac004000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac004000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu0_intc 7>, - <&cpu1_intc 7>, - <&cpu2_intc 7>, - <&cpu3_intc 7>; - }; - - clint_mtimer1: timer@70ac014000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac014000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu4_intc 7>, - <&cpu5_intc 7>, - <&cpu6_intc 7>, - <&cpu7_intc 7>; - }; - - clint_mtimer2: timer@70ac024000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac024000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu8_intc 7>, - <&cpu9_intc 7>, - <&cpu10_intc 7>, - <&cpu11_intc 7>; - }; - - clint_mtimer3: timer@70ac034000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac034000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu12_intc 7>, - <&cpu13_intc 7>, - <&cpu14_intc 7>, - <&cpu15_intc 7>; - }; - - clint_mtimer4: timer@70ac044000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac044000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu16_intc 7>, - <&cpu17_intc 7>, - <&cpu18_intc 7>, - <&cpu19_intc 7>; - }; - - clint_mtimer5: timer@70ac054000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac054000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu20_intc 7>, - <&cpu21_intc 7>, - <&cpu22_intc 7>, - <&cpu23_intc 7>; - }; - - clint_mtimer6: timer@70ac064000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac064000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu24_intc 7>, - <&cpu25_intc 7>, - <&cpu26_intc 7>, - <&cpu27_intc 7>; - }; - - clint_mtimer7: timer@70ac074000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac074000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu28_intc 7>, - <&cpu29_intc 7>, - <&cpu30_intc 7>, - <&cpu31_intc 7>; - }; - - clint_mtimer8: timer@70ac084000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac084000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu32_intc 7>, - <&cpu33_intc 7>, - <&cpu34_intc 7>, - <&cpu35_intc 7>; - }; - - clint_mtimer9: timer@70ac094000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac094000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu36_intc 7>, - <&cpu37_intc 7>, - <&cpu38_intc 7>, - <&cpu39_intc 7>; - }; - - clint_mtimer10: timer@70ac0a4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0a4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu40_intc 7>, - <&cpu41_intc 7>, - <&cpu42_intc 7>, - <&cpu43_intc 7>; - }; - - clint_mtimer11: timer@70ac0b4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0b4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu44_intc 7>, - <&cpu45_intc 7>, - <&cpu46_intc 7>, - <&cpu47_intc 7>; - }; - - clint_mtimer12: timer@70ac0c4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0c4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu48_intc 7>, - <&cpu49_intc 7>, - <&cpu50_intc 7>, - <&cpu51_intc 7>; - }; - - clint_mtimer13: timer@70ac0d4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0d4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu52_intc 7>, - <&cpu53_intc 7>, - <&cpu54_intc 7>, - <&cpu55_intc 7>; - }; - - clint_mtimer14: timer@70ac0e4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0e4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu56_intc 7>, - <&cpu57_intc 7>, - <&cpu58_intc 7>, - <&cpu59_intc 7>; - }; - - clint_mtimer15: timer@70ac0f4000 { - compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0f4000 0x00000000 0x0000c000>; - reg-names = "mtimecmp"; - interrupts-extended = <&cpu60_intc 7>, - <&cpu61_intc 7>, - <&cpu62_intc 7>, - <&cpu63_intc 7>; - }; - - intc: interrupt-controller@7090000000 { - compatible = "sophgo,sg2042-plic", "thead,c900-plic"; - #address-cells = <0>; - #interrupt-cells = <2>; - reg = <0x00000070 0x90000000 0x00000000 0x04000000>; - interrupt-controller; - interrupts-extended = - <&cpu0_intc 11>, <&cpu0_intc 9>, - <&cpu1_intc 11>, <&cpu1_intc 9>, - <&cpu2_intc 11>, <&cpu2_intc 9>, - <&cpu3_intc 11>, <&cpu3_intc 9>, - <&cpu4_intc 11>, <&cpu4_intc 9>, - <&cpu5_intc 11>, <&cpu5_intc 9>, - <&cpu6_intc 11>, <&cpu6_intc 9>, - <&cpu7_intc 11>, <&cpu7_intc 9>, - <&cpu8_intc 11>, <&cpu8_intc 9>, - <&cpu9_intc 11>, <&cpu9_intc 9>, - <&cpu10_intc 11>, <&cpu10_intc 9>, - <&cpu11_intc 11>, <&cpu11_intc 9>, - <&cpu12_intc 11>, <&cpu12_intc 9>, - <&cpu13_intc 11>, <&cpu13_intc 9>, - <&cpu14_intc 11>, <&cpu14_intc 9>, - <&cpu15_intc 11>, <&cpu15_intc 9>, - <&cpu16_intc 11>, <&cpu16_intc 9>, - <&cpu17_intc 11>, <&cpu17_intc 9>, - <&cpu18_intc 11>, <&cpu18_intc 9>, - <&cpu19_intc 11>, <&cpu19_intc 9>, - <&cpu20_intc 11>, <&cpu20_intc 9>, - <&cpu21_intc 11>, <&cpu21_intc 9>, - <&cpu22_intc 11>, <&cpu22_intc 9>, - <&cpu23_intc 11>, <&cpu23_intc 9>, - <&cpu24_intc 11>, <&cpu24_intc 9>, - <&cpu25_intc 11>, <&cpu25_intc 9>, - <&cpu26_intc 11>, <&cpu26_intc 9>, - <&cpu27_intc 11>, <&cpu27_intc 9>, - <&cpu28_intc 11>, <&cpu28_intc 9>, - <&cpu29_intc 11>, <&cpu29_intc 9>, - <&cpu30_intc 11>, <&cpu30_intc 9>, - <&cpu31_intc 11>, <&cpu31_intc 9>, - <&cpu32_intc 11>, <&cpu32_intc 9>, - <&cpu33_intc 11>, <&cpu33_intc 9>, - <&cpu34_intc 11>, <&cpu34_intc 9>, - <&cpu35_intc 11>, <&cpu35_intc 9>, - <&cpu36_intc 11>, <&cpu36_intc 9>, - <&cpu37_intc 11>, <&cpu37_intc 9>, - <&cpu38_intc 11>, <&cpu38_intc 9>, - <&cpu39_intc 11>, <&cpu39_intc 9>, - <&cpu40_intc 11>, <&cpu40_intc 9>, - <&cpu41_intc 11>, <&cpu41_intc 9>, - <&cpu42_intc 11>, <&cpu42_intc 9>, - <&cpu43_intc 11>, <&cpu43_intc 9>, - <&cpu44_intc 11>, <&cpu44_intc 9>, - <&cpu45_intc 11>, <&cpu45_intc 9>, - <&cpu46_intc 11>, <&cpu46_intc 9>, - <&cpu47_intc 11>, <&cpu47_intc 9>, - <&cpu48_intc 11>, <&cpu48_intc 9>, - <&cpu49_intc 11>, <&cpu49_intc 9>, - <&cpu50_intc 11>, <&cpu50_intc 9>, - <&cpu51_intc 11>, <&cpu51_intc 9>, - <&cpu52_intc 11>, <&cpu52_intc 9>, - <&cpu53_intc 11>, <&cpu53_intc 9>, - <&cpu54_intc 11>, <&cpu54_intc 9>, - <&cpu55_intc 11>, <&cpu55_intc 9>, - <&cpu56_intc 11>, <&cpu56_intc 9>, - <&cpu57_intc 11>, <&cpu57_intc 9>, - <&cpu58_intc 11>, <&cpu58_intc 9>, - <&cpu59_intc 11>, <&cpu59_intc 9>, - <&cpu60_intc 11>, <&cpu60_intc 9>, - <&cpu61_intc 11>, <&cpu61_intc 9>, - <&cpu62_intc 11>, <&cpu62_intc 9>, - <&cpu63_intc 11>, <&cpu63_intc 9>; - riscv,ndev = <224>; - }; - rstgen: reset-controller@7030013000 { compatible = "sophgo,sg2042-reset"; reg = <0x00000070 0x30013000 0x00000000 0x0000000c>; @@ -518,6 +284,93 @@ status = "disabled"; }; + spi0: spi@7040004000 { + compatible = "sophgo,sg2042-spi", "snps,dw-apb-ssi"; + reg = <0x70 0x40004000 0x00 0x1000>; + clocks = <&clkgen GATE_CLK_APB_SPI>; + interrupt-parent = <&intc>; + interrupts = <110 IRQ_TYPE_LEVEL_HIGH>; + #address-cells = <1>; + #size-cells = <0>; + num-cs = <2>; + resets = <&rstgen RST_SPI0>; + status = "disabled"; + }; + + spi1: spi@7040005000 { + compatible = "sophgo,sg2042-spi", "snps,dw-apb-ssi"; + reg = <0x70 0x40005000 0x00 0x1000>; + clocks = <&clkgen GATE_CLK_APB_SPI>; + interrupt-parent = <&intc>; + interrupts = <111 IRQ_TYPE_LEVEL_HIGH>; + #address-cells = <1>; + #size-cells = <0>; + num-cs = <2>; + resets = <&rstgen RST_SPI1>; + status = "disabled"; + }; + + gmac0: ethernet@7040026000 { + compatible = "sophgo,sg2042-dwmac", "snps,dwmac-5.00a"; + reg = <0x70 0x40026000 0x0 0x4000>; + clocks = <&clkgen GATE_CLK_AXI_ETH0>, + <&clkgen GATE_CLK_PTP_REF_I_ETH0>, + <&clkgen GATE_CLK_TX_ETH0>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + dma-noncoherent; + interrupt-parent = <&intc>; + interrupts = <132 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq"; + resets = <&rstgen RST_ETH0>; + reset-names = "stmmaceth"; + snps,multicast-filter-bins = <0>; + snps,perfect-filter-entries = <1>; + snps,aal; + snps,tso; + snps,txpbl = <32>; + snps,rxpbl = <32>; + snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; + snps,axi-config = <&gmac0_stmmac_axi_setup>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac0_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <8>; + queue0 {}; + queue1 {}; + queue2 {}; + queue3 {}; + queue4 {}; + queue5 {}; + queue6 {}; + queue7 {}; + }; + + gmac0_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <8>; + queue0 {}; + queue1 {}; + queue2 {}; + queue3 {}; + queue4 {}; + queue5 {}; + queue6 {}; + queue7 {}; + }; + + gmac0_stmmac_axi_setup: stmmac-axi-config { + snps,blen = <16 8 4 0 0 0 0>; + snps,wr_osr_lmt = <1>; + snps,rd_osr_lmt = <2>; + }; + }; + emmc: mmc@704002a000 { compatible = "sophgo,sg2042-dwcmshc"; reg = <0x70 0x4002a000 0x0 0x1000>; @@ -545,5 +398,93 @@ "timer"; status = "disabled"; }; + + pcie_rc0: pcie@7060000000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x60000000 0x0 0x00800000>, + <0x40 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x40 0xc0000000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x40 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x40 0xe0000000 0x0 0x20000000>, + <0x43000000 0x42 0x00000000 0x42 0x00000000 0x2 0x00000000>, + <0x03000000 0x41 0x00000000 0x41 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc1: pcie@7060800000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x60800000 0x0 0x00800000>, + <0x44 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x44 0xc0400000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x44 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x44 0xe0000000 0x0 0x20000000>, + <0x43000000 0x46 0x00000000 0x46 0x00000000 0x2 0x00000000>, + <0x03000000 0x45 0x00000000 0x45 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc2: pcie@7062000000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x62000000 0x0 0x00800000>, + <0x48 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <2>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x48 0xc0800000 0x0 0x00400000>, + <0x42000000 0x0 0xd0000000 0x48 0xd0000000 0x0 0x10000000>, + <0x02000000 0x0 0xe0000000 0x48 0xe0000000 0x0 0x20000000>, + <0x03000000 0x49 0x00000000 0x49 0x00000000 0x1 0x00000000>, + <0x43000000 0x4a 0x00000000 0x4a 0x00000000 0x2 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + msi-parent = <&msi>; + status = "disabled"; + }; + + pcie_rc3: pcie@7062800000 { + compatible = "sophgo,sg2042-pcie-host"; + device_type = "pci"; + reg = <0x70 0x62800000 0x0 0x00800000>, + <0x4c 0x00000000 0x0 0x00001000>; + reg-names = "reg", "cfg"; + linux,pci-domain = <3>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x4c 0xc0c00000 0x0 0x00400000>, + <0x42000000 0x0 0xf8000000 0x4c 0xf8000000 0x0 0x04000000>, + <0x02000000 0x0 0xfc000000 0x4c 0xfc000000 0x0 0x04000000>, + <0x43000000 0x4e 0x00000000 0x4e 0x00000000 0x2 0x00000000>, + <0x03000000 0x4d 0x00000000 0x4d 0x00000000 0x1 0x00000000>; + bus-range = <0x0 0xff>; + vendor-id = <0x1f1c>; + device-id = <0x2042>; + cdns,no-bar-match-nbits = <48>; + msi-parent = <&msi>; + status = "disabled"; + }; }; }; diff --git a/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi new file mode 100644 index 000000000000..3135409c2149 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2044-cpus.dtsi @@ -0,0 +1,3157 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <50000000>; + + cpu0: cpu@0 { + compatible = "thead,c920", "riscv"; + reg = <0>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache0>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu1: cpu@1 { + compatible = "thead,c920", "riscv"; + reg = <1>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache0>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu1_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu2: cpu@2 { + compatible = "thead,c920", "riscv"; + reg = <2>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache0>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu2_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu3: cpu@3 { + compatible = "thead,c920", "riscv"; + reg = <3>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache0>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu3_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu4: cpu@4 { + compatible = "thead,c920", "riscv"; + reg = <4>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache1>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu4_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu5: cpu@5 { + compatible = "thead,c920", "riscv"; + reg = <5>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache1>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu5_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu6: cpu@6 { + compatible = "thead,c920", "riscv"; + reg = <6>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache1>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu6_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu7: cpu@7 { + compatible = "thead,c920", "riscv"; + reg = <7>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache1>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu7_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu8: cpu@8 { + compatible = "thead,c920", "riscv"; + reg = <8>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache2>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu8_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu9: cpu@9 { + compatible = "thead,c920", "riscv"; + reg = <9>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache2>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu9_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu10: cpu@10 { + compatible = "thead,c920", "riscv"; + reg = <10>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache2>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu10_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu11: cpu@11 { + compatible = "thead,c920", "riscv"; + reg = <11>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache2>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu11_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu12: cpu@12 { + compatible = "thead,c920", "riscv"; + reg = <12>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache3>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu12_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu13: cpu@13 { + compatible = "thead,c920", "riscv"; + reg = <13>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache3>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu13_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu14: cpu@14 { + compatible = "thead,c920", "riscv"; + reg = <14>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache3>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu14_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu15: cpu@15 { + compatible = "thead,c920", "riscv"; + reg = <15>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache3>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu15_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu16: cpu@16 { + compatible = "thead,c920", "riscv"; + reg = <16>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache4>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu16_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu17: cpu@17 { + compatible = "thead,c920", "riscv"; + reg = <17>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache4>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu17_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu18: cpu@18 { + compatible = "thead,c920", "riscv"; + reg = <18>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache4>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu18_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu19: cpu@19 { + compatible = "thead,c920", "riscv"; + reg = <19>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache4>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu19_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu20: cpu@20 { + compatible = "thead,c920", "riscv"; + reg = <20>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache5>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu20_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu21: cpu@21 { + compatible = "thead,c920", "riscv"; + reg = <21>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache5>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu21_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu22: cpu@22 { + compatible = "thead,c920", "riscv"; + reg = <22>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache5>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu22_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu23: cpu@23 { + compatible = "thead,c920", "riscv"; + reg = <23>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache5>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu23_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu24: cpu@24 { + compatible = "thead,c920", "riscv"; + reg = <24>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache6>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu24_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu25: cpu@25 { + compatible = "thead,c920", "riscv"; + reg = <25>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache6>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu25_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu26: cpu@26 { + compatible = "thead,c920", "riscv"; + reg = <26>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache6>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu26_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu27: cpu@27 { + compatible = "thead,c920", "riscv"; + reg = <27>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache6>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu27_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu28: cpu@28 { + compatible = "thead,c920", "riscv"; + reg = <28>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache7>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu28_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu29: cpu@29 { + compatible = "thead,c920", "riscv"; + reg = <29>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache7>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu29_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu30: cpu@30 { + compatible = "thead,c920", "riscv"; + reg = <30>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache7>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu30_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu31: cpu@31 { + compatible = "thead,c920", "riscv"; + reg = <31>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache7>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu31_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu32: cpu@32 { + compatible = "thead,c920", "riscv"; + reg = <32>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache8>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu32_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu33: cpu@33 { + compatible = "thead,c920", "riscv"; + reg = <33>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache8>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu33_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu34: cpu@34 { + compatible = "thead,c920", "riscv"; + reg = <34>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache8>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu34_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu35: cpu@35 { + compatible = "thead,c920", "riscv"; + reg = <35>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache8>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu35_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu36: cpu@36 { + compatible = "thead,c920", "riscv"; + reg = <36>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache9>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu36_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu37: cpu@37 { + compatible = "thead,c920", "riscv"; + reg = <37>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache9>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu37_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu38: cpu@38 { + compatible = "thead,c920", "riscv"; + reg = <38>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache9>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu38_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu39: cpu@39 { + compatible = "thead,c920", "riscv"; + reg = <39>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache9>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu39_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu40: cpu@40 { + compatible = "thead,c920", "riscv"; + reg = <40>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache10>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu40_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu41: cpu@41 { + compatible = "thead,c920", "riscv"; + reg = <41>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache10>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu41_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu42: cpu@42 { + compatible = "thead,c920", "riscv"; + reg = <42>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache10>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu42_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu43: cpu@43 { + compatible = "thead,c920", "riscv"; + reg = <43>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache10>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu43_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu44: cpu@44 { + compatible = "thead,c920", "riscv"; + reg = <44>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache11>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu44_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu45: cpu@45 { + compatible = "thead,c920", "riscv"; + reg = <45>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache11>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu45_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu46: cpu@46 { + compatible = "thead,c920", "riscv"; + reg = <46>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache11>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu46_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu47: cpu@47 { + compatible = "thead,c920", "riscv"; + reg = <47>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache11>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu47_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu48: cpu@48 { + compatible = "thead,c920", "riscv"; + reg = <48>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache12>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu48_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu49: cpu@49 { + compatible = "thead,c920", "riscv"; + reg = <49>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache12>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu49_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu50: cpu@50 { + compatible = "thead,c920", "riscv"; + reg = <50>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache12>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu50_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu51: cpu@51 { + compatible = "thead,c920", "riscv"; + reg = <51>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache12>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu51_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu52: cpu@52 { + compatible = "thead,c920", "riscv"; + reg = <52>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache13>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu52_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu53: cpu@53 { + compatible = "thead,c920", "riscv"; + reg = <53>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache13>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu53_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu54: cpu@54 { + compatible = "thead,c920", "riscv"; + reg = <54>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache13>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu54_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu55: cpu@55 { + compatible = "thead,c920", "riscv"; + reg = <55>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache13>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu55_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu56: cpu@56 { + compatible = "thead,c920", "riscv"; + reg = <56>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache14>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu56_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu57: cpu@57 { + compatible = "thead,c920", "riscv"; + reg = <57>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache14>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu57_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu58: cpu@58 { + compatible = "thead,c920", "riscv"; + reg = <58>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache14>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu58_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu59: cpu@59 { + compatible = "thead,c920", "riscv"; + reg = <59>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache14>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu59_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu60: cpu@60 { + compatible = "thead,c920", "riscv"; + reg = <60>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache15>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu60_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu61: cpu@61 { + compatible = "thead,c920", "riscv"; + reg = <61>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache15>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu61_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu62: cpu@62 { + compatible = "thead,c920", "riscv"; + reg = <62>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache15>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu62_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu63: cpu@63 { + compatible = "thead,c920", "riscv"; + reg = <63>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <512>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <512>; + device_type = "cpu"; + mmu-type = "riscv,sv48"; + next-level-cache = <&l2_cache15>; + riscv,isa = "rv64imafdcbv"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "b", "v", "sscofpmf", "sstc", + "svinval", "svnapot", "svpbmt", + "zawrs", "zba", "zbb", "zbc", + "zbs", "zca", "zcb", "zcd", + "zfa", "zfbfmin", "zfh", "zfhmin", + "zicbom", "zicbop", "zicboz", "ziccrse", + "zicntr", "zicond","zicsr", "zifencei", + "zihintntl", "zihintpause", "zihpm", + "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + + cpu63_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + interrupt-controller; + #interrupt-cells = <1>; + }; + }; + + cpu-map { + socket0 { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu4>; + }; + + core1 { + cpu = <&cpu5>; + }; + + core2 { + cpu = <&cpu6>; + }; + + core3 { + cpu = <&cpu7>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu8>; + }; + + core1 { + cpu = <&cpu9>; + }; + + core2 { + cpu = <&cpu10>; + }; + + core3 { + cpu = <&cpu11>; + }; + }; + + cluster3 { + core0 { + cpu = <&cpu12>; + }; + + core1 { + cpu = <&cpu13>; + }; + + core2 { + cpu = <&cpu14>; + }; + + core3 { + cpu = <&cpu15>; + }; + }; + + cluster4 { + core0 { + cpu = <&cpu16>; + }; + + core1 { + cpu = <&cpu17>; + }; + + core2 { + cpu = <&cpu18>; + }; + + core3 { + cpu = <&cpu19>; + }; + }; + + cluster5 { + core0 { + cpu = <&cpu20>; + }; + + core1 { + cpu = <&cpu21>; + }; + + core2 { + cpu = <&cpu22>; + }; + + core3 { + cpu = <&cpu23>; + }; + }; + + cluster6 { + core0 { + cpu = <&cpu24>; + }; + + core1 { + cpu = <&cpu25>; + }; + + core2 { + cpu = <&cpu26>; + }; + + core3 { + cpu = <&cpu27>; + }; + }; + + cluster7 { + core0 { + cpu = <&cpu28>; + }; + + core1 { + cpu = <&cpu29>; + }; + + core2 { + cpu = <&cpu30>; + }; + + core3 { + cpu = <&cpu31>; + }; + }; + + cluster8 { + core0 { + cpu = <&cpu32>; + }; + + core1 { + cpu = <&cpu33>; + }; + + core2 { + cpu = <&cpu34>; + }; + + core3 { + cpu = <&cpu35>; + }; + }; + + cluster9 { + core0 { + cpu = <&cpu36>; + }; + + core1 { + cpu = <&cpu37>; + }; + + core2 { + cpu = <&cpu38>; + }; + + core3 { + cpu = <&cpu39>; + }; + }; + + cluster10 { + core0 { + cpu = <&cpu40>; + }; + + core1 { + cpu = <&cpu41>; + }; + + core2 { + cpu = <&cpu42>; + }; + + core3 { + cpu = <&cpu43>; + }; + }; + + cluster11 { + core0 { + cpu = <&cpu44>; + }; + + core1 { + cpu = <&cpu45>; + }; + + core2 { + cpu = <&cpu46>; + }; + + core3 { + cpu = <&cpu47>; + }; + }; + + cluster12 { + core0 { + cpu = <&cpu48>; + }; + + core1 { + cpu = <&cpu49>; + }; + + core2 { + cpu = <&cpu50>; + }; + + core3 { + cpu = <&cpu51>; + }; + }; + + cluster13 { + core0 { + cpu = <&cpu52>; + }; + + core1 { + cpu = <&cpu53>; + }; + + core2 { + cpu = <&cpu54>; + }; + + core3 { + cpu = <&cpu55>; + }; + }; + + cluster14 { + core0 { + cpu = <&cpu56>; + }; + + core1 { + cpu = <&cpu57>; + }; + + core2 { + cpu = <&cpu58>; + }; + + core3 { + cpu = <&cpu59>; + }; + }; + + cluster15 { + core0 { + cpu = <&cpu60>; + }; + + core1 { + cpu = <&cpu61>; + }; + + core2 { + cpu = <&cpu62>; + }; + + core3 { + cpu = <&cpu63>; + }; + }; + }; + }; + + l2_cache0: cache-controller-0 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache1: cache-controller-1 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache2: cache-controller-2 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache3: cache-controller-3 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache4: cache-controller-4 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache5: cache-controller-5 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache6: cache-controller-6 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache7: cache-controller-7 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache8: cache-controller-8 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache9: cache-controller-9 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache10: cache-controller-10 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache11: cache-controller-11 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache12: cache-controller-12 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache13: cache-controller-13 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache14: cache-controller-14 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l2_cache15: cache-controller-15 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <2097152>; + cache-sets = <2048>; + cache-unified; + next-level-cache = <&l3_cache>; + }; + + l3_cache: cache-controller-16 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <3>; + cache-size = <67108864>; + cache-sets = <4096>; + cache-unified; + }; + }; + + pmu { + compatible = "riscv,pmu"; + riscv,event-to-mhpmevent = + <0x00003 0x00000000 0x00000010>, + <0x00004 0x00000000 0x00000011>, + <0x00005 0x00000000 0x00000007>, + <0x00006 0x00000000 0x00000006>, + <0x00008 0x00000000 0x00000027>, + <0x00009 0x00000000 0x00000028>, + <0x10000 0x00000000 0x0000000c>, + <0x10001 0x00000000 0x0000000d>, + <0x10002 0x00000000 0x0000000e>, + <0x10003 0x00000000 0x0000000f>, + <0x10008 0x00000000 0x00000001>, + <0x10009 0x00000000 0x00000002>, + <0x10010 0x00000000 0x00000010>, + <0x10011 0x00000000 0x00000011>, + <0x10012 0x00000000 0x00000012>, + <0x10013 0x00000000 0x00000013>, + <0x10019 0x00000000 0x00000004>, + <0x10021 0x00000000 0x00000003>, + <0x10030 0x00000000 0x0000001c>, + <0x10031 0x00000000 0x0000001b>; + riscv,event-to-mhpmcounters = + <0x00003 0x00003 0xfffffff8>, + <0x00004 0x00004 0xfffffff8>, + <0x00005 0x00005 0xfffffff8>, + <0x00006 0x00006 0xfffffff8>, + <0x00007 0x00007 0xfffffff8>, + <0x00008 0x00008 0xfffffff8>, + <0x00009 0x00009 0xfffffff8>, + <0x0000a 0x0000a 0xfffffff8>, + <0x10000 0x10000 0xfffffff8>, + <0x10001 0x10001 0xfffffff8>, + <0x10002 0x10002 0xfffffff8>, + <0x10003 0x10003 0xfffffff8>, + <0x10008 0x10008 0xfffffff8>, + <0x10009 0x10009 0xfffffff8>, + <0x10010 0x10010 0xfffffff8>, + <0x10011 0x10011 0xfffffff8>, + <0x10012 0x10012 0xfffffff8>, + <0x10013 0x10013 0xfffffff8>, + <0x10019 0x10019 0xfffffff8>, + <0x10021 0x10021 0xfffffff8>, + <0x10030 0x10030 0xfffffff8>, + <0x10031 0x10031 0xfffffff8>; + riscv,raw-event-to-mhpmcounters = + <0x00000000 0x00000001 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000002 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000003 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000004 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000005 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000006 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000007 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000008 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000009 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000a 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000b 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000c 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000d 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000e 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000000f 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000010 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000011 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000012 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000013 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000014 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000015 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000016 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000017 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000018 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000019 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001a 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001b 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001c 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001d 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001e 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000001f 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000020 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000021 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000022 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000023 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000024 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000025 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000026 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000027 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000028 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x00000029 0xffffffff 0xffffffff 0xfffffff8>, + <0x00000000 0x0000002a 0xffffffff 0xffffffff 0xfffffff8>; + }; + + soc { + intc: interrupt-controller@6d40000000 { + compatible = "sophgo,sg2044-plic", "thead,c900-plic"; + #address-cells = <0>; + #interrupt-cells = <2>; + reg = <0x6d 0x40000000 0x0 0x4000000>; + interrupt-controller; + interrupts-extended = + <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>, + <&cpu4_intc 11>, <&cpu4_intc 9>, + <&cpu5_intc 11>, <&cpu5_intc 9>, + <&cpu6_intc 11>, <&cpu6_intc 9>, + <&cpu7_intc 11>, <&cpu7_intc 9>, + <&cpu8_intc 11>, <&cpu8_intc 9>, + <&cpu9_intc 11>, <&cpu9_intc 9>, + <&cpu10_intc 11>, <&cpu10_intc 9>, + <&cpu11_intc 11>, <&cpu11_intc 9>, + <&cpu12_intc 11>, <&cpu12_intc 9>, + <&cpu13_intc 11>, <&cpu13_intc 9>, + <&cpu14_intc 11>, <&cpu14_intc 9>, + <&cpu15_intc 11>, <&cpu15_intc 9>, + <&cpu16_intc 11>, <&cpu16_intc 9>, + <&cpu17_intc 11>, <&cpu17_intc 9>, + <&cpu18_intc 11>, <&cpu18_intc 9>, + <&cpu19_intc 11>, <&cpu19_intc 9>, + <&cpu20_intc 11>, <&cpu20_intc 9>, + <&cpu21_intc 11>, <&cpu21_intc 9>, + <&cpu22_intc 11>, <&cpu22_intc 9>, + <&cpu23_intc 11>, <&cpu23_intc 9>, + <&cpu24_intc 11>, <&cpu24_intc 9>, + <&cpu25_intc 11>, <&cpu25_intc 9>, + <&cpu26_intc 11>, <&cpu26_intc 9>, + <&cpu27_intc 11>, <&cpu27_intc 9>, + <&cpu28_intc 11>, <&cpu28_intc 9>, + <&cpu29_intc 11>, <&cpu29_intc 9>, + <&cpu30_intc 11>, <&cpu30_intc 9>, + <&cpu31_intc 11>, <&cpu31_intc 9>, + <&cpu32_intc 11>, <&cpu32_intc 9>, + <&cpu33_intc 11>, <&cpu33_intc 9>, + <&cpu34_intc 11>, <&cpu34_intc 9>, + <&cpu35_intc 11>, <&cpu35_intc 9>, + <&cpu36_intc 11>, <&cpu36_intc 9>, + <&cpu37_intc 11>, <&cpu37_intc 9>, + <&cpu38_intc 11>, <&cpu38_intc 9>, + <&cpu39_intc 11>, <&cpu39_intc 9>, + <&cpu40_intc 11>, <&cpu40_intc 9>, + <&cpu41_intc 11>, <&cpu41_intc 9>, + <&cpu42_intc 11>, <&cpu42_intc 9>, + <&cpu43_intc 11>, <&cpu43_intc 9>, + <&cpu44_intc 11>, <&cpu44_intc 9>, + <&cpu45_intc 11>, <&cpu45_intc 9>, + <&cpu46_intc 11>, <&cpu46_intc 9>, + <&cpu47_intc 11>, <&cpu47_intc 9>, + <&cpu48_intc 11>, <&cpu48_intc 9>, + <&cpu49_intc 11>, <&cpu49_intc 9>, + <&cpu50_intc 11>, <&cpu50_intc 9>, + <&cpu51_intc 11>, <&cpu51_intc 9>, + <&cpu52_intc 11>, <&cpu52_intc 9>, + <&cpu53_intc 11>, <&cpu53_intc 9>, + <&cpu54_intc 11>, <&cpu54_intc 9>, + <&cpu55_intc 11>, <&cpu55_intc 9>, + <&cpu56_intc 11>, <&cpu56_intc 9>, + <&cpu57_intc 11>, <&cpu57_intc 9>, + <&cpu58_intc 11>, <&cpu58_intc 9>, + <&cpu59_intc 11>, <&cpu59_intc 9>, + <&cpu60_intc 11>, <&cpu60_intc 9>, + <&cpu61_intc 11>, <&cpu61_intc 9>, + <&cpu62_intc 11>, <&cpu62_intc 9>, + <&cpu63_intc 11>, <&cpu63_intc 9>; + riscv,ndev = <863>; + }; + + aclint_mswi: interrupt-controller@6d44000000 { + compatible = "sophgo,sg2044-aclint-mswi", "thead,c900-aclint-mswi"; + reg = <0x6d 0x44000000 0x0 0x4000>; + interrupts-extended = <&cpu0_intc 3>, + <&cpu1_intc 3>, + <&cpu2_intc 3>, + <&cpu3_intc 3>, + <&cpu4_intc 3>, + <&cpu5_intc 3>, + <&cpu6_intc 3>, + <&cpu7_intc 3>, + <&cpu8_intc 3>, + <&cpu9_intc 3>, + <&cpu10_intc 3>, + <&cpu11_intc 3>, + <&cpu12_intc 3>, + <&cpu13_intc 3>, + <&cpu14_intc 3>, + <&cpu15_intc 3>, + <&cpu16_intc 3>, + <&cpu17_intc 3>, + <&cpu18_intc 3>, + <&cpu19_intc 3>, + <&cpu20_intc 3>, + <&cpu21_intc 3>, + <&cpu22_intc 3>, + <&cpu23_intc 3>, + <&cpu24_intc 3>, + <&cpu25_intc 3>, + <&cpu26_intc 3>, + <&cpu27_intc 3>, + <&cpu28_intc 3>, + <&cpu29_intc 3>, + <&cpu30_intc 3>, + <&cpu31_intc 3>, + <&cpu32_intc 3>, + <&cpu33_intc 3>, + <&cpu34_intc 3>, + <&cpu35_intc 3>, + <&cpu36_intc 3>, + <&cpu37_intc 3>, + <&cpu38_intc 3>, + <&cpu39_intc 3>, + <&cpu40_intc 3>, + <&cpu41_intc 3>, + <&cpu42_intc 3>, + <&cpu43_intc 3>, + <&cpu44_intc 3>, + <&cpu45_intc 3>, + <&cpu46_intc 3>, + <&cpu47_intc 3>, + <&cpu48_intc 3>, + <&cpu49_intc 3>, + <&cpu50_intc 3>, + <&cpu51_intc 3>, + <&cpu52_intc 3>, + <&cpu53_intc 3>, + <&cpu54_intc 3>, + <&cpu55_intc 3>, + <&cpu56_intc 3>, + <&cpu57_intc 3>, + <&cpu58_intc 3>, + <&cpu59_intc 3>, + <&cpu60_intc 3>, + <&cpu61_intc 3>, + <&cpu62_intc 3>, + <&cpu63_intc 3>; + }; + + aclint_mtimer: timer@6d44004000 { + compatible = "sophgo,sg2044-aclint-mtimer", "thead,c900-aclint-mtimer"; + reg = <0x6d 0x44004000 0x0 0x8000>; + reg-names = "mtimecmp"; + interrupts-extended = <&cpu0_intc 7>, + <&cpu1_intc 7>, + <&cpu2_intc 7>, + <&cpu3_intc 7>, + <&cpu4_intc 7>, + <&cpu5_intc 7>, + <&cpu6_intc 7>, + <&cpu7_intc 7>, + <&cpu8_intc 7>, + <&cpu9_intc 7>, + <&cpu10_intc 7>, + <&cpu11_intc 7>, + <&cpu12_intc 7>, + <&cpu13_intc 7>, + <&cpu14_intc 7>, + <&cpu15_intc 7>, + <&cpu16_intc 7>, + <&cpu17_intc 7>, + <&cpu18_intc 7>, + <&cpu19_intc 7>, + <&cpu20_intc 7>, + <&cpu21_intc 7>, + <&cpu22_intc 7>, + <&cpu23_intc 7>, + <&cpu24_intc 7>, + <&cpu25_intc 7>, + <&cpu26_intc 7>, + <&cpu27_intc 7>, + <&cpu28_intc 7>, + <&cpu29_intc 7>, + <&cpu30_intc 7>, + <&cpu31_intc 7>, + <&cpu32_intc 7>, + <&cpu33_intc 7>, + <&cpu34_intc 7>, + <&cpu35_intc 7>, + <&cpu36_intc 7>, + <&cpu37_intc 7>, + <&cpu38_intc 7>, + <&cpu39_intc 7>, + <&cpu40_intc 7>, + <&cpu41_intc 7>, + <&cpu42_intc 7>, + <&cpu43_intc 7>, + <&cpu44_intc 7>, + <&cpu45_intc 7>, + <&cpu46_intc 7>, + <&cpu47_intc 7>, + <&cpu48_intc 7>, + <&cpu49_intc 7>, + <&cpu50_intc 7>, + <&cpu51_intc 7>, + <&cpu52_intc 7>, + <&cpu53_intc 7>, + <&cpu54_intc 7>, + <&cpu55_intc 7>, + <&cpu56_intc 7>, + <&cpu57_intc 7>, + <&cpu58_intc 7>, + <&cpu59_intc 7>, + <&cpu60_intc 7>, + <&cpu61_intc 7>, + <&cpu62_intc 7>, + <&cpu63_intc 7>; + }; + + aclint_sswi: interrupt-controller@6d4400c000 { + compatible = "sophgo,sg2044-aclint-sswi", "thead,c900-aclint-sswi"; + reg = <0x6d 0x4400c000 0x0 0x1000>; + #interrupt-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 1>, + <&cpu1_intc 1>, + <&cpu2_intc 1>, + <&cpu3_intc 1>, + <&cpu4_intc 1>, + <&cpu5_intc 1>, + <&cpu6_intc 1>, + <&cpu7_intc 1>, + <&cpu8_intc 1>, + <&cpu9_intc 1>, + <&cpu10_intc 1>, + <&cpu11_intc 1>, + <&cpu12_intc 1>, + <&cpu13_intc 1>, + <&cpu14_intc 1>, + <&cpu15_intc 1>, + <&cpu16_intc 1>, + <&cpu17_intc 1>, + <&cpu18_intc 1>, + <&cpu19_intc 1>, + <&cpu20_intc 1>, + <&cpu21_intc 1>, + <&cpu22_intc 1>, + <&cpu23_intc 1>, + <&cpu24_intc 1>, + <&cpu25_intc 1>, + <&cpu26_intc 1>, + <&cpu27_intc 1>, + <&cpu28_intc 1>, + <&cpu29_intc 1>, + <&cpu30_intc 1>, + <&cpu31_intc 1>, + <&cpu32_intc 1>, + <&cpu33_intc 1>, + <&cpu34_intc 1>, + <&cpu35_intc 1>, + <&cpu36_intc 1>, + <&cpu37_intc 1>, + <&cpu38_intc 1>, + <&cpu39_intc 1>, + <&cpu40_intc 1>, + <&cpu41_intc 1>, + <&cpu42_intc 1>, + <&cpu43_intc 1>, + <&cpu44_intc 1>, + <&cpu45_intc 1>, + <&cpu46_intc 1>, + <&cpu47_intc 1>, + <&cpu48_intc 1>, + <&cpu49_intc 1>, + <&cpu50_intc 1>, + <&cpu51_intc 1>, + <&cpu52_intc 1>, + <&cpu53_intc 1>, + <&cpu54_intc 1>, + <&cpu55_intc 1>, + <&cpu56_intc 1>, + <&cpu57_intc 1>, + <&cpu58_intc 1>, + <&cpu59_intc 1>, + <&cpu60_intc 1>, + <&cpu61_intc 1>, + <&cpu62_intc 1>, + <&cpu63_intc 1>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2044-reset.h b/arch/riscv/boot/dts/sophgo/sg2044-reset.h new file mode 100644 index 000000000000..3a7bbfdb4bae --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2044-reset.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */ +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +#ifndef _SG2044_RESET_H +#define _SG2044_RESET_H + +#define RST_AP_SYS 0 +#define RST_AP_SYS_CORE0 1 +#define RST_AP_SYS_CORE1 2 +#define RST_AP_SYS_CORE2 3 +#define RST_AP_SYS_CORE3 4 +#define RST_AP_PIC 5 +#define RST_AP_TDT 6 +#define RST_RP_PIC_TDT 7 +#define RST_HSDMA 8 +#define RST_SYSDMA 9 +#define RST_EFUSE0 10 +#define RST_EFUSE1 11 +#define RST_RTC 12 +#define RST_TIMER 13 +#define RST_WDT 14 +#define RST_AHB_ROM0 15 +#define RST_AHB_ROM1 16 +#define RST_I2C0 17 +#define RST_I2C1 18 +#define RST_I2C2 19 +#define RST_I2C3 20 +#define RST_GPIO0 21 +#define RST_GPIO1 22 +#define RST_GPIO2 23 +#define RST_PWM 24 +#define RST_AXI_SRAM0 25 +#define RST_AXI_SRAM1 26 +#define RST_SPIFMC0 27 +#define RST_SPIFMC1 28 +#define RST_MAILBOX 29 +#define RST_ETH0 30 +#define RST_EMMC 31 +#define RST_SD 32 +#define RST_UART0 33 +#define RST_UART1 34 +#define RST_UART2 35 +#define RST_UART3 36 +#define RST_SPI0 37 +#define RST_SPI1 38 +#define RST_MTLI 39 +#define RST_DBG_I2C 40 +#define RST_C2C0 41 +#define RST_C2C1 42 +#define RST_C2C2 43 +#define RST_C2C3 44 +#define RST_CXP 45 +#define RST_DDR0 46 +#define RST_DDR1 47 +#define RST_DDR2 48 +#define RST_DDR3 49 +#define RST_DDR4 50 +#define RST_DDR5 51 +#define RST_DDR6 52 +#define RST_DDR7 53 +#define RST_DDR8 54 +#define RST_DDR9 55 +#define RST_DDR10 56 +#define RST_DDR11 57 +#define RST_DDR12 58 +#define RST_DDR13 59 +#define RST_DDR14 60 +#define RST_DDR15 61 +#define RST_BAR 62 +#define RST_K2K 63 +#define RST_CC_SYS_X1Y1 64 +#define RST_CC_SYS_X1Y2 65 +#define RST_CC_SYS_X1Y3 66 +#define RST_CC_SYS_X1Y4 67 +#define RST_CC_SYS_X0Y1 68 +#define RST_CC_SYS_X0Y2 69 +#define RST_CC_SYS_X0Y3 70 +#define RST_CC_SYS_X0Y4 71 +#define RST_SC_X1Y1 80 +#define RST_SC_X1Y2 81 +#define RST_SC_X1Y3 82 +#define RST_SC_X1Y4 83 +#define RST_SC_X0Y1 84 +#define RST_SC_X0Y2 85 +#define RST_SC_X0Y3 86 +#define RST_SC_X0Y4 87 +#define RST_RP_CLUSTER_X1Y1_S0 160 +#define RST_RP_CLUSTER_X1Y1_S1 161 +#define RST_RP_CLUSTER_X1Y2_S0 162 +#define RST_RP_CLUSTER_X1Y2_S1 163 +#define RST_RP_CLUSTER_X1Y3_S0 164 +#define RST_RP_CLUSTER_X1Y3_S1 165 +#define RST_RP_CLUSTER_X1Y4_S0 166 +#define RST_RP_CLUSTER_X1Y4_S1 167 +#define RST_RP_CLUSTER_X0Y1_W0 168 +#define RST_RP_CLUSTER_X0Y1_W1 169 +#define RST_RP_CLUSTER_X0Y2_W0 170 +#define RST_RP_CLUSTER_X0Y2_W1 171 +#define RST_RP_CLUSTER_X0Y3_W0 172 +#define RST_RP_CLUSTER_X0Y3_W1 173 +#define RST_RP_CLUSTER_X0Y4_W0 174 +#define RST_RP_CLUSTER_X0Y4_W1 175 +#define RST_TPSYS_X1Y1 180 +#define RST_TPSYS_X1Y2 181 +#define RST_TPSYS_X1Y3 182 +#define RST_TPSYS_X1Y4 183 +#define RST_TPSYS_X0Y1 184 +#define RST_TPSYS_X0Y2 185 +#define RST_TPSYS_X0Y3 186 +#define RST_TPSYS_X0Y4 187 +#define RST_SPACC 188 +#define RST_PKA 189 +#define RST_SE_TRNG 190 +#define RST_SE_DBG 191 +#define RST_SE_FAB_FW 192 +#define RST_SE_CTRL 193 +#define RST_MAILBOX0 194 +#define RST_MAILBOX1 195 +#define RST_MAILBOX2 196 +#define RST_MAILBOX3 197 +#define RST_INTC0 198 +#define RST_INTC1 199 +#define RST_INTC2 200 +#define RST_INTC3 201 + +#endif /* _DT_BINDINGS_SG2044_RESET_H */ diff --git a/arch/riscv/boot/dts/sophgo/sg2044-sophgo-srd3-10.dts b/arch/riscv/boot/dts/sophgo/sg2044-sophgo-srd3-10.dts new file mode 100644 index 000000000000..fed3d9a384a0 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2044-sophgo-srd3-10.dts @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +/dts-v1/; + +#include "sg2044.dtsi" + +/ { + model = "Sophgo SG2044 SRD3-10"; + compatible = "sophgo,srd3-10", "sophgo,sg2044"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; +}; + +&osc { + clock-frequency = <25000000>; +}; + +&emmc { + bus-width = <4>; + no-sdio; + no-sd; + non-removable; + wp-inverted; + status = "okay"; +}; + +&gmac0 { + phy-handle = <&phy0>; + phy-mode = "rgmii-id"; + status = "okay"; + + mdio { + phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + reset-gpios = <&porta 28 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <10000>; + rx-internal-delay-ps = <2050>; + }; + }; +}; + +&i2c1 { + status = "okay"; + + mcu: syscon@17 { + compatible = "sophgo,sg2044-hwmon-mcu", "sophgo,sg2042-hwmon-mcu"; + reg = <0x17>; + #thermal-sensor-cells = <1>; + }; +}; + +&msi { + status = "okay"; +}; + +&pcie0 { + bus-range = <0x00 0xff>; + linux,pci-domain = <1>; + status = "okay"; +}; + +&pcie1 { + bus-range = <0x00 0xff>; + linux,pci-domain = <0>; + status = "okay"; +}; + +&pcie2 { + bus-range = <0x00 0xff>; + linux,pci-domain = <3>; + status = "okay"; +}; + +&pcie3 { + bus-range = <0x00 0xff>; + linux,pci-domain = <2>; + status = "okay"; +}; + +&pcie4 { + bus-range = <0x00 0xff>; + linux,pci-domain = <4>; + status = "okay"; +}; + +&pwm { + status = "okay"; +}; + +&sd { + bus-width = <4>; + no-sdio; + no-mmc; + wp-inverted; + status = "okay"; +}; + +&uart0 { + /* for firmware */ + status = "reserved"; +}; + +&uart1 { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/sophgo/sg2044.dtsi b/arch/riscv/boot/dts/sophgo/sg2044.dtsi new file mode 100644 index 000000000000..320c4d1d08e6 --- /dev/null +++ b/arch/riscv/boot/dts/sophgo/sg2044.dtsi @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +#include <dt-bindings/clock/sophgo,sg2044-pll.h> +#include <dt-bindings/clock/sophgo,sg2044-clk.h> +#include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/pinctrl/pinctrl-sg2044.h> + +#include "sg2044-cpus.dtsi" +#include "sg2044-reset.h" + +/ { + compatible = "sophgo,sg2044"; + + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0x00000010 0x00000000>; + }; + + osc: oscillator { + compatible = "fixed-clock"; + clock-output-names = "osc"; + #clock-cells = <0>; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + pcie0: pcie@6c00000000 { + compatible = "sophgo,sg2044-pcie"; + reg = <0x6c 0x00000000 0x0 0x00001000>, + <0x6c 0x00300000 0x0 0x00004000>, + <0x48 0x00000000 0x0 0x00001000>, + <0x6c 0x000c0000 0x0 0x00001000>; + reg-names = "dbi", "atu", "config", "app"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + clocks = <&clk CLK_GATE_PCIE_1G>; + clock-names = "core"; + device_type = "pci"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + msi-parent = <&msi>; + ranges = <0x01000000 0x0 0x00000000 0x48 0x10000000 0x0 0x00200000>, + <0x42000000 0x0 0x10000000 0x0 0x10000000 0x0 0x04000000>, + <0x02000000 0x0 0x14000000 0x0 0x14000000 0x0 0x04000000>, + <0x43000000 0x4a 0x00000000 0x4a 0x00000000 0x2 0x00000000>, + <0x03000000 0x49 0x00000000 0x49 0x00000000 0x1 0x00000000>; + status = "disabled"; + + pcie_intc0: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <65 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + }; + }; + + pcie1: pcie@6c00400000 { + compatible = "sophgo,sg2044-pcie"; + reg = <0x6c 0x00400000 0x0 0x00001000>, + <0x6c 0x00700000 0x0 0x00004000>, + <0x40 0x00000000 0x0 0x00001000>, + <0x6c 0x00780000 0x0 0x00001000>; + reg-names = "dbi", "atu", "config", "app"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + clocks = <&clk CLK_GATE_PCIE_1G>; + clock-names = "core"; + device_type = "pci"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc1 0>, + <0 0 0 2 &pcie_intc1 1>, + <0 0 0 3 &pcie_intc1 2>, + <0 0 0 4 &pcie_intc1 3>; + msi-parent = <&msi>; + ranges = <0x01000000 0x0 0x00000000 0x40 0x10000000 0x0 0x00200000>, + <0x42000000 0x0 0x00000000 0x0 0x00000000 0x0 0x04000000>, + <0x02000000 0x0 0x04000000 0x0 0x04000000 0x0 0x04000000>, + <0x43000000 0x42 0x00000000 0x42 0x00000000 0x2 0x00000000>, + <0x03000000 0x41 0x00000000 0x41 0x00000000 0x1 0x00000000>; + status = "disabled"; + + pcie_intc1: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <64 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + }; + }; + + pcie2: pcie@6c04000000 { + compatible = "sophgo,sg2044-pcie"; + reg = <0x6c 0x04000000 0x0 0x00001000>, + <0x6c 0x04300000 0x0 0x00004000>, + <0x58 0x00000000 0x0 0x00001000>, + <0x6c 0x040c0000 0x0 0x00001000>; + reg-names = "dbi", "atu", "config", "app"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + clocks = <&clk CLK_GATE_PCIE_1G>; + clock-names = "core"; + device_type = "pci"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc2 0>, + <0 0 0 2 &pcie_intc2 1>, + <0 0 0 3 &pcie_intc2 2>, + <0 0 0 4 &pcie_intc2 3>; + msi-parent = <&msi>; + ranges = <0x01000000 0x0 0x00000000 0x58 0x10000000 0x0 0x00200000>, + <0x42000000 0x0 0x30000000 0x0 0x30000000 0x0 0x04000000>, + <0x02000000 0x0 0x34000000 0x0 0x34000000 0x0 0x04000000>, + <0x43000000 0x5a 0x00000000 0x5a 0x00000000 0x2 0x00000000>, + <0x03000000 0x59 0x00000000 0x59 0x00000000 0x1 0x00000000>; + status = "disabled"; + + pcie_intc2: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <74 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + }; + }; + + pcie3: pcie@6c04400000 { + compatible = "sophgo,sg2044-pcie"; + reg = <0x6c 0x04400000 0x0 0x00001000>, + <0x6c 0x04700000 0x0 0x00004000>, + <0x50 0x00000000 0x0 0x00001000>, + <0x6c 0x04780000 0x0 0x00001000>; + reg-names = "dbi", "atu", "config", "app"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + clocks = <&clk CLK_GATE_PCIE_1G>; + clock-names = "core"; + device_type = "pci"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc3 0>, + <0 0 0 2 &pcie_intc3 1>, + <0 0 0 3 &pcie_intc3 2>, + <0 0 0 4 &pcie_intc3 3>; + msi-parent = <&msi>; + ranges = <0x01000000 0x0 0x00000000 0x50 0x10000000 0x0 0x00200000>, + <0x42000000 0x0 0x20000000 0x0 0x20000000 0x0 0x04000000>, + <0x02000000 0x0 0x24000000 0x0 0x24000000 0x0 0x04000000>, + <0x43000000 0x52 0x00000000 0x52 0x00000000 0x2 0x00000000>, + <0x03000000 0x51 0x00000000 0x51 0x00000000 0x1 0x00000000>; + status = "disabled"; + + pcie_intc3: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <73 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + }; + }; + + pcie4: pcie@6c08400000 { + compatible = "sophgo,sg2044-pcie"; + reg = <0x6c 0x08400000 0x0 0x00001000>, + <0x6c 0x08700000 0x0 0x00004000>, + <0x60 0x00000000 0x0 0x00001000>, + <0x6c 0x08780000 0x0 0x00001000>; + reg-names = "dbi", "atu", "config", "app"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + clocks = <&clk CLK_GATE_PCIE_1G>; + clock-names = "core"; + device_type = "pci"; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc4 0>, + <0 0 0 2 &pcie_intc4 1>, + <0 0 0 3 &pcie_intc4 2>, + <0 0 0 4 &pcie_intc4 3>; + msi-parent = <&msi>; + ranges = <0x01000000 0x0 0x00000000 0x60 0x10000000 0x0 0x00200000>, + <0x42000000 0x0 0x40000000 0x0 0x40000000 0x0 0x04000000>, + <0x02000000 0x0 0x44000000 0x0 0x44000000 0x0 0x04000000>, + <0x43000000 0x62 0x00000000 0x62 0x00000000 0x2 0x00000000>, + <0x03000000 0x61 0x00000000 0x61 0x00000000 0x1 0x00000000>; + status = "disabled"; + + pcie_intc4: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <125 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + }; + }; + + msi: msi-controller@6d50000000 { + compatible = "sophgo,sg2044-msi"; + reg = <0x6d 0x50000000 0x0 0x800>, + <0x0 0x7ee00000 0x0 0x40>; + reg-names = "clr", "doorbell"; + #msi-cells = <0>; + msi-controller; + msi-ranges = <&intc 352 IRQ_TYPE_EDGE_RISING 512>; + status = "disabled"; + }; + + spifmc0: spi@7001000000 { + compatible = "sophgo,sg2044-spifmc-nor"; + reg = <0x70 0x01000000 0x0 0x4000000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk CLK_GATE_AHB_SPIFMC>; + interrupt-parent = <&intc>; + interrupts = <37 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPIFMC0>; + status = "disabled"; + }; + + spifmc1: spi@7005000000 { + compatible = "sophgo,sg2044-spifmc-nor"; + reg = <0x70 0x05000000 0x0 0x4000000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk CLK_GATE_AHB_SPIFMC>; + interrupt-parent = <&intc>; + interrupts = <38 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_SPIFMC1>; + status = "disabled"; + }; + + dmac0: dma-controller@7020000000 { + compatible = "snps,axi-dma-1.01a"; + reg = <0x70 0x20000000 0x0 0x10000>; + #dma-cells = <1>; + clock-names = "core-clk", "cfgr-clk"; + clocks = <&clk CLK_GATE_SYSDMA_AXI>, + <&clk CLK_GATE_SYSDMA_AXI>; + dma-noncoherent; + interrupt-parent = <&intc>; + interrupts = <36 IRQ_TYPE_LEVEL_HIGH>; + dma-channels = <8>; + snps,priority = <0 1 2 3 4 5 6 7>; + snps,block-size = <4096 4096 4096 4096 + 4096 4096 4096 4096>; + snps,dma-masters = <2>; + snps,data-width = <2>; + snps,axi-max-burst-len = <4>; + status = "disabled"; + }; + + uart0: serial@7030000000 { + compatible = "sophgo,sg2044-uart", "snps,dw-apb-uart"; + reg = <0x70 0x30000000 0x0 0x1000>; + clock-frequency = <500000000>; + clocks = <&clk CLK_GATE_UART_500M>, + <&clk CLK_GATE_APB_UART>; + clock-names = "baudclk", "apb_pclk"; + interrupt-parent = <&intc>; + interrupts = <41 IRQ_TYPE_LEVEL_HIGH>; + reg-shift = <2>; + reg-io-width = <4>; + resets = <&rst RST_UART0>; + status = "disabled"; + }; + + uart1: serial@7030001000 { + compatible = "sophgo,sg2044-uart", "snps,dw-apb-uart"; + reg = <0x70 0x30001000 0x0 0x1000>; + clock-frequency = <500000000>; + clocks = <&clk CLK_GATE_UART_500M>, + <&clk CLK_GATE_APB_UART>; + clock-names = "baudclk", "apb_pclk"; + interrupt-parent = <&intc>; + interrupts = <42 IRQ_TYPE_LEVEL_HIGH>; + reg-shift = <2>; + reg-io-width = <4>; + resets = <&rst RST_UART1>; + status = "disabled"; + }; + + uart2: serial@7030002000 { + compatible = "sophgo,sg2044-uart", "snps,dw-apb-uart"; + reg = <0x70 0x30002000 0x0 0x1000>; + clock-frequency = <500000000>; + clocks = <&clk CLK_GATE_UART_500M>, + <&clk CLK_GATE_APB_UART>; + clock-names = "baudclk", "apb_pclk"; + interrupt-parent = <&intc>; + interrupts = <43 IRQ_TYPE_LEVEL_HIGH>; + reg-shift = <2>; + reg-io-width = <4>; + resets = <&rst RST_UART2>; + status = "disabled"; + }; + + uart3: serial@7030003000 { + compatible = "sophgo,sg2044-uart", "snps,dw-apb-uart"; + reg = <0x70 0x30003000 0x0 0x1000>; + clock-frequency = <500000000>; + clocks = <&clk CLK_GATE_UART_500M>, + <&clk CLK_GATE_APB_UART>; + clock-names = "baudclk", "apb_pclk"; + interrupt-parent = <&intc>; + interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; + reg-shift = <2>; + reg-io-width = <4>; + resets = <&rst RST_UART3>; + status = "disabled"; + }; + + gmac0: ethernet@7030006000 { + compatible = "sophgo,sg2044-dwmac", "snps,dwmac-5.30a"; + reg = <0x70 0x30006000 0x0 0x4000>; + clocks = <&clk CLK_GATE_AXI_ETH0>, + <&clk CLK_GATE_PTP_REF_I_ETH0>, + <&clk CLK_GATE_TX_ETH0>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + dma-noncoherent; + interrupt-parent = <&intc>; + interrupts = <296 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq"; + resets = <&rst RST_ETH0>; + reset-names = "stmmaceth"; + snps,multicast-filter-bins = <0>; + snps,perfect-filter-entries = <1>; + snps,aal; + snps,tso; + snps,txpbl = <32>; + snps,rxpbl = <32>; + snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; + snps,axi-config = <&gmac0_stmmac_axi_setup>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac0_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <8>; + snps,rx-sched-wsp; + queue0 {}; + queue1 {}; + queue2 {}; + queue3 {}; + queue4 {}; + queue5 {}; + queue6 {}; + queue7 {}; + }; + + gmac0_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <8>; + queue0 {}; + queue1 {}; + queue2 {}; + queue3 {}; + queue4 {}; + queue5 {}; + queue6 {}; + queue7 {}; + }; + + gmac0_stmmac_axi_setup: stmmac-axi-config { + snps,blen = <16 8 4 0 0 0 0>; + snps,wr_osr_lmt = <1>; + snps,rd_osr_lmt = <2>; + }; + }; + + emmc: mmc@703000a000 { + compatible = "sophgo,sg2044-dwcmshc", "sophgo,sg2042-dwcmshc"; + reg = <0x70 0x3000a000 0x0 0x1000>; + clocks = <&clk CLK_GATE_EMMC>, + <&clk CLK_GATE_AXI_EMMC>, + <&clk CLK_GATE_EMMC_100K>; + clock-names = "core", "bus", "timer"; + interrupt-parent = <&intc>; + interrupts = <298 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + sd: mmc@703000b000 { + compatible = "sophgo,sg2044-dwcmshc", "sophgo,sg2042-dwcmshc"; + reg = <0x70 0x3000b000 0x0 0x1000>; + clocks = <&clk CLK_GATE_SD>, + <&clk CLK_GATE_AXI_SD>, + <&clk CLK_GATE_SD_100K>; + clock-names = "core", "bus", "timer"; + interrupt-parent = <&intc>; + interrupts = <300 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + i2c0: i2c@7040005000 { + compatible = "sophgo,sg2044-i2c", "snps,designware-i2c"; + reg = <0x70 0x40005000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <100000>; + clocks = <&clk CLK_GATE_APB_I2C>; + clock-names = "ref"; + interrupt-parent = <&intc>; + interrupts = <31 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C0>; + status = "disabled"; + }; + + i2c1: i2c@7040006000 { + compatible = "sophgo,sg2044-i2c", "snps,designware-i2c"; + reg = <0x70 0x40006000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <100000>; + clocks = <&clk CLK_GATE_APB_I2C>; + clock-names = "ref"; + interrupt-parent = <&intc>; + interrupts = <32 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C1>; + status = "disabled"; + }; + + i2c2: i2c@7040007000 { + compatible = "sophgo,sg2044-i2c", "snps,designware-i2c"; + reg = <0x70 0x40007000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <100000>; + clocks = <&clk CLK_GATE_APB_I2C>; + clock-names = "ref"; + interrupt-parent = <&intc>; + interrupts = <33 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C2>; + status = "disabled"; + }; + + i2c3: i2c@7040008000 { + compatible = "sophgo,sg2044-i2c", "snps,designware-i2c"; + reg = <0x70 0x40008000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <100000>; + clocks = <&clk CLK_GATE_APB_I2C>; + clock-names = "ref"; + interrupt-parent = <&intc>; + interrupts = <34 IRQ_TYPE_LEVEL_HIGH>; + resets = <&rst RST_I2C3>; + status = "disabled"; + }; + + gpio0: gpio@7040009000 { + compatible = "snps,dw-apb-gpio"; + reg = <0x70 0x40009000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk CLK_GATE_APB_GPIO>, + <&clk CLK_GATE_GPIO_DB>; + clock-names = "bus", "db"; + resets = <&rst RST_GPIO0>; + + porta: gpio-controller@0 { + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupts = <26 IRQ_TYPE_LEVEL_HIGH>; + }; + }; + + gpio1: gpio@704000a000 { + compatible = "snps,dw-apb-gpio"; + reg = <0x70 0x4000a000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk CLK_GATE_APB_GPIO>, + <&clk CLK_GATE_GPIO_DB>; + clock-names = "bus", "db"; + resets = <&rst RST_GPIO1>; + + portb: gpio-controller@0 { + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupts = <27 IRQ_TYPE_LEVEL_HIGH>; + }; + }; + + gpio2: gpio@704000b000 { + compatible = "snps,dw-apb-gpio"; + reg = <0x70 0x4000b000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk CLK_GATE_APB_GPIO>, + <&clk CLK_GATE_GPIO_DB>; + clock-names = "bus", "db"; + resets = <&rst RST_GPIO2>; + + portc: gpio-controller@0 { + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupts = <28 IRQ_TYPE_LEVEL_HIGH>; + }; + }; + + pwm: pwm@704000c000 { + compatible = "sophgo,sg2044-pwm"; + reg = <0x70 0x4000c000 0x0 0x1000>; + #pwm-cells = <3>; + clocks = <&clk CLK_GATE_APB_PWM>; + clock-names = "apb"; + resets = <&rst RST_PWM>; + status = "disabled"; + }; + + syscon: syscon@7050000000 { + compatible = "sophgo,sg2044-top-syscon", "syscon"; + reg = <0x70 0x50000000 0x0 0x1000>; + #clock-cells = <1>; + clocks = <&osc>; + }; + + pinctrl: pinctrl@7050001000 { + compatible = "sophgo,sg2044-pinctrl"; + reg = <0x70 0x50001000 0x0 0x1000>; + }; + + clk: clock-controller@7050002000 { + compatible = "sophgo,sg2044-clk"; + reg = <0x70 0x50002000 0x0 0x1000>; + #clock-cells = <1>; + clocks = <&syscon CLK_FPLL0>, <&syscon CLK_FPLL1>, + <&syscon CLK_FPLL2>, <&syscon CLK_DPLL0>, + <&syscon CLK_DPLL1>, <&syscon CLK_DPLL2>, + <&syscon CLK_DPLL3>, <&syscon CLK_DPLL4>, + <&syscon CLK_DPLL5>, <&syscon CLK_DPLL6>, + <&syscon CLK_DPLL7>, <&syscon CLK_MPLL0>, + <&syscon CLK_MPLL1>, <&syscon CLK_MPLL2>, + <&syscon CLK_MPLL3>, <&syscon CLK_MPLL4>, + <&syscon CLK_MPLL5>; + clock-names = "fpll0", "fpll1", "fpll2", "dpll0", + "dpll1", "dpll2", "dpll3", "dpll4", + "dpll5", "dpll6", "dpll7", "mpll0", + "mpll1", "mpll2", "mpll3", "mpll4", + "mpll5"; + }; + + rst: reset-controller@7050003000 { + compatible = "sophgo,sg2044-reset", + "sophgo,sg2042-reset"; + reg = <0x70 0x50003000 0x0 0x1000>; + #reset-cells = <1>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/Makefile b/arch/riscv/boot/dts/spacemit/Makefile index ac617319a574..7e2b87702571 100644 --- a/arch/riscv/boot/dts/spacemit/Makefile +++ b/arch/riscv/boot/dts/spacemit/Makefile @@ -1,2 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_SPACEMIT) += k1-bananapi-f3.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-milkv-jupiter.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-musepi-pro.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-r2s.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-rv2.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k3-pico-itx.dtb diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index 1d617b40a2d5..5790d927b93d 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -11,12 +11,328 @@ compatible = "bananapi,bpi-f3", "spacemit,k1"; aliases { + ethernet0 = ð0; + ethernet1 = ð1; serial0 = &uart0; + i2c2 = &i2c2; + i2c8 = &i2c8; }; chosen { stdout-path = "serial0"; }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "sys-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "on"; + }; + }; + + pcie_vcc_3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "PCIE_VCC3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + reg_dc_in: regulator-dc-in-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_in_12v"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vcc_4v: regulator-vcc-4v { + compatible = "regulator-fixed"; + regulator-name = "vcc_4v"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + regulator-usb3-vbus-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_VBUS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + gpio = <&gpio K1_GPIO(97) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + usb3_hub_5v: regulator-usb3-hub-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_HUB"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&combo_phy { + status = "okay"; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +ð1 { + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(115) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&i2c2_0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + vcc-supply = <&buck3_1v8>; /* EEPROM_VCC1V8 */ + pagesize = <16>; + read-only; + size = <256>; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + mac-address { + #nvmem-cell-cells = <1>; + }; + + num-macs { + }; + + serial-number { + }; + }; + }; +}; + +&i2c8 { + pinctrl-0 = <&i2c8_cfg>; + pinctrl-names = "default"; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie1 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie2 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; }; &uart0 { @@ -24,3 +340,30 @@ pinctrl-0 = <&uart0_2_cfg>; status = "okay"; }; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb2109,2817"; + reg = <0x1>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_3_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; + + hub_3_0: hub@2 { + compatible = "usb2109,817"; + reg = <0x2>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_2_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts new file mode 100644 index 000000000000..afaad59e6bce --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + * Copyright (C) 2025 Javier Martinez Canillas <javierm@redhat.com> + */ + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "Milk-V Jupiter (K1)"; + compatible = "milkv,jupiter", "spacemit,k1"; + + aliases { + ethernet0 = ð0; + ethernet1 = ð1; + serial0 = &uart0; + i2c2 = &i2c2; + i2c8 = &i2c8; + }; + + chosen { + stdout-path = "serial0"; + }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "pwr-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "default-on"; + default-state = "on"; + }; + + led2 { + label = "hdd-led"; + gpios = <&gpio K1_GPIO(92) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "disk-activity"; + }; + }; + + pcie_vcc_3v3: regulator-pcie-vcc3v3 { + compatible = "regulator-fixed"; + regulator-name = "pcie_vcc3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + reg_dc_in: regulator-dc-in-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_in_12v"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vcc_4v: regulator-vcc-4v { + compatible = "regulator-fixed"; + regulator-name = "vcc_4v"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <4000000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + reg_vcc_5v: regulator-vcc-5v { + compatible = "regulator-fixed"; + regulator-name = "vcc_5v"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-boot-on; + regulator-always-on; + vin-supply = <®_dc_in>; + }; + + regulator-usb3-vbus-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_VBUS"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + vin-supply = <®_vcc_5v>; + gpio = <&gpio K1_GPIO(97) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + usb3_hub_5v: regulator-usb3-hub-5v { + compatible = "regulator-fixed"; + regulator-name = "USB30_HUB"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <®_vcc_5v>; + gpio = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&combo_phy { + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +ð1 { + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(115) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&i2c2_0_cfg>; + pinctrl-names = "default"; + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c04"; + reg = <0x50>; + vcc-supply = <&buck3_1v8>; /* EEPROM_VCC18 */ + pagesize = <16>; + read-only; + size = <512>; + + nvmem-layout { + compatible = "onie,tlv-layout"; + + product-name { + }; + }; + }; +}; + +&i2c8 { + pinctrl-0 = <&i2c8_cfg>; + pinctrl-names = "default"; + status = "okay"; + + pmic@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64>; + vin1-supply = <®_vcc_4v>; + vin2-supply = <®_vcc_4v>; + vin3-supply = <®_vcc_4v>; + vin4-supply = <®_vcc_4v>; + vin5-supply = <®_vcc_4v>; + vin6-supply = <®_vcc_4v>; + aldoin-supply = <®_vcc_4v>; + dldoin1-supply = <&buck5>; + dldoin2-supply = <&buck5>; + + regulators { + buck1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3_1v8: buck3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4_3v3: buck4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3300000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3450000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + aldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo1 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-boot-on; + }; + + dldo2 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo4 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo5 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + dldo6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + regulator-always-on; + }; + + dldo7 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + }; + }; +}; + +&pcie1_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_3_cfg>; + status = "okay"; +}; + +&pcie1_port { + phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie1 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&pcie2_phy { + pinctrl-names = "default"; + pinctrl-0 = <&pcie2_4_cfg>; + status = "okay"; +}; + +&pcie2_port { + phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; +}; + +&pcie2 { + vpcie3v3-supply = <&pcie_vcc_3v3>; + status = "okay"; +}; + +&qspi { + pinctrl-names = "default"; + pinctrl-0 = <&qspi_cfg>; + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <26500000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + vcc-supply = <&buck4_3v3>; /* QSPI_VCC1833 */ + m25p,fast-read; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootinfo@0 { + reg = <0x0 0x10000>; + }; + private@10000 { + reg = <0x10000 0x10000>; + }; + fsbl@20000 { + reg = <0x20000 0x40000>; + }; + env@60000 { + reg = <0x60000 0x10000>; + }; + opensbi@70000 { + reg = <0x70000 0x30000>; + }; + uboot@a00000 { + reg = <0xa0000 0x760000>; + }; + }; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_2_cfg>; + status = "okay"; +}; + +&usbphy2 { + status = "okay"; +}; + +&usb_dwc3 { + dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + hub_2_0: hub@1 { + compatible = "usb2109,2817"; + reg = <0x1>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_3_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; + + hub_3_0: hub@2 { + compatible = "usb2109,817"; + reg = <0x2>; + vdd-supply = <&usb3_hub_5v>; + peer-hub = <&hub_2_0>; + reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts new file mode 100644 index 000000000000..29e333b670cf --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + * Copyright (C) 2025 SpacemiT, Inc + * Copyright (C) 2025 Troy Mitchell <troy.mitchell@linux.spacemit.com> + */ + +/dts-v1/; + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "SpacemiT MusePi Pro"; + compatible = "spacemit,musepi-pro", "spacemit,k1"; + + aliases { + ethernet0 = ð0; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0"; + }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "sys-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "on"; + }; + }; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-0 = <&gmac0_cfg>; + pinctrl-names = "default"; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&uart0 { + pinctrl-0 = <&uart0_2_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts new file mode 100644 index 000000000000..de75f6aac740 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Michael Opdenacker <michael.opdenacker@rootcommit.com> + */ + +/dts-v1/; + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "OrangePi R2S"; + compatible = "xunlong,orangepi-r2s", "spacemit,k1"; + + aliases { + serial0 = &uart0; + ethernet0 = ð0; + ethernet1 = ð1; + }; + + chosen { + stdout-path = "serial0"; + }; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + motorcomm,auto-sleep-disabled; + }; + }; +}; + +ð1 { + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(115) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + motorcomm,auto-sleep-disabled; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_2_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts new file mode 100644 index 000000000000..7b7331cb3c72 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> + * Copyright (C) 2025 Hendrik Hamerlinck <hendrik.hamerlinck@hammernet.be> + */ + +/dts-v1/; + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "OrangePi RV2"; + compatible = "xunlong,orangepi-rv2", "spacemit,k1"; + + aliases { + serial0 = &uart0; + ethernet0 = ð0; + ethernet1 = ð1; + }; + + chosen { + stdout-path = "serial0"; + }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "sys-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_LOW>; + linux,default-trigger = "heartbeat"; + default-state = "on"; + }; + }; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + motorcomm,auto-sleep-disabled; + }; + }; +}; + +ð1 { + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(115) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + motorcomm,auto-sleep-disabled; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_2_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi index a8eac5517f85..b13dcb10f4d6 100644 --- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi @@ -7,12 +7,565 @@ #define K1_PADCONF(pin, func) (((pin) << 16) | (func)) +/* Map GPIO pin to each bank's <index, offset> */ +#define K1_GPIO(x) (x / 32) (x % 32) + &pinctrl { + gmac0_cfg: gmac0-cfg { + gmac0-pins { + pinmux = <K1_PADCONF(0, 1)>, /* gmac0_rxdv */ + <K1_PADCONF(1, 1)>, /* gmac0_rx_d0 */ + <K1_PADCONF(2, 1)>, /* gmac0_rx_d1 */ + <K1_PADCONF(3, 1)>, /* gmac0_rx_clk */ + <K1_PADCONF(4, 1)>, /* gmac0_rx_d2 */ + <K1_PADCONF(5, 1)>, /* gmac0_rx_d3 */ + <K1_PADCONF(6, 1)>, /* gmac0_tx_d0 */ + <K1_PADCONF(7, 1)>, /* gmac0_tx_d1 */ + <K1_PADCONF(8, 1)>, /* gmac0_tx */ + <K1_PADCONF(9, 1)>, /* gmac0_tx_d2 */ + <K1_PADCONF(10, 1)>, /* gmac0_tx_d3 */ + <K1_PADCONF(11, 1)>, /* gmac0_tx_en */ + <K1_PADCONF(12, 1)>, /* gmac0_mdc */ + <K1_PADCONF(13, 1)>, /* gmac0_mdio */ + <K1_PADCONF(14, 1)>, /* gmac0_int_n */ + <K1_PADCONF(45, 1)>; /* gmac0_clk_ref */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac1_cfg: gmac1-cfg { + gmac1-pins { + pinmux = <K1_PADCONF(29, 1)>, /* gmac1_rxdv */ + <K1_PADCONF(30, 1)>, /* gmac1_rx_d0 */ + <K1_PADCONF(31, 1)>, /* gmac1_rx_d1 */ + <K1_PADCONF(32, 1)>, /* gmac1_rx_clk */ + <K1_PADCONF(33, 1)>, /* gmac1_rx_d2 */ + <K1_PADCONF(34, 1)>, /* gmac1_rx_d3 */ + <K1_PADCONF(35, 1)>, /* gmac1_tx_d0 */ + <K1_PADCONF(36, 1)>, /* gmac1_tx_d1 */ + <K1_PADCONF(37, 1)>, /* gmac1_tx */ + <K1_PADCONF(38, 1)>, /* gmac1_tx_d2 */ + <K1_PADCONF(39, 1)>, /* gmac1_tx_d3 */ + <K1_PADCONF(40, 1)>, /* gmac1_tx_en */ + <K1_PADCONF(41, 1)>, /* gmac1_mdc */ + <K1_PADCONF(42, 1)>, /* gmac1_mdio */ + <K1_PADCONF(43, 1)>, /* gmac1_int_n */ + <K1_PADCONF(46, 1)>; /* gmac1_clk_ref */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + i2c2_0_cfg: i2c2-0-cfg { + i2c2-0-pins { + pinmux = <K1_PADCONF(84, 4)>, /* I2C2_SCL */ + <K1_PADCONF(85, 4)>; /* I2C2_SDA */ + }; + }; + + i2c8_cfg: i2c8-cfg { + i2c8-0-pins { + pinmux = <K1_PADCONF(93, 0)>, /* PWR_SCL */ + <K1_PADCONF(94, 0)>; /* PWR_SDA */ + }; + }; + + qspi_cfg: qspi-cfg { + qspi-pins { + pinmux = <K1_PADCONF(98, 0)>, /* QSPI_DATA3 */ + <K1_PADCONF(99, 0)>, /* QSPI_DATA2 */ + <K1_PADCONF(100, 0)>, /* QSPI_DATA1 */ + <K1_PADCONF(101, 0)>, /* QSPI_DATA0 */ + <K1_PADCONF(102, 0)>; /* QSPI_CLK */ + + bias-disable; + drive-strength = <19>; + power-source = <3300>; + }; + + qspi-cs1-pins { + pinmux = <K1_PADCONF(103, 0)>; /* QSPI_CS1 */ + bias-pull-up = <0>; + drive-strength = <19>; + power-source = <3300>; + }; + }; + + /omit-if-no-ref/ + uart0_0_cfg: uart0-0-cfg { + uart0-0-pins { + pinmux = <K1_PADCONF(104, 3)>, /* uart0_txd */ + <K1_PADCONF(105, 3)>; /* uart0_rxd */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ + uart0_1_cfg: uart0-1-cfg { + uart0-1-pins { + pinmux = <K1_PADCONF(108, 1)>, /* uart0_txd */ + <K1_PADCONF(80, 3)>; /* uart0_rxd */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ uart0_2_cfg: uart0-2-cfg { uart0-2-pins { - pinmux = <K1_PADCONF(68, 2)>, - <K1_PADCONF(69, 2)>; + pinmux = <K1_PADCONF(68, 2)>, /* uart0_txd */ + <K1_PADCONF(69, 2)>; /* uart0_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart2_0_cfg: uart2-0-cfg { + uart2-0-pins { + pinmux = <K1_PADCONF(21, 1)>, /* uart2_txd */ + <K1_PADCONF(22, 1)>; /* uart2_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart2_0_cts_rts_cfg: uart2-0-cts-rts-cfg { + uart2-0-pins { + pinmux = <K1_PADCONF(23, 1)>, /* uart2_cts */ + <K1_PADCONF(24, 1)>; /* uart2_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_0_cfg: uart3-0-cfg { + uart3-0-pins { + pinmux = <K1_PADCONF(81, 2)>, /* uart3_txd */ + <K1_PADCONF(82, 2)>; /* uart3_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_0_cts_rts_cfg: uart3-0-cts-rts-cfg { + uart3-0-pins { + pinmux = <K1_PADCONF(83, 2)>, /* uart3_cts */ + <K1_PADCONF(84, 2)>; /* uart3_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_1_cfg: uart3-1-cfg { + uart3-1-pins { + pinmux = <K1_PADCONF(18, 2)>, /* uart3_txd */ + <K1_PADCONF(19, 2)>; /* uart3_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_1_cts_rts_cfg: uart3-1-cts-rts-cfg { + uart3-1-pins { + pinmux = <K1_PADCONF(20, 2)>, /* uart3_cts */ + <K1_PADCONF(21, 2)>; /* uart3_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_2_cfg: uart3-2-cfg { + uart3-2-pins { + pinmux = <K1_PADCONF(53, 4)>, /* uart3_txd */ + <K1_PADCONF(54, 4)>; /* uart3_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart3_2_cts_rts_cfg: uart3-2-cts-rts-cfg { + uart3-2-pins { + pinmux = <K1_PADCONF(55, 4)>, /* uart3_cts */ + <K1_PADCONF(56, 4)>; /* uart3_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_0_cfg: uart4-0-cfg { + uart4-0-pins { + pinmux = <K1_PADCONF(100, 4)>, /* uart4_txd */ + <K1_PADCONF(101, 4)>; /* uart4_rxd */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ + uart4_1_cfg: uart4-1-cfg { + uart4-1-pins { + pinmux = <K1_PADCONF(83, 3)>, /* uart4_txd */ + <K1_PADCONF(84, 3)>; /* uart4_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_1_cts_rts_cfg: uart4-1-cts-rts-cfg { + uart4-1-pins { + pinmux = <K1_PADCONF(81, 3)>, /* uart4_cts */ + <K1_PADCONF(82, 3)>; /* uart4_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_2_cfg: uart4-2-cfg { + uart4-2-pins { + pinmux = <K1_PADCONF(23, 2)>, /* uart4_txd */ + <K1_PADCONF(24, 2)>; /* uart4_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_3_cfg: uart4-3-cfg { + uart4-3-pins { + pinmux = <K1_PADCONF(33, 2)>, /* uart4_txd */ + <K1_PADCONF(34, 2)>; /* uart4_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_3_cts_rts_cfg: uart4-3-cts-rts-cfg { + uart4-3-pins { + pinmux = <K1_PADCONF(35, 2)>, /* uart4_cts */ + <K1_PADCONF(36, 2)>; /* uart4_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_4_cfg: uart4-4-cfg { + uart4-4-pins { + pinmux = <K1_PADCONF(111, 4)>, /* uart4_txd */ + <K1_PADCONF(112, 4)>; /* uart4_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart4_4_cts_rts_cfg: uart4-4-cts-rts-cfg { + uart4-4-pins { + pinmux = <K1_PADCONF(113, 4)>, /* uart4_cts */ + <K1_PADCONF(114, 4)>; /* uart4_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_0_cfg: uart5-0-cfg { + uart5-0-pins { + pinmux = <K1_PADCONF(102, 3)>, /* uart5_txd */ + <K1_PADCONF(103, 3)>; /* uart5_rxd */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ + uart5_1_cfg: uart5-1-cfg { + uart5-1-pins { + pinmux = <K1_PADCONF(25, 2)>, /* uart5_txd */ + <K1_PADCONF(26, 2)>; /* uart5_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_1_cts_rts_cfg: uart5-1-cts-rts-cfg { + uart5-1-pins { + pinmux = <K1_PADCONF(27, 2)>, /* uart5_cts */ + <K1_PADCONF(28, 2)>; /* uart5_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_2_cfg: uart5-2-cfg { + uart5-2-pins { + pinmux = <K1_PADCONF(42, 2)>, /* uart5_txd */ + <K1_PADCONF(43, 2)>; /* uart5_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_2_cts_rts_cfg: uart5-2-cts-rts-cfg { + uart5-2-pins { + pinmux = <K1_PADCONF(44, 2)>, /* uart5_cts */ + <K1_PADCONF(45, 2)>; /* uart5_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_3_cfg: uart5-3-cfg { + uart5-3-pins { + pinmux = <K1_PADCONF(70, 4)>, /* uart5_txd */ + <K1_PADCONF(71, 4)>; /* uart5_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart5_3_cts_rts_cfg: uart5-3-cts-rts-cfg { + uart5-3-pins { + pinmux = <K1_PADCONF(72, 4)>, /* uart5_cts */ + <K1_PADCONF(73, 4)>; /* uart5_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart6_0_cfg: uart6-0-cfg { + uart6-0-pins { + pinmux = <K1_PADCONF(86, 2)>, /* uart6_txd */ + <K1_PADCONF(87, 2)>; /* uart6_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart6_0_cts_rts_cfg: uart6-0-cts-rts-cfg { + uart6-0-pins { + pinmux = <K1_PADCONF(85, 2)>, /* uart6_cts */ + <K1_PADCONF(90, 2)>; /* uart6_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart6_1_cfg: uart6-1-cfg { + uart6-1-pins { + pinmux = <K1_PADCONF(0, 2)>, /* uart6_txd */ + <K1_PADCONF(1, 2)>; /* uart6_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart6_1_cts_rts_cfg: uart6-1-cts-rts-cfg { + uart6-1-pins { + pinmux = <K1_PADCONF(2, 2)>, /* uart6_cts */ + <K1_PADCONF(3, 2)>; /* uart6_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart6_2_cfg: uart6-2-cfg { + uart6-2-pins { + pinmux = <K1_PADCONF(56, 2)>, /* uart6_txd */ + <K1_PADCONF(57, 2)>; /* uart6_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart7_0_cfg: uart7-0-cfg { + uart7-0-pins { + pinmux = <K1_PADCONF(88, 2)>, /* uart7_txd */ + <K1_PADCONF(89, 2)>; /* uart7_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart7_1_cfg: uart7-1-cfg { + uart7-1-pins { + pinmux = <K1_PADCONF(4, 2)>, /* uart7_txd */ + <K1_PADCONF(5, 2)>; /* uart7_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart7_1_cts_rts_cfg: uart7-1-cts-rts-cfg { + uart7-1-pins { + pinmux = <K1_PADCONF(6, 2)>, /* uart7_cts */ + <K1_PADCONF(7, 2)>; /* uart7_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart8_0_cfg: uart8-0-cfg { + uart8-0-pins { + pinmux = <K1_PADCONF(82, 4)>, /* uart8_txd */ + <K1_PADCONF(83, 4)>; /* uart8_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart8_1_cfg: uart8-1-cfg { + uart8-1-pins { + pinmux = <K1_PADCONF(8, 2)>, /* uart8_txd */ + <K1_PADCONF(9, 2)>; /* uart8_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart8_1_cts_rts_cfg: uart8-1-cts-rts-cfg { + uart8-1-pins { + pinmux = <K1_PADCONF(10, 2)>, /* uart8_cts */ + <K1_PADCONF(11, 2)>; /* uart8_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart8_2_cfg: uart8-2-cfg { + uart8-2-pins { + pinmux = <K1_PADCONF(75, 4)>, /* uart8_txd */ + <K1_PADCONF(76, 4)>; /* uart8_rxd */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ + uart8_2_cts_rts_cfg: uart8-2-cts-rts-cfg { + uart8-2-pins { + pinmux = <K1_PADCONF(77, 4)>, /* uart8_cts */ + <K1_PADCONF(78, 4)>; /* uart8_rts */ + power-source = <3300>; + bias-pull-up = <0>; + drive-strength = <19>; + }; + }; + + /omit-if-no-ref/ + uart9_0_cfg: uart9-0-cfg { + uart9-0-pins { + pinmux = <K1_PADCONF(12, 2)>, /* uart9_txd */ + <K1_PADCONF(13, 2)>; /* uart9_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart9_1_cfg: uart9-1-cfg { + uart9-1-pins { + pinmux = <K1_PADCONF(116, 3)>, /* uart9_txd */ + <K1_PADCONF(117, 3)>; /* uart9_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart9_1_cts_rts_cfg: uart9-1-cts-rts-cfg { + uart9-1-pins { + pinmux = <K1_PADCONF(110, 3)>, /* uart9_cts */ + <K1_PADCONF(115, 3)>; /* uart9_rts */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + /omit-if-no-ref/ + uart9_2_cfg: uart9-2-cfg { + uart9-2-pins { + pinmux = <K1_PADCONF(72, 2)>, /* uart9_txd */ + <K1_PADCONF(73, 2)>; /* uart9_rxd */ + bias-pull-up = <0>; + drive-strength = <32>; + }; + }; + + pcie0_3_cfg: pcie0-3-cfg { + pcie0-3-pins { + pinmux = <K1_PADCONF(54, 3)>, /* PERST# */ + <K1_PADCONF(55, 3)>, /* WAKE# */ + <K1_PADCONF(53, 3)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + pcie1_3_cfg: pcie1-3-cfg { + pcie1-3-pins { + pinmux = <K1_PADCONF(59, 4)>, /* PERST# */ + <K1_PADCONF(60, 4)>, /* WAKE# */ + <K1_PADCONF(61, 4)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + pcie2_4_cfg: pcie2-4-cfg { + pcie2-4-pins { + pinmux = <K1_PADCONF(62, 4)>, /* PERST# */ + <K1_PADCONF(112, 3)>, /* WAKE# */ + <K1_PADCONF(117, 4)>; /* CLKREQ# */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + pwm14_1_cfg: pwm14-1-cfg { + pwm14-1-pins { + pinmux = <K1_PADCONF(44, 4)>; bias-pull-up = <0>; drive-strength = <32>; }; diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi index c670ebf8fa12..f0bad6855c97 100644 --- a/arch/riscv/boot/dts/spacemit/k1.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1.dtsi @@ -3,6 +3,9 @@ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name> */ +#include <dt-bindings/clock/spacemit,k1-syscon.h> +#include <dt-bindings/phy/phy.h> + /dts-v1/; / { #address-cells = <2>; @@ -51,9 +54,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <0>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -81,9 +84,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <1>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -111,9 +114,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <2>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -141,9 +144,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <3>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -171,9 +174,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <4>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -201,9 +204,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <5>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -231,9 +234,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <6>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -261,9 +264,9 @@ compatible = "spacemit,x60", "riscv"; device_type = "cpu"; reg = <7>; - riscv,isa = "rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; + riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicbom", + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom", "zicbop", "zicboz", "zicntr", "zicond", "zicsr", "zifencei", "zihintpause", "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zkt", "zvfh", "zvkt", @@ -306,6 +309,36 @@ }; }; + clocks { + vctcxo_1m: clock-1m { + compatible = "fixed-clock"; + clock-frequency = <1000000>; + clock-output-names = "vctcxo_1m"; + #clock-cells = <0>; + }; + + vctcxo_24m: clock-24m { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "vctcxo_24m"; + #clock-cells = <0>; + }; + + vctcxo_3m: clock-3m { + compatible = "fixed-clock"; + clock-frequency = <3000000>; + clock-output-names = "vctcxo_3m"; + #clock-cells = <0>; + }; + + osc_32k: clock-32k { + compatible = "fixed-clock"; + clock-frequency = <32000>; + clock-output-names = "osc_32k"; + #clock-cells = <0>; + }; + }; + soc { compatible = "simple-bus"; interrupt-parent = <&plic>; @@ -314,99 +347,432 @@ dma-noncoherent; ranges; - uart0: serial@d4017000 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017000 0x0 0x100>; - interrupts = <42>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + syscon_rcpu: system-controller@c0880000 { + compatible = "spacemit,k1-syscon-rcpu"; + reg = <0x0 0xc0880000 0x0 0x2048>; + #reset-cells = <1>; + }; + + syscon_rcpu2: system-controller@c0888000 { + compatible = "spacemit,k1-syscon-rcpu2"; + reg = <0x0 0xc0888000 0x0 0x28>; + #reset-cells = <1>; + }; + + usbphy2: phy@c0a30000 { + compatible = "spacemit,k1-usb2-phy"; + reg = <0x0 0xc0a30000 0x0 0x200>; + clocks = <&syscon_apmu CLK_USB30>; + #phy-cells = <0>; + status = "disabled"; + }; + + combo_phy: phy@c0b10000 { + compatible = "spacemit,k1-combo-phy"; + reg = <0x0 0xc0b10000 0x0 0x1000>; + clocks = <&vctcxo_24m>, + <&syscon_apmu CLK_PCIE0_DBI>, + <&syscon_apmu CLK_PCIE0_MASTER>, + <&syscon_apmu CLK_PCIE0_SLAVE>; + clock-names = "refclk", + "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE0_GLOBAL>, + <&syscon_apmu RESET_PCIE0_DBI>, + <&syscon_apmu RESET_PCIE0_MASTER>, + <&syscon_apmu RESET_PCIE0_SLAVE>; + reset-names = "phy", + "dbi", + "mstr", + "slv"; + #phy-cells = <1>; + spacemit,apmu = <&syscon_apmu>; + status = "disabled"; + }; + + pcie1_phy: phy@c0c10000 { + compatible = "spacemit,k1-pcie-phy"; + reg = <0x0 0xc0c10000 0x0 0x1000>; + clocks = <&vctcxo_24m>; + clock-names = "refclk"; + resets = <&syscon_apmu RESET_PCIE1_GLOBAL>; + reset-names = "phy"; + #phy-cells = <0>; status = "disabled"; }; - uart2: serial@d4017100 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017100 0x0 0x100>; - interrupts = <44>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + pcie2_phy: phy@c0d10000 { + compatible = "spacemit,k1-pcie-phy"; + reg = <0x0 0xc0d10000 0x0 0x1000>; + clocks = <&vctcxo_24m>; + clock-names = "refclk"; + resets = <&syscon_apmu RESET_PCIE2_GLOBAL>; + reset-names = "phy"; + #phy-cells = <0>; status = "disabled"; }; - uart3: serial@d4017200 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017200 0x0 0x100>; - interrupts = <45>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + i2c0: i2c@d4010800 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4010800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI0>, + <&syscon_apbc CLK_TWSI0_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI0>; + interrupts = <36>; status = "disabled"; }; - uart4: serial@d4017300 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017300 0x0 0x100>; - interrupts = <46>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + i2c1: i2c@d4011000 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4011000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI1>, + <&syscon_apbc CLK_TWSI1_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI1>; + interrupts = <37>; status = "disabled"; }; - uart5: serial@d4017400 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017400 0x0 0x100>; - interrupts = <47>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + i2c2: i2c@d4012000 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4012000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI2>, + <&syscon_apbc CLK_TWSI2_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI2>; + interrupts = <38>; status = "disabled"; }; - uart6: serial@d4017500 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017500 0x0 0x100>; - interrupts = <48>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + i2c4: i2c@d4012800 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4012800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI4>, + <&syscon_apbc CLK_TWSI4_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI4>; + interrupts = <40>; status = "disabled"; }; - uart7: serial@d4017600 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017600 0x0 0x100>; - interrupts = <49>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + i2c5: i2c@d4013800 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4013800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI5>, + <&syscon_apbc CLK_TWSI5_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI5>; + interrupts = <41>; + status = "disabled"; + }; + + syscon_apbc: system-controller@d4015000 { + compatible = "spacemit,k1-syscon-apbc"; + reg = <0x0 0xd4015000 0x0 0x1000>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, + <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", + "vctcxo_24m"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + i2c6: i2c@d4018800 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd4018800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI6>, + <&syscon_apbc CLK_TWSI6_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI6>; + interrupts = <70>; status = "disabled"; }; - uart8: serial@d4017700 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017700 0x0 0x100>; - interrupts = <50>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + gpio: gpio@d4019000 { + compatible = "spacemit,k1-gpio"; + reg = <0x0 0xd4019000 0x0 0x100>; + clocks = <&syscon_apbc CLK_GPIO>, + <&syscon_apbc CLK_GPIO_BUS>; + clock-names = "core", "bus"; + gpio-controller; + #gpio-cells = <3>; + interrupts = <58>; + interrupt-parent = <&plic>; + interrupt-controller; + #interrupt-cells = <3>; + gpio-ranges = <&pinctrl 0 0 0 32>, + <&pinctrl 1 0 32 32>, + <&pinctrl 2 0 64 32>, + <&pinctrl 3 0 96 32>; + }; + + pwm0: pwm@d401a000 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a000 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM0>; + resets = <&syscon_apbc RESET_PWM0>; status = "disabled"; }; - uart9: serial@d4017800 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xd4017800 0x0 0x100>; - interrupts = <51>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; + pwm1: pwm@d401a400 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a400 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM1>; + resets = <&syscon_apbc RESET_PWM1>; + status = "disabled"; + }; + + pwm2: pwm@d401a800 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401a800 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM2>; + resets = <&syscon_apbc RESET_PWM2>; + status = "disabled"; + }; + + pwm3: pwm@d401ac00 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401ac00 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM3>; + resets = <&syscon_apbc RESET_PWM3>; + status = "disabled"; + }; + + pwm4: pwm@d401b000 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b000 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM4>; + resets = <&syscon_apbc RESET_PWM4>; + status = "disabled"; + }; + + pwm5: pwm@d401b400 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b400 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM5>; + resets = <&syscon_apbc RESET_PWM5>; + status = "disabled"; + }; + + pwm6: pwm@d401b800 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401b800 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM6>; + resets = <&syscon_apbc RESET_PWM6>; + status = "disabled"; + }; + + pwm7: pwm@d401bc00 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd401bc00 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM7>; + resets = <&syscon_apbc RESET_PWM7>; + status = "disabled"; + }; + + i2c7: i2c@d401d000 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd401d000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI7>, + <&syscon_apbc CLK_TWSI7_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI7>; + interrupts = <18>; + status = "disabled"; + }; + + i2c8: i2c@d401d800 { + compatible = "spacemit,k1-i2c"; + reg = <0x0 0xd401d800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&syscon_apbc CLK_TWSI8>, + <&syscon_apbc CLK_TWSI8_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_TWSI8>; + interrupts = <19>; status = "disabled"; }; pinctrl: pinctrl@d401e000 { compatible = "spacemit,k1-pinctrl"; - reg = <0x0 0xd401e000 0x0 0x400>; + reg = <0x0 0xd401e000 0x0 0x1000>; + clocks = <&syscon_apbc CLK_AIB>, + <&syscon_apbc CLK_AIB_BUS>; + clock-names = "func", "bus"; + spacemit,apbc = <&syscon_apbc>; + }; + + pwm8: pwm@d4020000 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020000 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM8>; + resets = <&syscon_apbc RESET_PWM8>; + status = "disabled"; + }; + + pwm9: pwm@d4020400 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020400 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM9>; + resets = <&syscon_apbc RESET_PWM9>; + status = "disabled"; + }; + + pwm10: pwm@d4020800 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020800 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM10>; + resets = <&syscon_apbc RESET_PWM10>; + status = "disabled"; + }; + + pwm11: pwm@d4020c00 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4020c00 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM11>; + resets = <&syscon_apbc RESET_PWM11>; + status = "disabled"; + }; + + pwm12: pwm@d4021000 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021000 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM12>; + resets = <&syscon_apbc RESET_PWM12>; + status = "disabled"; + }; + + pwm13: pwm@d4021400 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021400 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM13>; + resets = <&syscon_apbc RESET_PWM13>; + status = "disabled"; + }; + + pwm14: pwm@d4021800 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021800 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM14>; + resets = <&syscon_apbc RESET_PWM14>; + status = "disabled"; + }; + + pwm15: pwm@d4021c00 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4021c00 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM15>; + resets = <&syscon_apbc RESET_PWM15>; + status = "disabled"; + }; + + pwm16: pwm@d4022000 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022000 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM16>; + resets = <&syscon_apbc RESET_PWM16>; + status = "disabled"; + }; + + pwm17: pwm@d4022400 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022400 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM17>; + resets = <&syscon_apbc RESET_PWM17>; + status = "disabled"; + }; + + pwm18: pwm@d4022800 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022800 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM18>; + resets = <&syscon_apbc RESET_PWM18>; + status = "disabled"; + }; + + pwm19: pwm@d4022c00 { + compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm"; + reg = <0x0 0xd4022c00 0x0 0x10>; + #pwm-cells = <3>; + clocks = <&syscon_apbc CLK_PWM19>; + resets = <&syscon_apbc RESET_PWM19>; + status = "disabled"; + }; + + syscon_mpmu: system-controller@d4050000 { + compatible = "spacemit,k1-syscon-mpmu"; + reg = <0x0 0xd4050000 0x0 0x209c>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, + <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", + "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + pll: clock-controller@d4090000 { + compatible = "spacemit,k1-pll"; + reg = <0x0 0xd4090000 0x0 0x1000>; + clocks = <&vctcxo_24m>; + spacemit,mpmu = <&syscon_mpmu>; + #clock-cells = <1>; + }; + + syscon_apmu: system-controller@d4282800 { + compatible = "spacemit,k1-syscon-apmu"; + reg = <0x0 0xd4282800 0x0 0x400>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, + <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", + "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; }; plic: interrupt-controller@e0000000 { @@ -439,14 +805,412 @@ <&cpu7_intc 3>, <&cpu7_intc 7>; }; - sec_uart1: serial@f0612000 { - compatible = "spacemit,k1-uart", "intel,xscale-uart"; - reg = <0x0 0xf0612000 0x0 0x100>; - interrupts = <43>; - clock-frequency = <14857000>; - reg-shift = <2>; - reg-io-width = <4>; - status = "reserved"; /* for TEE usage */ + syscon_apbc2: system-controller@f0610000 { + compatible = "spacemit,k1-syscon-apbc2"; + reg = <0x0 0xf0610000 0x0 0x20>; + #reset-cells = <1>; + }; + + /* sec_i2c3: 0xf0614000, not available from Linux */ + + camera-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, + <0x0 0x80000000 0x1 0x00000000 0x1 0x80000000>; + }; + + dma-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, + <0x1 0x00000000 0x1 0x80000000 0x3 0x00000000>; + + pdma: dma-controller@d4000000 { + compatible = "spacemit,k1-pdma"; + reg = <0x0 0xd4000000 0x0 0x4000>; + clocks = <&syscon_apmu CLK_DMA>; + resets = <&syscon_apmu RESET_DMA>; + interrupts = <72>; + dma-channels = <16>; + #dma-cells= <1>; + status = "disabled"; + }; + + uart0: serial@d4017000 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017000 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART0>, + <&syscon_apbc CLK_UART0_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART0>; + interrupts = <42>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart2: serial@d4017100 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017100 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART2>, + <&syscon_apbc CLK_UART2_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART2>; + interrupts = <44>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart3: serial@d4017200 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017200 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART3>, + <&syscon_apbc CLK_UART3_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART3>; + interrupts = <45>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart4: serial@d4017300 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017300 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART4>, + <&syscon_apbc CLK_UART4_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART4>; + interrupts = <46>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart5: serial@d4017400 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017400 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART5>, + <&syscon_apbc CLK_UART5_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART5>; + interrupts = <47>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart6: serial@d4017500 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017500 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART6>, + <&syscon_apbc CLK_UART6_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART6>; + interrupts = <48>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart7: serial@d4017600 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017600 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART7>, + <&syscon_apbc CLK_UART7_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART7>; + interrupts = <49>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart8: serial@d4017700 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017700 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART8>, + <&syscon_apbc CLK_UART8_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART8>; + interrupts = <50>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart9: serial@d4017800 { + compatible = "spacemit,k1-uart", + "intel,xscale-uart"; + reg = <0x0 0xd4017800 0x0 0x100>; + clocks = <&syscon_apbc CLK_UART9>, + <&syscon_apbc CLK_UART9_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_UART9>; + interrupts = <51>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + qspi: spi@d420c000 { + compatible = "spacemit,k1-qspi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0 0xd420c000 0x0 0x1000>, + <0x0 0xb8000000 0x0 0xc00000>; + reg-names = "QuadSPI", "QuadSPI-memory"; + clocks = <&syscon_apmu CLK_QSPI_BUS>, + <&syscon_apmu CLK_QSPI>; + clock-names = "qspi_en", "qspi"; + resets = <&syscon_apmu RESET_QSPI>, + <&syscon_apmu RESET_QSPI_BUS>; + interrupts = <117>; + status = "disabled"; + }; + + /* sec_uart1: 0xf0612000, not available from Linux */ + }; + + multimedia-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, + <0x0 0x80000000 0x1 0x00000000 0x3 0x80000000>; + }; + + network-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, + <0x0 0x80000000 0x1 0x00000000 0x0 0x80000000>; + + eth0: ethernet@cac80000 { + compatible = "spacemit,k1-emac"; + reg = <0x0 0xcac80000 0x0 0x420>; + clocks = <&syscon_apmu CLK_EMAC0_BUS>; + interrupts = <131>; + mac-address = [ 00 00 00 00 00 00 ]; + resets = <&syscon_apmu RESET_EMAC0>; + spacemit,apmu = <&syscon_apmu 0x3e4>; + status = "disabled"; + }; + + eth1: ethernet@cac81000 { + compatible = "spacemit,k1-emac"; + reg = <0x0 0xcac81000 0x0 0x420>; + clocks = <&syscon_apmu CLK_EMAC1_BUS>; + interrupts = <133>; + mac-address = [ 00 00 00 00 00 00 ]; + resets = <&syscon_apmu RESET_EMAC1>; + spacemit,apmu = <&syscon_apmu 0x3ec>; + status = "disabled"; + }; + }; + + pcie-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>, + <0x0 0xb8000000 0x1 0x38000000 0x3 0x48000000>; + + pcie0: pcie@ca000000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca000000 0x0 0x00001000>, + <0x0 0xca300000 0x0 0x0001ff24>, + <0x0 0x8f000000 0x0 0x00002000>, + <0x0 0xc0b20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x8f002000 0x0 0x00100000>, + <0x02000000 0x0 0x80000000 0x0 0x80000000 0x0 0x0f000000>; + interrupts = <141>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE0_DBI>, + <&syscon_apmu CLK_PCIE0_MASTER>, + <&syscon_apmu CLK_PCIE0_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE0_DBI>, + <&syscon_apmu RESET_PCIE0_MASTER>, + <&syscon_apmu RESET_PCIE0_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x03cc>; + status = "disabled"; + + pcie0_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie1: pcie@ca400000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca400000 0x0 0x00001000>, + <0x0 0xca700000 0x0 0x0001ff24>, + <0x0 0x9f000000 0x0 0x00002000>, + <0x0 0xc0c20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x9f002000 0x0 0x00100000>, + <0x02000000 0x0 0x90000000 0x0 0x90000000 0x0 0x0f000000>; + interrupts = <142>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE1_DBI>, + <&syscon_apmu CLK_PCIE1_MASTER>, + <&syscon_apmu CLK_PCIE1_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE1_DBI>, + <&syscon_apmu RESET_PCIE1_MASTER>, + <&syscon_apmu RESET_PCIE1_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x3d4>; + status = "disabled"; + + pcie1_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie2: pcie@ca800000 { + device_type = "pci"; + compatible = "spacemit,k1-pcie"; + reg = <0x0 0xca800000 0x0 0x00001000>, + <0x0 0xcab00000 0x0 0x0001ff24>, + <0x0 0xb7000000 0x0 0x00002000>, + <0x0 0xc0d20000 0x0 0x00001000>; + reg-names = "dbi", + "atu", + "config", + "link"; + linux,pci-domain = <2>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0xb7002000 0x0 0x00100000>, + <0x42000000 0x0 0xa0000000 0x0 0xa0000000 0x0 0x10000000>, + <0x02000000 0x0 0xb0000000 0x0 0xb0000000 0x0 0x07000000>; + interrupts = <143>; + interrupt-names = "msi"; + clocks = <&syscon_apmu CLK_PCIE2_DBI>, + <&syscon_apmu CLK_PCIE2_MASTER>, + <&syscon_apmu CLK_PCIE2_SLAVE>; + clock-names = "dbi", + "mstr", + "slv"; + resets = <&syscon_apmu RESET_PCIE2_DBI>, + <&syscon_apmu RESET_PCIE2_MASTER>, + <&syscon_apmu RESET_PCIE2_SLAVE>; + reset-names = "dbi", + "mstr", + "slv"; + spacemit,apmu = <&syscon_apmu 0x3dc>; + status = "disabled"; + + pcie2_port: pcie@0 { + device_type = "pci"; + compatible = "pciclass,0604"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + }; + + storage-bus { + compatible = "simple-bus"; + ranges; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges = <0x0 0x00000000 0x0 0x00000000 0x0 0x80000000>; + + usb_dwc3: usb@c0a00000 { + compatible = "spacemit,k1-dwc3"; + reg = <0x0 0xc0a00000 0x0 0x10000>; + clocks = <&syscon_apmu CLK_USB30>; + clock-names = "usbdrd30"; + interrupts = <125>; + phys = <&usbphy2>, <&combo_phy PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi"; + resets = <&syscon_apmu RESET_USB30_AHB>, + <&syscon_apmu RESET_USB30_VCC>, + <&syscon_apmu RESET_USB30_PHY>; + reset-names = "ahb", "vcc", "phy"; + reset-delay = <2>; + snps,hsphy_interface = "utmi"; + snps,dis_enblslpm_quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis_rxdet_inp3_quirk; + status = "disabled"; + }; + + emmc: mmc@d4281000 { + compatible = "spacemit,k1-sdhci"; + reg = <0x0 0xd4281000 0x0 0x200>; + clocks = <&syscon_apmu CLK_SDH_AXI>, + <&syscon_apmu CLK_SDH2>; + clock-names = "core", "io"; + resets = <&syscon_apmu RESET_SDH_AXI>, + <&syscon_apmu RESET_SDH2>; + reset-names = "axi", "sdh"; + interrupts = <101>; + status = "disabled"; + }; }; }; }; diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts new file mode 100644 index 000000000000..4486dc1fe114 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Guodong Xu <guodong@riscstar.com> + */ +#include <dt-bindings/gpio/gpio.h> + +#include "k3.dtsi" +#include "k3-pinctrl.dtsi" + +/ { + model = "SpacemiT K3 Pico-ITX"; + compatible = "spacemit,k3-pico-itx", "spacemit,k3"; + + aliases { + ethernet0 = ð0; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0"; + }; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x00000000 0x4 0x00000000>; + }; + + reg_aux_vcc5v: regulator-aux-vcc5v { + compatible = "regulator-fixed"; + regulator-name = "AUX_VCC5V"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; +}; + +&i2c8 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_cfg>; + status = "okay"; + + p1@41 { + compatible = "spacemit,p1"; + reg = <0x41>; + interrupts = <64 IRQ_TYPE_LEVEL_HIGH>; + vin1-supply = <®_aux_vcc5v>; + vin2-supply = <®_aux_vcc5v>; + vin3-supply = <®_aux_vcc5v>; + vin4-supply = <®_aux_vcc5v>; + vin5-supply = <®_aux_vcc5v>; + vin6-supply = <®_aux_vcc5v>; + aldoin-supply = <®_aux_vcc5v>; + dldoin1-supply = <&buck4>; + dldoin2-supply = <&buck4>; + + regulators { + buck1: buck1 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck2: buck2 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck3: buck3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck4: buck4 { + regulator-min-microvolt = <2100000>; + regulator-max-microvolt = <2100000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck5: buck5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + buck6: buck6 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <500000>; + regulator-ramp-delay = <5000>; + regulator-always-on; + }; + + aldo1: aldo1 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo2: aldo2 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + aldo3: aldo3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + + aldo4: aldo4 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo1: dldo1 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo2: dldo2 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo3: dldo3 { + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo4: dldo4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + }; + + dldo5: dldo5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo6: dldo6 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + + dldo7: dldo7 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + }; + }; + }; +}; + +ð0 { + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_rgmii_0_cfg>, <&gmac0_phy_0_cfg>; + phy-mode = "rgmii-id"; + phy-handle = <&phy0>; + status = "okay"; + + mdio { + phy0: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <10000>; + }; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_0_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi new file mode 100644 index 000000000000..23899d3f308a --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (c) 2026 Yixun Lan <dlan@kernel.org> + */ + +#include <dt-bindings/gpio/gpio.h> + +#define K3_PADCONF(pin, func) (((pin) << 16) | (func)) + +/* Map GPIO pin to each bank's <index, offset> */ +#define K3_GPIO(x) (x / 32) (x % 32) + +&pinctrl { + gmac0_rgmii_0_cfg: gmac0-rgmii-0-cfg { + gmac0-rgmii-0-pins { + pinmux = <K3_PADCONF(0, 1)>, /* gmac0_rxdv */ + <K3_PADCONF(1, 1)>, /* gmac0_rx_d0 */ + <K3_PADCONF(2, 1)>, /* gmac0_rx_d1 */ + <K3_PADCONF(3, 1)>, /* gmac0_rx_clk */ + <K3_PADCONF(4, 1)>, /* gmac0_rx_d2 */ + <K3_PADCONF(5, 1)>, /* gmac0_rx_d3 */ + <K3_PADCONF(6, 1)>, /* gmac0_tx_d0 */ + <K3_PADCONF(7, 1)>, /* gmac0_tx_d1 */ + <K3_PADCONF(8, 1)>, /* gmac0_tx_clk */ + <K3_PADCONF(9, 1)>, /* gmac0_tx_d2 */ + <K3_PADCONF(10, 1)>, /* gmac0_tx_d3 */ + <K3_PADCONF(11, 1)>, /* gmac0_tx_en */ + <K3_PADCONF(12, 1)>, /* gmac0_mdc */ + <K3_PADCONF(13, 1)>; /* gmac0_mdio */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + + }; + + gmac0_phy_0_cfg: gmac0-phy-0-cfg { + gmac0-phy-0-pins { + pinmux = <K3_PADCONF(14, 1)>; /* gmac0_int */ + + bias-disable; + drive-strength = <25>; + power-source = <1800>; + }; + }; + + /omit-if-no-ref/ + i2c8_cfg: i2c8-cfg { + i2c8-pins { + pinmux = <K3_PADCONF(128, 0)>, /* i2c8 scl */ + <K3_PADCONF(129, 0)>; /* i2c8 sda */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; + + /omit-if-no-ref/ + uart0_0_cfg: uart0-0-cfg { + uart0-0-pins { + pinmux = <K3_PADCONF(149, 2)>, /* uart0 tx */ + <K3_PADCONF(150, 2)>; /* uart0 rx */ + + bias-pull-up = <0>; + drive-strength = <25>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi new file mode 100644 index 000000000000..815debd16409 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k3.dtsi @@ -0,0 +1,927 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2026 SpacemiT (Hangzhou) Technology Co. Ltd + * Copyright (c) 2026 Guodong Xu <guodong@riscstar.com> + */ + +#include <dt-bindings/clock/spacemit,k3-clocks.h> +#include <dt-bindings/reset/spacemit,k3-resets.h> +#include <dt-bindings/interrupt-controller/irq.h> + +/dts-v1/; + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "SpacemiT K3"; + compatible = "spacemit,k3"; + + cpus: cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <24000000>; + + cpu_0: cpu@0 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <0>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_1: cpu@1 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <1>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu1_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_2: cpu@2 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <2>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu2_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_3: cpu@3 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <3>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache0>; + mmu-type = "riscv,sv39"; + + cpu3_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_4: cpu@4 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <4>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu4_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_5: cpu@5 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <5>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu5_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_6: cpu@6 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <6>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu6_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu_7: cpu@7 { + compatible = "spacemit,x100", "riscv"; + device_type = "cpu"; + reg = <7>; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "h", + "sha", "shcounterenw", "shgatpa", "shtvala", + "shvsatpa", "shvstvala", "shvstvecd", "smaia", + "smstateen", "ssaia", "ssccptr", "sscofpmf", + "sscounterenw", "ssnpm", "ssstateen", "sstc", + "sstvala", "sstvecd", "ssu64xl", "svade", + "svinval", "svnapot", "svpbmt", "za64rs", + "zawrs", "zba", "zbb", "zbc", "zbs", "zca", + "zcb", "zcd", "zcmop", "zfa", "zfbfmin", + "zfh", "zfhmin", "zicbom", "zicbop", "zicboz", + "ziccamoa", "ziccif", "zicclsm", "zicntr", + "zicond", "zicsr", "zifencei", "zihintntl", + "zihintpause", "zihpm", "zimop", "zkt", "zvbb", + "zvbc", "zvfbfmin", "zvfbfwma", "zvfh", + "zvfhmin", "zvkb", "zvkg", "zvkn", "zvknc", + "zvkned", "zvkng", "zvknha", "zvknhb", "zvks", + "zvksc", "zvksed", "zvksg", "zvksh", "zvkt"; + riscv,cbom-block-size = <64>; + riscv,cbop-block-size = <64>; + riscv,cboz-block-size = <64>; + i-cache-block-size = <64>; + i-cache-size = <65536>; + i-cache-sets = <256>; + d-cache-block-size = <64>; + d-cache-size = <65536>; + d-cache-sets = <256>; + next-level-cache = <&l2_cache1>; + mmu-type = "riscv,sv39"; + + cpu7_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + l2_cache0: cache-controller-0 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <4194304>; + cache-sets = <4096>; + cache-unified; + }; + + l2_cache1: cache-controller-1 { + compatible = "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-size = <4194304>; + cache-sets = <4096>; + cache-unified; + }; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_0>; + }; + core1 { + cpu = <&cpu_1>; + }; + core2 { + cpu = <&cpu_2>; + }; + core3 { + cpu = <&cpu_3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_4>; + }; + core1 { + cpu = <&cpu_5>; + }; + core2 { + cpu = <&cpu_6>; + }; + core3 { + cpu = <&cpu_7>; + }; + }; + }; + }; + + clocks { + vctcxo_1m: clock-1m { + compatible = "fixed-clock"; + clock-frequency = <1000000>; + clock-output-names = "vctcxo_1m"; + #clock-cells = <0>; + }; + + vctcxo_24m: clock-24m { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "vctcxo_24m"; + #clock-cells = <0>; + }; + + vctcxo_3m: clock-3m { + compatible = "fixed-clock"; + clock-frequency = <3000000>; + clock-output-names = "vctcxo_3m"; + #clock-cells = <0>; + }; + + osc_32k: clock-32k { + compatible = "fixed-clock"; + clock-frequency = <32000>; + clock-output-names = "osc_32k"; + #clock-cells = <0>; + }; + }; + + soc: soc { + compatible = "simple-bus"; + interrupt-parent = <&saplic>; + #address-cells = <2>; + #size-cells = <2>; + dma-noncoherent; + ranges; + + eth0: ethernet@cac80000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac80000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC0_BUS>, + <&syscon_apmu CLK_APMU_EMAC0_1588>, + <&syscon_apmu CLK_APMU_EMAC0_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <131 IRQ_TYPE_LEVEL_HIGH>, + <276 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC0>; + reset-names = "stmmaceth"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac0_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x3e4 0x3e8>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac0_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + eth1: ethernet@cac82000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac82000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC1_BUS>, + <&syscon_apmu CLK_APMU_EMAC1_1588>, + <&syscon_apmu CLK_APMU_EMAC1_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <133 IRQ_TYPE_LEVEL_HIGH>, + <277 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC1>; + reset-names = "stmmaceth"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac1_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x3ec 0x3f0>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac1_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + eth2: ethernet@cac8e000 { + compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a"; + reg = <0x0 0xcac8e000 0x0 0x2000>; + clocks = <&syscon_apmu CLK_APMU_EMAC2_BUS>, + <&syscon_apmu CLK_APMU_EMAC2_1588>, + <&syscon_apmu CLK_APMU_EMAC2_RGMII_TX>; + clock-names = "stmmaceth", "ptp_ref", "tx"; + interrupts = <130 IRQ_TYPE_LEVEL_HIGH>, + <278 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "macirq", "eth_wake_irq"; + resets = <&syscon_apmu RESET_APMU_EMAC2>; + reset-names = "stmmaceth"; + rx-fifo-depth = <4096>; + tx-fifo-depth = <4096>; + snps,multicast-filter-bins = <64>; + snps,perfect-filter-entries = <32>; + snps,aal; + snps,tso; + snps,txpbl = <8>; + snps,rxpbl = <8>; + snps,force_sf_dma_mode; + snps,axi-config = <&gmac2_axi_setup>; + spacemit,apmu = <&syscon_apmu 0x248 0x24c>; + status = "disabled"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + gmac2_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <0xf>; + snps,rd_osr_lmt = <0xf>; + /* max axi burst len is 256 */ + snps,blen = <256 128 64 32 16 0 0>; + }; + }; + + i2c0: i2c@d4010800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4010800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <36 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI0>, + <&syscon_apbc CLK_APBC_TWSI0_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI0>; + status = "disabled"; + }; + + i2c1: i2c@d4011000 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4011000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <37 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI1>, + <&syscon_apbc CLK_APBC_TWSI1_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI1>; + status = "disabled"; + }; + + i2c2: i2c@d4012000 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4012000 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <38 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI2>, + <&syscon_apbc CLK_APBC_TWSI2_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI2>; + status = "disabled"; + }; + + i2c4: i2c@d4012800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4012800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <40 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI4>, + <&syscon_apbc CLK_APBC_TWSI4_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI4>; + status = "disabled"; + }; + + i2c5: i2c@d4013800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4013800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <41 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI5>, + <&syscon_apbc CLK_APBC_TWSI5_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI5>; + status = "disabled"; + }; + + syscon_apbc: system-controller@d4015000 { + compatible = "spacemit,k3-syscon-apbc"; + reg = <0x0 0xd4015000 0x0 0x1000>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + uart0: serial@d4017000 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017000 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART0>, + <&syscon_apbc CLK_APBC_UART0_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART0>; + interrupts = <42 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart2: serial@d4017100 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017100 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART2>, + <&syscon_apbc CLK_APBC_UART2_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART2>; + interrupts = <44 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart3: serial@d4017200 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017200 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART3>, + <&syscon_apbc CLK_APBC_UART3_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART3>; + interrupts = <45 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart4: serial@d4017300 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017300 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART4>, + <&syscon_apbc CLK_APBC_UART4_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART4>; + interrupts = <46 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart5: serial@d4017400 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017400 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART5>, + <&syscon_apbc CLK_APBC_UART5_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART5>; + interrupts = <47 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart6: serial@d4017500 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017500 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART6>, + <&syscon_apbc CLK_APBC_UART6_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART6>; + interrupts = <48 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart7: serial@d4017600 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017600 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART7>, + <&syscon_apbc CLK_APBC_UART7_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART7>; + interrupts = <49 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart8: serial@d4017700 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017700 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART8>, + <&syscon_apbc CLK_APBC_UART8_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART8>; + interrupts = <50 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + uart9: serial@d4017800 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd4017800 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART9>, + <&syscon_apbc CLK_APBC_UART9_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART9>; + interrupts = <51 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + i2c6: i2c@d4018800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd4018800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <70 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI6>, + <&syscon_apbc CLK_APBC_TWSI6_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI6>; + status = "disabled"; + }; + + gpio: gpio@d4019000 { + compatible = "spacemit,k3-gpio"; + reg = <0x0 0xd4019000 0x0 0x100>; + clocks = <&syscon_apbc CLK_APBC_GPIO>, + <&syscon_apbc CLK_APBC_GPIO_BUS>; + clock-names = "core", "bus"; + gpio-controller; + #gpio-cells = <3>; + interrupts = <58 IRQ_TYPE_LEVEL_HIGH>; + interrupt-parent = <&saplic>; + interrupt-controller; + #interrupt-cells = <3>; + gpio-ranges = <&pinctrl 0 0 0 32>, + <&pinctrl 1 0 32 32>, + <&pinctrl 2 0 64 32>, + <&pinctrl 3 0 96 32>; + }; + + i2c8: i2c@d401d800 { + compatible = "spacemit,k3-i2c", "spacemit,k1-i2c"; + reg = <0x0 0xd401d800 0x0 0x38>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&syscon_apbc CLK_APBC_TWSI8>, + <&syscon_apbc CLK_APBC_TWSI8_BUS>; + clock-names = "func", "bus"; + clock-frequency = <400000>; + resets = <&syscon_apbc RESET_APBC_TWSI8>; + status = "disabled"; + }; + + pinctrl: pinctrl@d401e000 { + compatible = "spacemit,k3-pinctrl"; + reg = <0x0 0xd401e000 0x0 0x1000>; + clocks = <&syscon_apbc CLK_APBC_AIB>, + <&syscon_apbc CLK_APBC_AIB_BUS>; + clock-names = "func", "bus"; + }; + + uart10: serial@d401f000 { + compatible = "spacemit,k3-uart", "intel,xscale-uart"; + reg = <0x0 0xd401f000 0x0 0x100>; + reg-shift = <2>; + reg-io-width = <4>; + clocks = <&syscon_apbc CLK_APBC_UART10>, + <&syscon_apbc CLK_APBC_UART10_BUS>; + clock-names = "core", "bus"; + resets = <&syscon_apbc RESET_APBC_UART10>; + interrupts = <281 IRQ_TYPE_LEVEL_HIGH>; + status = "disabled"; + }; + + syscon_mpmu: system-controller@d4050000 { + compatible = "spacemit,k3-syscon-mpmu"; + reg = <0x0 0xd4050000 0x0 0x10000>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + pll: clock-controller@d4090000 { + compatible = "spacemit,k3-pll"; + reg = <0x0 0xd4090000 0x0 0x10000>; + clocks = <&vctcxo_24m>; + spacemit,mpmu = <&syscon_mpmu>; + #clock-cells = <1>; + }; + + syscon_apmu: system-controller@d4282800 { + compatible = "spacemit,k3-syscon-apmu"; + reg = <0x0 0xd4282800 0x0 0x400>; + clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>; + clock-names = "osc", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + syscon_dciu: system-controller@d8440000 { + compatible = "spacemit,k3-syscon-dciu"; + reg = <0x0 0xd8440000 0x0 0xc000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + simsic: interrupt-controller@e0400000 { + compatible = "spacemit,k3-imsics", "riscv,imsics"; + reg = <0x0 0xe0400000 0x0 0x200000>; + #interrupt-cells = <0>; + #msi-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 9>, <&cpu1_intc 9>, + <&cpu2_intc 9>, <&cpu3_intc 9>, + <&cpu4_intc 9>, <&cpu5_intc 9>, + <&cpu6_intc 9>, <&cpu7_intc 9>; + msi-controller; + riscv,guest-index-bits = <6>; + riscv,hart-index-bits = <4>; + riscv,num-guest-ids = <511>; + riscv,num-ids = <511>; + }; + + saplic: interrupt-controller@e0804000 { + compatible = "spacemit,k3-aplic", "riscv,aplic"; + reg = <0x0 0xe0804000 0x0 0x4000>; + #interrupt-cells = <2>; + interrupt-controller; + msi-parent = <&simsic>; + riscv,num-sources = <512>; + }; + + clint: timer@e081c000 { + compatible = "spacemit,k3-clint", "sifive,clint0"; + reg = <0x0 0xe081c000 0x0 0x4000>; + interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>, + <&cpu1_intc 3>, <&cpu1_intc 7>, + <&cpu2_intc 3>, <&cpu2_intc 7>, + <&cpu3_intc 3>, <&cpu3_intc 7>, + <&cpu4_intc 3>, <&cpu4_intc 7>, + <&cpu5_intc 3>, <&cpu5_intc 7>, + <&cpu6_intc 3>, <&cpu6_intc 7>, + <&cpu7_intc 3>, <&cpu7_intc 7>; + }; + + /* sec_i2c3: 0xf0614000, not available from Linux */ + + mimsic: interrupt-controller@f1000000 { + compatible = "spacemit,k3-imsics", "riscv,imsics"; + reg = <0x0 0xf1000000 0x0 0x10000>; + #interrupt-cells = <0>; + #msi-cells = <0>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11>, <&cpu1_intc 11>, + <&cpu2_intc 11>, <&cpu3_intc 11>, + <&cpu4_intc 11>, <&cpu5_intc 11>, + <&cpu6_intc 11>, <&cpu7_intc 11>; + msi-controller; + riscv,guest-index-bits = <6>; + riscv,hart-index-bits = <4>; + riscv,num-guest-ids = <511>; + riscv,num-ids = <511>; + status = "reserved"; + }; + + maplic: interrupt-controller@f1800000 { + compatible = "spacemit,k3-aplic", "riscv,aplic"; + reg = <0x0 0xf1800000 0x0 0x4000>; + #interrupt-cells = <2>; + interrupt-controller; + msi-parent = <&mimsic>; + riscv,children = <&saplic>; + riscv,delegation = <&saplic 1 512>; + riscv,num-sources = <512>; + status = "reserved"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/starfive/Makefile b/arch/riscv/boot/dts/starfive/Makefile index b3bb12f78e7d..3dd1f05283f7 100644 --- a/arch/riscv/boot/dts/starfive/Makefile +++ b/arch/riscv/boot/dts/starfive/Makefile @@ -10,6 +10,11 @@ dtb-$(CONFIG_ARCH_STARFIVE) += jh7100-starfive-visionfive-v1.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-deepcomputing-fml13v01.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-milkv-mars.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-milkv-marscm-emmc.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-milkv-marscm-lite.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-orangepi-rv.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-pine64-star64.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-lite.dtb +dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-lite-emmc.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-v1.2a.dtb dtb-$(CONFIG_ARCH_STARFIVE) += jh7110-starfive-visionfive-2-v1.3b.dtb diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi index 48fb5091b817..a7a1c09a2c90 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi @@ -8,6 +8,8 @@ #include "jh7110.dtsi" #include "jh7110-pinfunc.h" #include <dt-bindings/gpio/gpio.h> +#include <dt-bindings/leds/common.h> +#include <dt-bindings/pinctrl/starfive,jh7110-pinctrl.h> / { aliases { @@ -28,6 +30,7 @@ memory@40000000 { device_type = "memory"; reg = <0x0 0x40000000 0x1 0x0>; + bootph-pre-ram; }; gpio-restart { @@ -36,6 +39,14 @@ priority = <224>; }; + leds { + compatible = "gpio-leds"; + + led_status_power: led-0 { + gpios = <&aongpio 3 GPIO_ACTIVE_HIGH>; + }; + }; + pwmdac_codec: audio-codec { compatible = "linux,spdif-dit"; #sound-dai-cells = <0>; @@ -124,29 +135,6 @@ clock-frequency = <49152000>; }; -&camss { - assigned-clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, - <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>; - assigned-clock-rates = <49500000>, <198000000>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - }; - - port@1 { - reg = <1>; - - camss_from_csi2rx: endpoint { - remote-endpoint = <&csi2rx_to_camss>; - }; - }; - }; -}; - &csi2rx { assigned-clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>; assigned-clock-rates = <297000000>; @@ -164,9 +152,7 @@ port@1 { reg = <1>; - csi2rx_to_camss: endpoint { - remote-endpoint = <&camss_from_csi2rx>; - }; + /* remote CAMSS endpoint */ }; }; }; @@ -233,7 +219,7 @@ regulator-always-on; regulator-min-microvolt = <500000>; regulator-max-microvolt = <1540000>; - regulator-name = "vdd-cpu"; + regulator-name = "vdd_cpu"; }; emmc_vdd: aldo4 { @@ -245,6 +231,13 @@ }; }; }; + + eeprom@50 { + compatible = "atmel,24c04"; + reg = <0x50>; + bootph-pre-ram; + pagesize = <16>; + }; }; &i2c6 { @@ -262,15 +255,9 @@ assigned-clocks = <&syscrg JH7110_SYSCLK_SDIO0_SDCARD>; assigned-clock-rates = <50000000>; bus-width = <8>; - cap-mmc-highspeed; - mmc-ddr-1_8v; - mmc-hs200-1_8v; - cap-mmc-hw-reset; - post-power-on-delay-ms = <200>; + bootph-pre-ram; pinctrl-names = "default"; pinctrl-0 = <&mmc0_pins>; - vmmc-supply = <&vcc_3v3>; - vqmmc-supply = <&emmc_vdd>; status = "okay"; }; @@ -279,12 +266,8 @@ assigned-clocks = <&syscrg JH7110_SYSCLK_SDIO1_SDCARD>; assigned-clock-rates = <50000000>; bus-width = <4>; - no-sdio; - no-mmc; - cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; - disable-wp; + bootph-pre-ram; cap-sd-highspeed; - post-power-on-delay-ms = <200>; pinctrl-names = "default"; pinctrl-0 = <&mmc1_pins>; status = "okay"; @@ -317,8 +300,9 @@ nor_flash: flash@0 { compatible = "jedec,spi-nor"; reg = <0>; - cdns,read-delay = <5>; - spi-max-frequency = <12000000>; + bootph-pre-ram; + cdns,read-delay = <2>; + spi-max-frequency = <100000000>; cdns,tshsl-ns = <1>; cdns,tsd2d-ns = <1>; cdns,tchsh-ns = <1>; @@ -350,18 +334,20 @@ &spi0 { pinctrl-names = "default"; pinctrl-0 = <&spi0_pins>; - - spi_dev0: spi@0 { - compatible = "rohm,dh2228fv"; - reg = <0>; - spi-max-frequency = <10000000>; - }; }; &syscrg { - assigned-clocks = <&syscrg JH7110_SYSCLK_CPU_CORE>, + assigned-clocks = <&syscrg JH7110_SYSCLK_CPU_ROOT>, + <&syscrg JH7110_SYSCLK_BUS_ROOT>, + <&syscrg JH7110_SYSCLK_PERH_ROOT>, + <&syscrg JH7110_SYSCLK_QSPI_REF>, + <&syscrg JH7110_SYSCLK_CPU_CORE>, <&pllclk JH7110_PLLCLK_PLL0_OUT>; - assigned-clock-rates = <500000000>, <1500000000>; + assigned-clock-parents = <&pllclk JH7110_PLLCLK_PLL0_OUT>, + <&pllclk JH7110_PLLCLK_PLL2_OUT>, + <&pllclk JH7110_PLLCLK_PLL2_OUT>, + <&syscrg JH7110_SYSCLK_QSPI_REF_SRC>; + assigned-clock-rates = <0>, <0>, <0>, <0>, <500000000>, <1500000000>; }; &sysgpio { @@ -394,6 +380,8 @@ }; i2c5_pins: i2c5-0 { + bootph-pre-ram; + i2c-pins { pinmux = <GPIOMUX(19, GPOUT_LOW, GPOEN_SYS_I2C5_CLK, @@ -402,6 +390,7 @@ GPOEN_SYS_I2C5_DATA, GPI_SYS_I2C5_DATA)>; bias-disable; /* external pull-up */ + bootph-pre-ram; input-enable; input-schmitt-enable; }; @@ -422,28 +411,17 @@ }; mmc0_pins: mmc0-0 { - rst-pins { - pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, - GPOEN_ENABLE, - GPI_NONE)>; - bias-pull-up; - drive-strength = <12>; - input-disable; - input-schmitt-disable; - slew-rate = <0>; - }; - mmc-pins { - pinmux = <PINMUX(64, 0)>, - <PINMUX(65, 0)>, - <PINMUX(66, 0)>, - <PINMUX(67, 0)>, - <PINMUX(68, 0)>, - <PINMUX(69, 0)>, - <PINMUX(70, 0)>, - <PINMUX(71, 0)>, - <PINMUX(72, 0)>, - <PINMUX(73, 0)>; + pinmux = <PINMUX(PAD_SD0_CLK, 0)>, + <PINMUX(PAD_SD0_CMD, 0)>, + <PINMUX(PAD_SD0_DATA0, 0)>, + <PINMUX(PAD_SD0_DATA1, 0)>, + <PINMUX(PAD_SD0_DATA2, 0)>, + <PINMUX(PAD_SD0_DATA3, 0)>, + <PINMUX(PAD_SD0_DATA4, 0)>, + <PINMUX(PAD_SD0_DATA5, 0)>, + <PINMUX(PAD_SD0_DATA6, 0)>, + <PINMUX(PAD_SD0_DATA7, 0)>; bias-pull-up; drive-strength = <12>; input-enable; @@ -630,6 +608,7 @@ }; &uart0 { + bootph-pre-ram; pinctrl-names = "default"; pinctrl-0 = <&uart0_pins>; status = "okay"; diff --git a/arch/riscv/boot/dts/starfive/jh7110-deepcomputing-fml13v01.dts b/arch/riscv/boot/dts/starfive/jh7110-deepcomputing-fml13v01.dts index 30b0715196b6..d8db9ed4474d 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-deepcomputing-fml13v01.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-deepcomputing-fml13v01.dts @@ -11,7 +11,87 @@ compatible = "deepcomputing,fml13v01", "starfive,jh7110"; }; +&mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; +}; + +&mmc0_pins { + rst-pins { + pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, + GPOEN_ENABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; +}; + +&mmc1 { + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; + disable-wp; +}; + +&pcie1 { + perst-gpios = <&sysgpio 21 GPIO_ACTIVE_LOW>; + phys = <&pciephy1>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie1_pins>; + status = "okay"; +}; + +&sysgpio { + pcie1_pins: pcie1-0 { + clkreq-pins { + pinmux = <GPIOMUX(29, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-down; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + + wake-pins { + pinmux = <GPIOMUX(28, GPOUT_HIGH, + GPOEN_DISABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <2>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + + usb0_pins: usb0-0 { + vbus-pins { + pinmux = <GPIOMUX(25, GPOUT_SYS_USB_DRIVE_VBUS, + GPOEN_ENABLE, + GPI_NONE)>; + bias-disable; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; +}; + &usb0 { dr_mode = "host"; + pinctrl-names = "default"; + pinctrl-0 = <&usb0_pins>; status = "okay"; }; + +&usb_cdns3 { + phys = <&usbphy0>, <&pciephy0>; + phy-names = "cdns3,usb2-phy", "cdns3,usb3-phy"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts b/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts index 3bd62ab78523..21873612d993 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-mars.dts @@ -12,9 +12,9 @@ }; &gmac0 { - starfive,tx-use-rgmii-clk; assigned-clocks = <&aoncrg JH7110_AONCLK_GMAC0_TX>; assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; + starfive,tx-use-rgmii-clk; status = "okay"; }; @@ -22,6 +22,33 @@ status = "okay"; }; +&mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; +}; + +&mmc0_pins { + rst-pins { + pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, + GPOEN_ENABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; +}; + +&mmc1 { + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; + disable-wp; +}; + &pcie0 { status = "okay"; }; @@ -31,14 +58,14 @@ }; &phy0 { - motorcomm,tx-clk-adj-enabled; + rx-internal-delay-ps = <1500>; + tx-internal-delay-ps = <1500>; + motorcomm,rx-clk-drv-microamp = <3970>; + motorcomm,rx-data-drv-microamp = <2910>; motorcomm,tx-clk-10-inverted; motorcomm,tx-clk-100-inverted; motorcomm,tx-clk-1000-inverted; - motorcomm,rx-clk-drv-microamp = <3970>; - motorcomm,rx-data-drv-microamp = <2910>; - rx-internal-delay-ps = <1500>; - tx-internal-delay-ps = <1500>; + motorcomm,tx-clk-adj-enabled; }; &pwm { diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-emmc.dts b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-emmc.dts new file mode 100644 index 000000000000..ce95496263af --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-emmc.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 E Shattow <e@freeshell.de> + */ + +/dts-v1/; +#include "jh7110-milkv-marscm.dtsi" + +/ { + model = "Milk-V Mars CM"; + compatible = "milkv,marscm-emmc", "starfive,jh7110"; +}; + +&mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts new file mode 100644 index 000000000000..63aa94d65ab5 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm-lite.dts @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 E Shattow <e@freeshell.de> + */ + +/dts-v1/; +#include "jh7110-milkv-marscm.dtsi" + +/ { + model = "Milk-V Mars CM Lite"; + compatible = "milkv,marscm-lite", "starfive,jh7110"; +}; + +&mmc0 { + bus-width = <4>; + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; + disable-wp; +}; + +&mmc0_pins { + pwren-pins { + pinmux = <GPIOMUX(22, GPOUT_HIGH, + GPOEN_ENABLE, + GPI_NONE)>; + }; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm.dtsi b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm.dtsi new file mode 100644 index 000000000000..025471061d43 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-milkv-marscm.dtsi @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 E Shattow <e@freeshell.de> + */ + +/dts-v1/; +#include <dt-bindings/interrupt-controller/irq.h> +#include "jh7110-common.dtsi" + +/ { + aliases { + i2c1 = &i2c1; + i2c3 = &i2c3; + i2c4 = &i2c4; + serial3 = &uart3; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&sysgpio 33 GPIO_ACTIVE_LOW>; + }; +}; + +&gmac0 { + assigned-clocks = <&aoncrg JH7110_AONCLK_GMAC0_TX>; + assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; + starfive,tx-use-rgmii-clk; + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c2 { + status = "disabled"; +}; + +&i2c6 { + status = "disabled"; +}; + +&mmc0_pins { + rst-pins { + pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, + GPOEN_ENABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; +}; + +&mmc1 { + #address-cells = <1>; + #size-cells = <0>; + + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + status = "okay"; + + ap6256: wifi@1 { + compatible = "brcm,bcm43456-fmac", "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&sysgpio>; + interrupts = <34 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "host-wake"; + pinctrl-0 = <&wifi_host_wake_irq>; + pinctrl-names = "default"; + }; +}; + +&pcie0 { + status = "okay"; +}; + +&phy0 { + rx-internal-delay-ps = <1500>; + tx-internal-delay-ps = <1500>; + motorcomm,rx-clk-drv-microamp = <3970>; + motorcomm,rx-data-drv-microamp = <2910>; + motorcomm,tx-clk-10-inverted; + motorcomm,tx-clk-100-inverted; + motorcomm,tx-clk-1000-inverted; + motorcomm,tx-clk-adj-enabled; +}; + +&pwm { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&sysgpio { + uart1_pins: uart1-0 { + tx-pins { + pinmux = <GPIOMUX(16, GPOUT_SYS_UART1_TX, + GPOEN_ENABLE, + GPI_NONE)>; + bias-disable; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + }; + + rx-pins { + pinmux = <GPIOMUX(17, GPOUT_LOW, + GPOEN_DISABLE, + GPI_SYS_UART1_RX)>; + bias-pull-up; + input-enable; + input-schmitt-enable; + }; + + cts-pins { + pinmux = <GPIOMUX(3, GPOUT_LOW, + GPOEN_DISABLE, + GPI_SYS_UART1_CTS)>; + bias-disable; + input-enable; + input-schmitt-enable; + }; + + rts-pins { + pinmux = <GPIOMUX(2, GPOUT_SYS_UART1_RTS, + GPOEN_ENABLE, + GPI_NONE)>; + bias-disable; + input-disable; + input-schmitt-disable; + }; + }; + + usb0_pins: usb0-0 { + vbus-pins { + pinmux = <GPIOMUX(25, GPOUT_SYS_USB_DRIVE_VBUS, + GPOEN_ENABLE, + GPI_NONE)>; + bias-disable; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + + wifi_host_wake_irq: wifi-host-wake-irq-0 { + wake-pins { + pinmux = <GPIOMUX(34, GPOUT_LOW, + GPOEN_DISABLE, + GPI_NONE)>; + input-enable; + }; + }; +}; + +&uart1 { + uart-has-rtscts; + pinctrl-0 = <&uart1_pins>; + pinctrl-names = "default"; + status = "okay"; +}; + +&usb0 { + dr_mode = "host"; + pinctrl-names = "default"; + pinctrl-0 = <&usb0_pins>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-orangepi-rv.dts b/arch/riscv/boot/dts/starfive/jh7110-orangepi-rv.dts new file mode 100644 index 000000000000..053c35992ec3 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-orangepi-rv.dts @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 Icenowy Zheng <uwu@icenowy.me> + */ + +/dts-v1/; +#include "jh7110-common.dtsi" + +/ { + model = "Xunlong Orange Pi RV"; + compatible = "xunlong,orangepi-rv", "starfive,jh7110"; + + /* This regulator is always on by hardware */ + reg_vcc3v3_pcie: regulator-vcc3v3-pcie { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3-pcie"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + wifi_pwrseq: wifi-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&sysgpio 62 GPIO_ACTIVE_LOW>; + }; +}; + +&gmac0 { + assigned-clocks = <&aoncrg JH7110_AONCLK_GMAC0_TX>; + assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; + starfive,tx-use-rgmii-clk; + status = "okay"; +}; + +&mmc0 { + #address-cells = <1>; + #size-cells = <0>; + cap-sd-highspeed; + mmc-pwrseq = <&wifi_pwrseq>; + vmmc-supply = <®_vcc3v3_pcie>; + vqmmc-supply = <&vcc_3v3>; + status = "okay"; + + ap6256: wifi@1 { + compatible = "brcm,bcm43456-fmac", "brcm,bcm4329-fmac"; + reg = <1>; + /* TODO: out-of-band IRQ on GPIO21, lacking pinctrl support */ + }; +}; + +&mmc1 { + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_HIGH>; +}; + +&pcie0 { + status = "okay"; +}; + +&pcie1 { + status = "okay"; +}; + +&phy0 { + rx-internal-delay-ps = <1500>; + tx-internal-delay-ps = <1500>; + motorcomm,rx-clk-drv-microamp = <3970>; + motorcomm,rx-data-drv-microamp = <2910>; + motorcomm,tx-clk-adj-enabled; + motorcomm,tx-clk-10-inverted; + motorcomm,tx-clk-100-inverted; + motorcomm,tx-clk-1000-inverted; +}; + +&pwmdac { + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts b/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts index b764d4d92fd9..aec7ae3d1f5b 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts +++ b/arch/riscv/boot/dts/starfive/jh7110-pine64-star64.dts @@ -44,6 +44,33 @@ status = "okay"; }; +&mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; +}; + +&mmc0_pins { + rst-pins { + pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, + GPOEN_ENABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; +}; + +&mmc1 { + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; + disable-wp; +}; + &pcie1 { status = "okay"; }; @@ -100,3 +127,8 @@ pinctrl-0 = <&usb0_pins>; status = "okay"; }; + +&usb_cdns3 { + phys = <&usbphy0>, <&pciephy0>; + phy-names = "cdns3,usb2-phy", "cdns3,usb3-phy"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-pinfunc.h b/arch/riscv/boot/dts/starfive/jh7110-pinfunc.h index 256de17f5261..ae49c908e7fb 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-pinfunc.h +++ b/arch/riscv/boot/dts/starfive/jh7110-pinfunc.h @@ -89,7 +89,7 @@ #define GPOUT_SYS_SDIO1_DATA1 59 #define GPOUT_SYS_SDIO1_DATA2 60 #define GPOUT_SYS_SDIO1_DATA3 61 -#define GPOUT_SYS_SDIO1_DATA4 63 +#define GPOUT_SYS_SDIO1_DATA4 62 #define GPOUT_SYS_SDIO1_DATA5 63 #define GPOUT_SYS_SDIO1_DATA6 64 #define GPOUT_SYS_SDIO1_DATA7 65 diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts new file mode 100644 index 000000000000..7544efa95de4 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite-emmc.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 StarFive Technology Co., Ltd. + * Copyright (C) 2025 Hal Feng <hal.feng@starfivetech.com> + */ + +/dts-v1/; +#include "jh7110-starfive-visionfive-2-lite.dtsi" + +/ { + model = "StarFive VisionFive 2 Lite eMMC"; + compatible = "starfive,visionfive-2-lite-emmc", "starfive,jh7110s", "starfive,jh7110"; +}; + +&mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts new file mode 100644 index 000000000000..b9913991a1b7 --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 StarFive Technology Co., Ltd. + * Copyright (C) 2025 Hal Feng <hal.feng@starfivetech.com> + */ + +/dts-v1/; +#include "jh7110-starfive-visionfive-2-lite.dtsi" + +/ { + model = "StarFive VisionFive 2 Lite"; + compatible = "starfive,visionfive-2-lite", "starfive,jh7110s", "starfive,jh7110"; +}; + +&mmc0 { + bus-width = <4>; + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_HIGH>; + disable-wp; + cap-sd-highspeed; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dtsi b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dtsi new file mode 100644 index 000000000000..f8797a666dbf --- /dev/null +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2-lite.dtsi @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2025 StarFive Technology Co., Ltd. + * Copyright (C) 2025 Hal Feng <hal.feng@starfivetech.com> + */ + +/dts-v1/; +#include "jh7110-common.dtsi" + +/ { + vcc_3v3_pcie: regulator-vcc-3v3-pcie { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&sysgpio 27 GPIO_ACTIVE_HIGH>; + regulator-name = "vcc_3v3_pcie"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&cpu_opp { + /delete-node/ opp-375000000; + /delete-node/ opp-500000000; + /delete-node/ opp-750000000; + /delete-node/ opp-1500000000; + + opp-312500000 { + opp-hz = /bits/ 64 <312500000>; + opp-microvolt = <800000>; + }; + opp-417000000 { + opp-hz = /bits/ 64 <417000000>; + opp-microvolt = <800000>; + }; + opp-625000000 { + opp-hz = /bits/ 64 <625000000>; + opp-microvolt = <800000>; + }; + opp-1250000000 { + opp-hz = /bits/ 64 <1250000000>; + opp-microvolt = <1000000>; + }; +}; + +&gmac0 { + starfive,tx-use-rgmii-clk; + assigned-clocks = <&aoncrg JH7110_AONCLK_GMAC0_TX>; + assigned-clock-parents = <&aoncrg JH7110_AONCLK_GMAC0_RMII_RTX>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&mmc1 { + max-frequency = <50000000>; + keep-power-in-suspend; + non-removable; +}; + +&pcie1 { + vpcie3v3-supply = <&vcc_3v3_pcie>; + status = "okay"; +}; + +&phy0 { + motorcomm,tx-clk-adj-enabled; + motorcomm,tx-clk-100-inverted; + motorcomm,tx-clk-1000-inverted; + motorcomm,rx-clk-drv-microamp = <3970>; + motorcomm,rx-data-drv-microamp = <2910>; + rx-internal-delay-ps = <1500>; + tx-internal-delay-ps = <1500>; +}; + +&pwm { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&syscrg { + assigned-clock-rates = <0>, <0>, <0>, <0>, <500000000>, <1250000000>; +}; + +&sysgpio { + uart1_pins: uart1-0 { + tx-pins { + pinmux = <GPIOMUX(22, GPOUT_SYS_UART1_TX, + GPOEN_ENABLE, + GPI_NONE)>; + bias-disable; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + + rx-pins { + pinmux = <GPIOMUX(23, GPOUT_LOW, + GPOEN_DISABLE, + GPI_SYS_UART1_RX)>; + bias-pull-up; + drive-strength = <2>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + + cts-pins { + pinmux = <GPIOMUX(24, GPOUT_LOW, + GPOEN_DISABLE, + GPI_SYS_UART1_CTS)>; + input-enable; + }; + + rts-pins { + pinmux = <GPIOMUX(25, GPOUT_SYS_UART1_RTS, + GPOEN_ENABLE, + GPI_NONE)>; + input-enable; + }; + }; + + usb0_pins: usb0-0 { + power-pins { + pinmux = <GPIOMUX(26, GPOUT_HIGH, + GPOEN_ENABLE, + GPI_NONE)>; + input-disable; + }; + + switch-pins { + pinmux = <GPIOMUX(62, GPOUT_LOW, + GPOEN_ENABLE, + GPI_NONE)>; + input-disable; + }; + }; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_pins>; + status = "okay"; +}; + +&usb0 { + dr_mode = "host"; + pinctrl-names = "default"; + pinctrl-0 = <&usb0_pins>; + status = "okay"; +}; + +&usb_cdns3 { + phys = <&usbphy0>, <&pciephy0>; + phy-names = "cdns3,usb2-phy", "cdns3,usb3-phy"; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi index 5f14afb2c24d..edc8f4588133 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi @@ -38,9 +38,33 @@ }; &mmc0 { + cap-mmc-highspeed; + cap-mmc-hw-reset; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&emmc_vdd>; non-removable; }; +&mmc0_pins { + rst-pins { + pinmux = <GPIOMUX(62, GPOUT_SYS_SDIO0_RST, + GPOEN_ENABLE, + GPI_NONE)>; + bias-pull-up; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; +}; + +&mmc1 { + cd-gpios = <&sysgpio 41 GPIO_ACTIVE_LOW>; + disable-wp; +}; + &pcie0 { status = "okay"; }; diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi index 0d8339357bad..9c3e4598747e 100644 --- a/arch/riscv/boot/dts/starfive/jh7110.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi @@ -35,6 +35,7 @@ cpu0_intc: interrupt-controller { compatible = "riscv,cpu-intc"; + bootph-pre-ram; interrupt-controller; #interrupt-cells = <1>; }; @@ -68,6 +69,7 @@ cpu1_intc: interrupt-controller { compatible = "riscv,cpu-intc"; + bootph-pre-ram; interrupt-controller; #interrupt-cells = <1>; }; @@ -101,6 +103,7 @@ cpu2_intc: interrupt-controller { compatible = "riscv,cpu-intc"; + bootph-pre-ram; interrupt-controller; #interrupt-cells = <1>; }; @@ -134,6 +137,7 @@ cpu3_intc: interrupt-controller { compatible = "riscv,cpu-intc"; + bootph-pre-ram; interrupt-controller; #interrupt-cells = <1>; }; @@ -167,6 +171,7 @@ cpu4_intc: interrupt-controller { compatible = "riscv,cpu-intc"; + bootph-pre-ram; interrupt-controller; #interrupt-cells = <1>; }; @@ -273,12 +278,14 @@ gmac1_rgmii_rxin: gmac1-rgmii-rxin-clock { compatible = "fixed-clock"; + bootph-pre-ram; clock-output-names = "gmac1_rgmii_rxin"; #clock-cells = <0>; }; gmac1_rmii_refin: gmac1-rmii-refin-clock { compatible = "fixed-clock"; + bootph-pre-ram; clock-output-names = "gmac1_rmii_refin"; #clock-cells = <0>; }; @@ -321,6 +328,7 @@ osc: oscillator { compatible = "fixed-clock"; + bootph-pre-ram; clock-output-names = "osc"; #clock-cells = <0>; }; @@ -354,6 +362,7 @@ clint: timer@2000000 { compatible = "starfive,jh7110-clint", "sifive,clint0"; reg = <0x0 0x2000000 0x0 0x10000>; + bootph-pre-ram; interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>, <&cpu1_intc 3>, <&cpu1_intc 7>, <&cpu2_intc 3>, <&cpu2_intc 7>, @@ -611,6 +620,8 @@ pciephy0: phy@10210000 { compatible = "starfive,jh7110-pcie-phy"; reg = <0x0 0x10210000 0x0 0x10000>; + starfive,sys-syscon = <&sys_syscon 0x18>; + starfive,stg-syscon = <&stg_syscon 0x148 0x1f4>; #phy-cells = <0>; }; @@ -878,6 +889,7 @@ syscrg: clock-controller@13020000 { compatible = "starfive,jh7110-syscrg"; reg = <0x0 0x13020000 0x0 0x10000>; + bootph-pre-ram; clocks = <&osc>, <&gmac1_rmii_refin>, <&gmac1_rgmii_rxin>, <&i2stx_bclk_ext>, <&i2stx_lrck_ext>, @@ -902,6 +914,7 @@ pllclk: clock-controller { compatible = "starfive,jh7110-pll"; + bootph-pre-ram; clocks = <&osc>; #clock-cells = <1>; }; @@ -929,6 +942,19 @@ <&syscrg JH7110_SYSRST_WDT_CORE>; }; + memory-controller@15700000 { + compatible = "starfive,jh7110-dmc"; + reg = <0x0 0x15700000 0x0 0x10000>, + <0x0 0x13000000 0x0 0x10000>; + bootph-pre-ram; + clocks = <&syscrg JH7110_PLLCLK_PLL1_OUT>; + clock-names = "pll"; + resets = <&syscrg JH7110_SYSRST_DDR_AXI>, + <&syscrg JH7110_SYSRST_DDR_OSC>, + <&syscrg JH7110_SYSRST_DDR_APB>; + reset-names = "axi", "osc", "apb"; + }; + crypto: crypto@16000000 { compatible = "starfive,jh7110-crypto"; reg = <0x0 0x16000000 0x0 0x4000>; @@ -1022,7 +1048,6 @@ snps,force_thresh_dma_mode; snps,axi-config = <&stmmac_axi_setup>; snps,tso; - snps,en-tx-lpi-clockgating; snps,txpbl = <16>; snps,rxpbl = <16>; starfive,syscon = <&aon_syscon 0xc 0x12>; @@ -1053,7 +1078,6 @@ snps,force_thresh_dma_mode; snps,axi-config = <&stmmac_axi_setup>; snps,tso; - snps,en-tx-lpi-clockgating; snps,txpbl = <16>; snps,rxpbl = <16>; starfive,syscon = <&sys_syscon 0x90 0x2>; @@ -1175,34 +1199,6 @@ #phy-cells = <0>; }; - camss: isp@19840000 { - compatible = "starfive,jh7110-camss"; - reg = <0x0 0x19840000 0x0 0x10000>, - <0x0 0x19870000 0x0 0x30000>; - reg-names = "syscon", "isp"; - clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, - <&ispcrg JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C>, - <&ispcrg JH7110_ISPCLK_DVP_INV>, - <&ispcrg JH7110_ISPCLK_VIN_P_AXI_WR>, - <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>, - <&syscrg JH7110_SYSCLK_ISP_TOP_CORE>, - <&syscrg JH7110_SYSCLK_ISP_TOP_AXI>; - clock-names = "apb_func", "wrapper_clk_c", "dvp_inv", - "axiwr", "mipi_rx0_pxl", "ispcore_2x", - "isp_axi"; - resets = <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_P>, - <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_C>, - <&ispcrg JH7110_ISPRST_VIN_P_AXI_RD>, - <&ispcrg JH7110_ISPRST_VIN_P_AXI_WR>, - <&syscrg JH7110_SYSRST_ISP_TOP>, - <&syscrg JH7110_SYSRST_ISP_TOP_AXI>; - reset-names = "wrapper_p", "wrapper_c", "axird", - "axiwr", "isp_top_n", "isp_top_axi"; - power-domains = <&pwrc JH7110_PD_ISP>; - interrupts = <92>, <87>, <90>, <88>; - status = "disabled"; - }; - voutcrg: clock-controller@295c0000 { compatible = "starfive,jh7110-voutcrg"; reg = <0x0 0x295c0000 0x0 0x10000>; diff --git a/arch/riscv/boot/dts/tenstorrent/Makefile b/arch/riscv/boot/dts/tenstorrent/Makefile new file mode 100644 index 000000000000..2c81faaba462 --- /dev/null +++ b/arch/riscv/boot/dts/tenstorrent/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +dtb-$(CONFIG_ARCH_TENSTORRENT) += blackhole-card.dtb diff --git a/arch/riscv/boot/dts/tenstorrent/blackhole-card.dts b/arch/riscv/boot/dts/tenstorrent/blackhole-card.dts new file mode 100644 index 000000000000..f53667ce73a9 --- /dev/null +++ b/arch/riscv/boot/dts/tenstorrent/blackhole-card.dts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/dts-v1/; + +#include "blackhole.dtsi" + +/ { + model = "Tenstorrent Blackhole"; + compatible = "tenstorrent,blackhole-card", "tenstorrent,blackhole"; + + memory@400030000000 { + device_type = "memory"; + reg = <0x4000 0x30000000 0x1 0x00000000>; + }; +}; diff --git a/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi new file mode 100644 index 000000000000..6408810d8d80 --- /dev/null +++ b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// Copyright 2025 Tenstorrent AI ULC +/dts-v1/; + +/ { + compatible = "tenstorrent,blackhole"; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <50000000>; + + cpu@0 { + compatible = "sifive,x280", "sifive,rocket0", "riscv"; + device_type = "cpu"; + reg = <0>; + mmu-type = "riscv,sv57"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicsr", + "zifencei", "zfh", "zba", "zbb", "sscofpmf"; + + cpu0_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu@1 { + compatible = "sifive,x280", "sifive,rocket0", "riscv"; + device_type = "cpu"; + reg = <1>; + mmu-type = "riscv,sv57"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicsr", + "zifencei", "zfh", "zba", "zbb", "sscofpmf"; + + cpu1_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu@2 { + compatible = "sifive,x280", "sifive,rocket0", "riscv"; + device_type = "cpu"; + reg = <2>; + mmu-type = "riscv,sv57"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicsr", + "zifencei", "zfh", "zba", "zbb", "sscofpmf"; + + cpu2_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + + cpu@3 { + compatible = "sifive,x280", "sifive,rocket0", "riscv"; + device_type = "cpu"; + reg = <3>; + mmu-type = "riscv,sv57"; + riscv,isa-base = "rv64i"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "zicsr", + "zifencei", "zfh", "zba", "zbb", "sscofpmf"; + + cpu3_intc: interrupt-controller { + compatible = "riscv,cpu-intc"; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + + clint0: timer@2000000 { + compatible = "tenstorrent,blackhole-clint", "sifive,clint0"; + reg = <0x0 0x2000000 0x0 0x10000>; + interrupts-extended = <&cpu0_intc 0x3>, <&cpu0_intc 0x7>, + <&cpu1_intc 0x3>, <&cpu1_intc 0x7>, + <&cpu2_intc 0x3>, <&cpu2_intc 0x7>, + <&cpu3_intc 0x3>, <&cpu3_intc 0x7>; + }; + + plic0: interrupt-controller@c000000 { + compatible = "tenstorrent,blackhole-plic", "sifive,plic-1.0.0"; + reg = <0x0 0x0c000000 0x0 0x04000000>; + interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>, + <&cpu1_intc 11>, <&cpu1_intc 9>, + <&cpu2_intc 11>, <&cpu2_intc 9>, + <&cpu3_intc 11>, <&cpu3_intc 9>; + interrupt-controller; + #interrupt-cells = <1>; + #address-cells = <0>; + riscv,ndev = <128>; + }; + }; +}; diff --git a/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts b/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts index 21c33f165ba9..91f3f9b987bc 100644 --- a/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts +++ b/arch/riscv/boot/dts/thead/th1520-beaglev-ahead.dts @@ -75,6 +75,17 @@ label = "led5"; }; }; + + hdmi-connector { + compatible = "hdmi-connector"; + type = "d"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; }; &osc { @@ -236,6 +247,20 @@ status = "okay"; }; +&dpu { + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_out_port { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_pins>; diff --git a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts index 4020c727f09e..7cb7d28683bc 100644 --- a/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts +++ b/arch/riscv/boot/dts/thead/th1520-lichee-pi-4a.dts @@ -28,9 +28,87 @@ chosen { stdout-path = "serial0:115200n8"; }; + + hdmi-connector { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + + thermal-zones { + cpu-thermal { + polling-delay = <1000>; + polling-delay-passive = <1000>; + thermal-sensors = <&pvt 0>; + + trips { + fan_config0: fan-trip0 { + temperature = <39000>; + hysteresis = <5000>; + type = "active"; + }; + + fan_config1: fan-trip1 { + temperature = <50000>; + hysteresis = <5000>; + type = "active"; + }; + + fan_config2: fan-trip2 { + temperature = <60000>; + hysteresis = <5000>; + type = "active"; + }; + }; + + cooling-maps { + map-active-0 { + cooling-device = <&fan 1 1>; + trip = <&fan_config0>; + }; + + map-active-1 { + cooling-device = <&fan 2 2>; + trip = <&fan_config1>; + }; + + map-active-2 { + cooling-device = <&fan 3 3>; + trip = <&fan_config2>; + }; + }; + }; + }; + + fan: pwm-fan { + pinctrl-names = "default"; + pinctrl-0 = <&fan_pins>; + compatible = "pwm-fan"; + #cooling-cells = <2>; + pwms = <&pwm 1 10000000 0>; + cooling-levels = <0 66 196 255>; + }; + }; &padctrl0_apsys { + fan_pins: fan-0 { + pwm1-pins { + pins = "GPIO3_3"; /* PWM1 */ + function = "pwm"; + bias-disable; + drive-strength = <25>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + uart0_pins: uart0-0 { tx-pins { pins = "UART0_TXD"; @@ -54,6 +132,20 @@ }; }; +&dpu { + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_out_port { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + &uart0 { pinctrl-names = "default"; pinctrl-0 = <&uart0_pins>; diff --git a/arch/riscv/boot/dts/thead/th1520.dtsi b/arch/riscv/boot/dts/thead/th1520.dtsi index 527336417765..5e91dc1d2b9b 100644 --- a/arch/riscv/boot/dts/thead/th1520.dtsi +++ b/arch/riscv/boot/dts/thead/th1520.dtsi @@ -6,6 +6,8 @@ #include <dt-bindings/interrupt-controller/irq.h> #include <dt-bindings/clock/thead,th1520-clk-ap.h> +#include <dt-bindings/power/thead,th1520-power.h> +#include <dt-bindings/reset/thead,th1520-reset.h> / { compatible = "thead,th1520"; @@ -22,8 +24,11 @@ device_type = "cpu"; riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", - "zifencei", "zihpm"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <0>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -46,8 +51,11 @@ device_type = "cpu"; riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", - "zifencei", "zihpm"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <1>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -70,8 +78,11 @@ device_type = "cpu"; riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", - "zifencei", "zihpm"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <2>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -94,8 +105,11 @@ device_type = "cpu"; riscv,isa = "rv64imafdc"; riscv,isa-base = "rv64i"; - riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "zicntr", "zicsr", - "zifencei", "zihpm"; + riscv,isa-extensions = "i", "m", "a", "f", "d", "c", + "ziccrse", "zicntr", "zicsr", + "zifencei", "zihpm", "zfh", + "xtheadvector"; + thead,vlenb = <16>; reg = <3>; i-cache-block-size = <64>; i-cache-size = <65536>; @@ -223,12 +237,28 @@ #clock-cells = <0>; }; + gpu_mem_clk: mem-clk { + compatible = "fixed-clock"; + clock-frequency = <0>; + clock-output-names = "gpu_mem_clk"; + #clock-cells = <0>; + }; + stmmac_axi_config: stmmac-axi-config { snps,wr_osr_lmt = <15>; snps,rd_osr_lmt = <15>; snps,blen = <0 0 64 32 0 0 0>; }; + aon: aon { + compatible = "thead,th1520-aon"; + mboxes = <&mbox_910t 1>; + mbox-names = "aon"; + resets = <&rst TH1520_RESET_ID_GPU_CLKGEN>; + reset-names = "gpu-clkgen"; + #power-domain-cells = <1>; + }; + soc { compatible = "simple-bus"; interrupt-parent = <&plic>; @@ -259,6 +289,12 @@ <&cpu3_intc 3>, <&cpu3_intc 7>; }; + rst_vi: reset-controller@ffe4040100 { + compatible = "thead,th1520-reset-vi"; + reg = <0xff 0xe4040100 0x0 0x8>; + #reset-cells = <1>; + }; + spi0: spi@ffe700c000 { compatible = "thead,th1520-spi", "snps,dw-apb-ssi"; reg = <0xff 0xe700c000 0x0 0x1000>; @@ -286,8 +322,9 @@ reg-names = "dwmac", "apb"; interrupts = <67 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq"; - clocks = <&clk CLK_GMAC_AXI>, <&clk CLK_GMAC1>; - clock-names = "stmmaceth", "pclk"; + clocks = <&clk CLK_GMAC_AXI>, <&clk CLK_GMAC1>, + <&clk CLK_PERISYS_APB4_HCLK>; + clock-names = "stmmaceth", "pclk", "apb"; snps,pbl = <32>; snps,fixed-burst; snps,multicast-filter-bins = <64>; @@ -308,8 +345,9 @@ reg-names = "dwmac", "apb"; interrupts = <66 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "macirq"; - clocks = <&clk CLK_GMAC_AXI>, <&clk CLK_GMAC0>; - clock-names = "stmmaceth", "pclk"; + clocks = <&clk CLK_GMAC_AXI>, <&clk CLK_GMAC0>, + <&clk CLK_PERISYS_APB4_HCLK>; + clock-names = "stmmaceth", "pclk", "apb"; snps,pbl = <32>; snps,fixed-burst; snps,multicast-filter-bins = <64>; @@ -482,6 +520,25 @@ status = "disabled"; }; + pwm: pwm@ffec01c000 { + compatible = "thead,th1520-pwm"; + reg = <0xff 0xec01c000 0x0 0x4000>; + clocks = <&clk CLK_PWM>; + #pwm-cells = <3>; + }; + + rst_misc: reset-controller@ffec02c000 { + compatible = "thead,th1520-reset-misc"; + reg = <0xff 0xec02c000 0x0 0x18>; + #reset-cells = <1>; + }; + + rst_vp: reset-controller@ffecc30000 { + compatible = "thead,th1520-reset-vp"; + reg = <0xff 0xecc30000 0x0 0x14>; + #reset-cells = <1>; + }; + clk: clock-controller@ffef010000 { compatible = "thead,th1520-clk-ap"; reg = <0xff 0xef010000 0x0 0x1000>; @@ -489,6 +546,111 @@ #clock-cells = <1>; }; + rst_ap: reset-controller@ffef014000 { + compatible = "thead,th1520-reset-ap"; + reg = <0xff 0xef014000 0x0 0x1000>; + #reset-cells = <1>; + }; + + rst_dsp: reset-controller@ffef040028 { + compatible = "thead,th1520-reset-dsp"; + reg = <0xff 0xef040028 0x0 0x4>; + #reset-cells = <1>; + }; + + gpu: gpu@ffef400000 { + compatible = "thead,th1520-gpu", "img,img-bxm-4-64", + "img,img-rogue"; + reg = <0xff 0xef400000 0x0 0x100000>; + interrupt-parent = <&plic>; + interrupts = <102 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk_vo CLK_GPU_CORE>, + <&gpu_mem_clk>, + <&clk_vo CLK_GPU_CFG_ACLK>; + clock-names = "core", "mem", "sys"; + power-domains = <&aon TH1520_GPU_PD>; + resets = <&rst TH1520_RESET_ID_GPU>; + }; + + rst: reset-controller@ffef528000 { + compatible = "thead,th1520-reset"; + reg = <0xff 0xef528000 0x0 0x4f>; + #reset-cells = <1>; + }; + + clk_vo: clock-controller@ffef528050 { + compatible = "thead,th1520-clk-vo"; + reg = <0xff 0xef528050 0x0 0xfb0>; + clocks = <&clk CLK_VIDEO_PLL>; + #clock-cells = <1>; + }; + + hdmi: hdmi@ffef540000 { + compatible = "thead,th1520-dw-hdmi"; + reg = <0xff 0xef540000 0x0 0x40000>; + reg-io-width = <4>; + interrupts = <111 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk_vo CLK_HDMI_PCLK>, + <&clk_vo CLK_HDMI_SFR>, + <&clk_vo CLK_HDMI_CEC>, + <&clk_vo CLK_HDMI_PIXCLK>; + clock-names = "iahb", "isfr", "cec", "pix"; + resets = <&rst TH1520_RESET_ID_HDMI>, + <&rst TH1520_RESET_ID_HDMI_APB>; + reset-names = "main", "apb"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in: endpoint { + remote-endpoint = <&dpu_out_dp1>; + }; + }; + + hdmi_out_port: port@1 { + reg = <1>; + }; + }; + }; + + dpu: display@ffef600000 { + compatible = "thead,th1520-dc8200", "verisilicon,dc"; + reg = <0xff 0xef600000 0x0 0x100000>; + interrupts = <93 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk_vo CLK_DPU_CCLK>, + <&clk_vo CLK_DPU_ACLK>, + <&clk_vo CLK_DPU_HCLK>, + <&clk_vo CLK_DPU_PIXELCLK0>, + <&clk_vo CLK_DPU_PIXELCLK1>; + clock-names = "core", "axi", "ahb", "pix0", "pix1"; + resets = <&rst TH1520_RESET_ID_DPU_CORE>, + <&rst TH1520_RESET_ID_DPU_AXI>, + <&rst TH1520_RESET_ID_DPU_AHB>; + reset-names = "core", "axi", "ahb"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dpu_port1: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dpu_out_dp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&hdmi_in>; + }; + }; + }; + }; + dmac0: dma-controller@ffefc00000 { compatible = "snps,axi-dma-1.01a"; reg = <0xff 0xefc00000 0x0 0x1000>; @@ -634,6 +796,13 @@ }; }; + rst_ao: reset-controller@fffff44000 { + compatible = "thead,th1520-reset-ao"; + reg = <0xff 0xfff44000 0x0 0x2000>; + #reset-cells = <1>; + status = "reserved"; + }; + padctrl_aosys: pinctrl@fffff4a000 { compatible = "thead,th1520-pinctrl"; reg = <0xff 0xfff4a000 0x0 0x2000>; @@ -641,6 +810,21 @@ thead,pad-group = <1>; }; + pvt: pvt@fffff4e000 { + compatible = "moortec,mr75203"; + reg = <0xff 0xfff4e000 0x0 0x80>, + <0xff 0xfff4e080 0x0 0x100>, + <0xff 0xfff4e180 0x0 0x680>, + <0xff 0xfff4e800 0x0 0x600>; + reg-names = "common", "ts", "pd", "vm"; + clocks = <&aonsys_clk>; + #thermal-sensor-cells = <1>; + moortec,ts-coeff-g = <42740>; + moortec,ts-coeff-h = <220500>; + moortec,ts-coeff-j = <(-160)>; + moortec,ts-coeff-cal5 = <4094>; + }; + gpio@fffff52000 { compatible = "snps,dw-apb-gpio"; reg = <0xff 0xfff52000 0x0 0x1000>; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 0f7dcbe3c45b..c2c37327b987 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -18,20 +18,19 @@ CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_PERF=y CONFIG_CGROUP_BPF=y -CONFIG_NAMESPACES=y CONFIG_USER_NS=y CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -# CONFIG_SYSFS_SYSCALL is not set CONFIG_PROFILING=y +CONFIG_ARCH_ANDES=y +CONFIG_ARCH_ANLOGIC=y CONFIG_ARCH_MICROCHIP=y -CONFIG_ARCH_RENESAS=y CONFIG_ARCH_SIFIVE=y CONFIG_ARCH_SOPHGO=y CONFIG_ARCH_SPACEMIT=y CONFIG_SOC_STARFIVE=y CONFIG_ARCH_SUNXI=y +CONFIG_ARCH_TENSTORRENT=y CONFIG_ARCH_THEAD=y CONFIG_ARCH_VIRT=y CONFIG_ARCH_CANAAN=y @@ -138,10 +137,13 @@ CONFIG_MACB=y CONFIG_E1000E=y CONFIG_R8169=y CONFIG_STMMAC_ETH=m +CONFIG_DWMAC_THEAD=m CONFIG_MICREL_PHY=y CONFIG_MICROSEMI_PHY=y CONFIG_MOTORCOMM_PHY=y CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_GPIO=y CONFIG_KEYBOARD_SUN4I_LRADC=m CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y @@ -155,21 +157,22 @@ CONFIG_HW_RANDOM_JH7110=m CONFIG_I2C=y CONFIG_I2C_CHARDEV=m CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MV64XXX=m CONFIG_SPI=y CONFIG_SPI_CADENCE_QUADSPI=m +CONFIG_SPI_FSL_QUADSPI=m CONFIG_SPI_PL022=m CONFIG_SPI_SIFIVE=y CONFIG_SPI_SUN6I=y # CONFIG_PTP_1588_CLOCK is not set +CONFIG_PINCTRL_TH1520=y CONFIG_PINCTRL_SOPHGO_CV1800B=y CONFIG_PINCTRL_SOPHGO_CV1812H=y CONFIG_PINCTRL_SOPHGO_SG2000=y CONFIG_PINCTRL_SOPHGO_SG2002=y -CONFIG_PINCTRL_TH1520=y CONFIG_GPIO_DWAPB=y CONFIG_GPIO_SIFIVE=y +CONFIG_GPIO_SPACEMIT_K1=y CONFIG_POWER_RESET_GPIO_RESTART=y CONFIG_SENSORS_SFCTEMP=m CONFIG_CPU_THERMAL=y @@ -182,6 +185,7 @@ CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_REGULATOR_AXP20X=y CONFIG_REGULATOR_GPIO=y CONFIG_MEDIA_SUPPORT=m +CONFIG_MEDIA_PLATFORM_SUPPORT=y CONFIG_VIDEO_CADENCE_CSI2RX=m CONFIG_DRM=m CONFIG_DRM_RADEON=m @@ -193,7 +197,6 @@ CONFIG_SOUND=y CONFIG_SND=y CONFIG_SND_SOC=y CONFIG_SND_DESIGNWARE_I2S=m -CONFIG_SND_SOC_STARFIVE=m CONFIG_SND_SOC_JH7110_PWMDAC=m CONFIG_SND_SOC_JH7110_TDM=m CONFIG_SND_SOC_WM8978=m @@ -202,7 +205,6 @@ CONFIG_USB=y CONFIG_USB_OTG=y CONFIG_USB_XHCI_HCD=y CONFIG_USB_XHCI_PLATFORM=y -# CONFIG_USB_XHCI_RCAR is not set CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_HCD_PLATFORM=y CONFIG_USB_OHCI_HCD=y @@ -233,6 +235,7 @@ CONFIG_MMC=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_OF_DWCMSHC=y +CONFIG_MMC_SDHCI_OF_K1=y CONFIG_MMC_SDHCI_CADENCE=y CONFIG_MMC_SPI=y CONFIG_MMC_DW=y @@ -243,7 +246,7 @@ CONFIG_RTC_DRV_SUN6I=y CONFIG_DMADEVICES=y CONFIG_DMA_SUN6I=m CONFIG_DW_AXI_DMAC=y -CONFIG_DWMAC_THEAD=m +CONFIG_MMP_PDMA=m CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y CONFIG_VIRTIO_INPUT=y @@ -252,6 +255,8 @@ CONFIG_CLK_SOPHGO_CV1800=y CONFIG_CLK_SOPHGO_SG2042_PLL=y CONFIG_CLK_SOPHGO_SG2042_CLKGEN=y CONFIG_CLK_SOPHGO_SG2042_RPGATE=y +CONFIG_SPACEMIT_K1_CCU=y +CONFIG_SPACEMIT_K3_CCU=y CONFIG_SUN8I_DE2_CCU=m CONFIG_SUN50I_IOMMU=y CONFIG_RPMSG_CHAR=y @@ -259,6 +264,8 @@ CONFIG_RPMSG_CTRL=y CONFIG_RPMSG_VIRTIO=y CONFIG_PM_DEVFREQ=y CONFIG_IIO=y +CONFIG_PWM=y +CONFIG_PWM_PXA=m CONFIG_THEAD_C900_ACLINT_SSWI=y CONFIG_PHY_SUN4I_USB=m CONFIG_PHY_STARFIVE_JH7110_DPHY_RX=m @@ -288,7 +295,7 @@ CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_APPARMOR=y @@ -296,25 +303,7 @@ CONFIG_DEFAULT_SECURITY_DAC=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_FS=y -CONFIG_DEBUG_PAGEALLOC=y -CONFIG_SCHED_STACK_END_CHECK=y -CONFIG_DEBUG_VM=y -CONFIG_DEBUG_VM_PGFLAGS=y -CONFIG_DEBUG_MEMORY_INIT=y -CONFIG_DEBUG_PER_CPU_MAPS=y -CONFIG_SOFTLOCKUP_DETECTOR=y -CONFIG_WQ_WATCHDOG=y -CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_MUTEXES=y -CONFIG_DEBUG_RWSEMS=y -CONFIG_DEBUG_ATOMIC_SLEEP=y -CONFIG_DEBUG_LIST=y -CONFIG_DEBUG_PLIST=y -CONFIG_DEBUG_SG=y -# CONFIG_RCU_TRACE is not set -CONFIG_RCU_EQS_DEBUG=y -# CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set CONFIG_MEMTEST=y diff --git a/arch/riscv/configs/hardening.config b/arch/riscv/configs/hardening.config new file mode 100644 index 000000000000..089f4cee82f4 --- /dev/null +++ b/arch/riscv/configs/hardening.config @@ -0,0 +1,4 @@ +# RISCV specific kernel hardening options + +# Enable control flow integrity support for usermode. +CONFIG_RISCV_USER_CFI=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 87ff5a1233af..544c52067dc2 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -35,7 +35,7 @@ CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y CONFIG_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" +CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set @@ -55,7 +55,6 @@ CONFIG_DEVTMPFS_MOUNT=y # CONFIG_HW_RANDOM is not set # CONFIG_DEVMEM is not set CONFIG_I2C=y -# CONFIG_I2C_COMPAT is not set CONFIG_I2C_CHARDEV=y # CONFIG_I2C_HELPER_AUTO is not set CONFIG_I2C_DESIGNWARE_CORE=y @@ -89,7 +88,6 @@ CONFIG_PRINTK_TIME=y # CONFIG_FRAME_POINTER is not set # CONFIG_DEBUG_MISC is not set CONFIG_PANIC_ON_OOPS=y -# CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set # CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index 95cbd574f291..4a826e30fa3e 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -27,7 +27,7 @@ CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro" CONFIG_CMDLINE_FORCE=y CONFIG_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" +CONFIG_BUILTIN_DTB_NAME="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set @@ -86,7 +86,6 @@ CONFIG_PRINTK_TIME=y # CONFIG_FRAME_POINTER is not set # CONFIG_DEBUG_MISC is not set CONFIG_PANIC_ON_OOPS=y -# CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set # CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index d4b03dc3c2c0..4c38049633b7 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -48,7 +48,6 @@ CONFIG_VIRTIO_BLK=y # CONFIG_LEGACY_PTYS is not set # CONFIG_LDISC_AUTOLOAD is not set CONFIG_SERIAL_8250=y -# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_NR_UARTS=1 CONFIG_SERIAL_8250_RUNTIME_UARTS=1 @@ -67,7 +66,6 @@ CONFIG_EXT2_FS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_LSM="[]" CONFIG_PRINTK_TIME=y -# CONFIG_SCHED_DEBUG is not set # CONFIG_RCU_TRACE is not set # CONFIG_FTRACE is not set # CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index ad58dad9a580..6905232ddb03 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -4,12 +4,11 @@ menu "Accelerated Cryptographic Algorithms for CPU (riscv)" config CRYPTO_AES_RISCV64 tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_ALGAPI + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_LIB_AES select CRYPTO_SKCIPHER help - Block cipher: AES cipher algorithms Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS Architecture: riscv64 using: @@ -18,64 +17,10 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_CHACHA_RISCV64 - tristate "Ciphers: ChaCha" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - help - Length-preserving ciphers: ChaCha20 stream cipher algorithm - - Architecture: riscv64 using: - - Zvkb vector crypto extension - -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - -config CRYPTO_SHA256_RISCV64 - tristate "Hash functions: SHA-224 and SHA-256" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SHA256 - help - SHA-224 and SHA-256 secure hash algorithm (FIPS 180) - - Architecture: riscv64 using: - - Zvknha or Zvknhb vector crypto extensions - - Zvkb vector crypto extension - -config CRYPTO_SHA512_RISCV64 - tristate "Hash functions: SHA-384 and SHA-512" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_SHA512 - help - SHA-384 and SHA-512 secure hash algorithm (FIPS 180) - - Architecture: riscv64 using: - - Zvknhb vector crypto extension - - Zvkb vector crypto extension - -config CRYPTO_SM3_RISCV64 - tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_HASH - select CRYPTO_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: riscv64 using: - - Zvksh vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS select CRYPTO_ALGAPI select CRYPTO_SM4 help diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 247c7bc7288c..8cf31db57fc4 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,20 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o -chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o - -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - -obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o -sha256-riscv64-y := sha256-riscv64-glue.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o - -obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o -sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o - -obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o -sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o - obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/aes-macros.S b/arch/riscv/crypto/aes-macros.S index d1a258d04bc7..1384164621a5 100644 --- a/arch/riscv/crypto/aes-macros.S +++ b/arch/riscv/crypto/aes-macros.S @@ -51,8 +51,10 @@ // - If AES-256, loads round keys into v1-v15 and continues onwards. // // Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1. -.macro aes_begin keyp, label128, label192 +.macro aes_begin keyp, label128, label192, key_len +.ifb \key_len lwu t0, 480(\keyp) // t0 = key length in bytes +.endif li t1, 24 // t1 = key length for AES-192 vsetivli zero, 4, e32, m1, ta, ma vle32.v v1, (\keyp) @@ -76,12 +78,20 @@ vle32.v v10, (\keyp) addi \keyp, \keyp, 16 vle32.v v11, (\keyp) +.ifb \key_len blt t0, t1, \label128 // If AES-128, goto label128. +.else + blt \key_len, t1, \label128 // If AES-128, goto label128. +.endif addi \keyp, \keyp, 16 vle32.v v12, (\keyp) addi \keyp, \keyp, 16 vle32.v v13, (\keyp) +.ifb \key_len beq t0, t1, \label192 // If AES-192, goto label192. +.else + beq \key_len, t1, \label192 // If AES-192, goto label192. +.endif // Else, it's AES-256. addi \keyp, \keyp, 16 vle32.v v14, (\keyp) diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c index f814ee048555..8bbf7f348c23 100644 --- a/arch/riscv/crypto/aes-riscv64-glue.c +++ b/arch/riscv/crypto/aes-riscv64-glue.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * AES using the RISC-V vector crypto extensions. Includes the bare block - * cipher and the ECB, CBC, CBC-CTS, CTR, and XTS modes. + * AES modes using the RISC-V vector crypto extensions * * Copyright (C) 2023 VRULL GmbH * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> @@ -15,7 +14,6 @@ #include <asm/simd.h> #include <asm/vector.h> #include <crypto/aes.h> -#include <crypto/internal/cipher.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> @@ -23,13 +21,6 @@ #include <linux/linkage.h> #include <linux/module.h> -asmlinkage void aes_encrypt_zvkned(const struct crypto_aes_ctx *key, - const u8 in[AES_BLOCK_SIZE], - u8 out[AES_BLOCK_SIZE]); -asmlinkage void aes_decrypt_zvkned(const struct crypto_aes_ctx *key, - const u8 in[AES_BLOCK_SIZE], - u8 out[AES_BLOCK_SIZE]); - asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key, const u8 *in, u8 *out, size_t len); asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key, @@ -86,14 +77,6 @@ static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx, return aes_expandkey(ctx, key, keylen); } -static int riscv64_aes_setkey_cipher(struct crypto_tfm *tfm, - const u8 *key, unsigned int keylen) -{ - struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - return riscv64_aes_setkey(ctx, key, keylen); -} - static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { @@ -102,34 +85,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm, return riscv64_aes_setkey(ctx, key, keylen); } -/* Bare AES, without a mode of operation */ - -static void riscv64_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - aes_encrypt_zvkned(ctx, src, dst); - kernel_vector_end(); - } else { - aes_encrypt(ctx, dst, src); - } -} - -static void riscv64_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - aes_decrypt_zvkned(ctx, src, dst); - kernel_vector_end(); - } else { - aes_decrypt(ctx, dst, src); - } -} - /* AES-ECB */ static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc) @@ -338,7 +293,7 @@ static int riscv64_aes_ctr_crypt(struct skcipher_request *req) struct riscv64_aes_xts_ctx { struct crypto_aes_ctx ctx1; - struct crypto_aes_ctx ctx2; + struct aes_enckey tweak_key; }; static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, @@ -348,7 +303,7 @@ static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, return xts_verify_key(tfm, key, keylen) ?: riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?: - riscv64_aes_setkey(&ctx->ctx2, key + keylen / 2, keylen / 2); + aes_prepareenckey(&ctx->tweak_key, key + keylen / 2, keylen / 2); } static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc) @@ -366,9 +321,7 @@ static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc) return -EINVAL; /* Encrypt the IV with the tweak key to get the first tweak. */ - kernel_vector_begin(); - aes_encrypt_zvkned(&ctx->ctx2, req->iv, req->iv); - kernel_vector_end(); + aes_encrypt(&ctx->tweak_key, req->iv, req->iv); err = skcipher_walk_virt(&walk, req, false); @@ -456,23 +409,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req) /* Algorithm definitions */ -static struct crypto_alg riscv64_zvkned_aes_cipher_alg = { - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto_aes_ctx), - .cra_priority = 300, - .cra_name = "aes", - .cra_driver_name = "aes-riscv64-zvkned", - .cra_cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = riscv64_aes_setkey_cipher, - .cia_encrypt = riscv64_aes_encrypt, - .cia_decrypt = riscv64_aes_decrypt, - }, - .cra_module = THIS_MODULE, -}; - static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = { { .setkey = riscv64_aes_setkey_skcipher, @@ -574,15 +510,11 @@ static int __init riscv64_aes_mod_init(void) if (riscv_isa_extension_available(NULL, ZVKNED) && riscv_vector_vlen() >= 128) { - err = crypto_register_alg(&riscv64_zvkned_aes_cipher_alg); - if (err) - return err; - err = crypto_register_skciphers( riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); if (err) - goto unregister_zvkned_cipher_alg; + return err; if (riscv_isa_extension_available(NULL, ZVKB)) { err = crypto_register_skcipher( @@ -607,8 +539,6 @@ unregister_zvkned_zvkb_skcipher_alg: unregister_zvkned_skcipher_algs: crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); -unregister_zvkned_cipher_alg: - crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg); return err; } @@ -620,7 +550,6 @@ static void __exit riscv64_aes_mod_exit(void) crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg); crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs, ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs)); - crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg); } module_init(riscv64_aes_mod_init); diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S index 23d063f94ce6..d0fc4581a380 100644 --- a/arch/riscv/crypto/aes-riscv64-zvkned.S +++ b/arch/riscv/crypto/aes-riscv64-zvkned.S @@ -56,33 +56,6 @@ #define LEN a3 #define IVP a4 -.macro __aes_crypt_zvkned enc, keylen - vle32.v v16, (INP) - aes_crypt v16, \enc, \keylen - vse32.v v16, (OUTP) - ret -.endm - -.macro aes_crypt_zvkned enc - aes_begin KEYP, 128f, 192f - __aes_crypt_zvkned \enc, 256 -128: - __aes_crypt_zvkned \enc, 128 -192: - __aes_crypt_zvkned \enc, 192 -.endm - -// void aes_encrypt_zvkned(const struct crypto_aes_ctx *key, -// const u8 in[16], u8 out[16]); -SYM_FUNC_START(aes_encrypt_zvkned) - aes_crypt_zvkned 1 -SYM_FUNC_END(aes_encrypt_zvkned) - -// Same prototype and calling convention as the encryption function -SYM_FUNC_START(aes_decrypt_zvkned) - aes_crypt_zvkned 0 -SYM_FUNC_END(aes_decrypt_zvkned) - .macro __aes_ecb_crypt enc, keylen srli t0, LEN, 2 // t0 is the remaining length in 32-bit words. It's a multiple of 4. diff --git a/arch/riscv/crypto/chacha-riscv64-glue.c b/arch/riscv/crypto/chacha-riscv64-glue.c deleted file mode 100644 index 10b46f36375a..000000000000 --- a/arch/riscv/crypto/chacha-riscv64-glue.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * ChaCha20 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/skcipher.h> -#include <linux/linkage.h> -#include <linux/module.h> - -asmlinkage void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, - size_t len, const u32 iv[4]); - -static int riscv64_chacha20_crypt(struct skcipher_request *req) -{ - u32 iv[CHACHA_IV_SIZE / sizeof(u32)]; - u8 block_buffer[CHACHA_BLOCK_SIZE]; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - unsigned int tail_bytes; - int err; - - iv[0] = get_unaligned_le32(req->iv); - iv[1] = get_unaligned_le32(req->iv + 4); - iv[2] = get_unaligned_le32(req->iv + 8); - iv[3] = get_unaligned_le32(req->iv + 12); - - err = skcipher_walk_virt(&walk, req, false); - while (walk.nbytes) { - nbytes = walk.nbytes & ~(CHACHA_BLOCK_SIZE - 1); - tail_bytes = walk.nbytes & (CHACHA_BLOCK_SIZE - 1); - kernel_vector_begin(); - if (nbytes) { - chacha20_zvkb(ctx->key, walk.src.virt.addr, - walk.dst.virt.addr, nbytes, iv); - iv[0] += nbytes / CHACHA_BLOCK_SIZE; - } - if (walk.nbytes == walk.total && tail_bytes > 0) { - memcpy(block_buffer, walk.src.virt.addr + nbytes, - tail_bytes); - chacha20_zvkb(ctx->key, block_buffer, block_buffer, - CHACHA_BLOCK_SIZE, iv); - memcpy(walk.dst.virt.addr + nbytes, block_buffer, - tail_bytes); - tail_bytes = 0; - } - kernel_vector_end(); - - err = skcipher_walk_done(&walk, tail_bytes); - } - - return err; -} - -static struct skcipher_alg riscv64_chacha_alg = { - .setkey = chacha20_setkey, - .encrypt = riscv64_chacha20_crypt, - .decrypt = riscv64_chacha20_crypt, - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .walksize = 4 * CHACHA_BLOCK_SIZE, - .base = { - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct chacha_ctx), - .cra_priority = 300, - .cra_name = "chacha20", - .cra_driver_name = "chacha20-riscv64-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_chacha_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_skcipher(&riscv64_chacha_alg); - - return -ENODEV; -} - -static void __exit riscv64_chacha_mod_exit(void) -{ - crypto_unregister_skcipher(&riscv64_chacha_alg); -} - -module_init(riscv64_chacha_mod_init); -module_exit(riscv64_chacha_mod_exit); - -MODULE_DESCRIPTION("ChaCha20 (RISC-V accelerated)"); -MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("chacha20"); diff --git a/arch/riscv/crypto/chacha-riscv64-zvkb.S b/arch/riscv/crypto/chacha-riscv64-zvkb.S deleted file mode 100644 index bf057737ac69..000000000000 --- a/arch/riscv/crypto/chacha-riscv64-zvkb.S +++ /dev/null @@ -1,294 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') - -#include <linux/linkage.h> - -.text -.option arch, +zvkb - -#define KEYP a0 -#define INP a1 -#define OUTP a2 -#define LEN a3 -#define IVP a4 - -#define CONSTS0 a5 -#define CONSTS1 a6 -#define CONSTS2 a7 -#define CONSTS3 t0 -#define TMP t1 -#define VL t2 -#define STRIDE t3 -#define NROUNDS t4 -#define KEY0 s0 -#define KEY1 s1 -#define KEY2 s2 -#define KEY3 s3 -#define KEY4 s4 -#define KEY5 s5 -#define KEY6 s6 -#define KEY7 s7 -#define COUNTER s8 -#define NONCE0 s9 -#define NONCE1 s10 -#define NONCE2 s11 - -.macro chacha_round a0, b0, c0, d0, a1, b1, c1, d1, \ - a2, b2, c2, d2, a3, b3, c3, d3 - // a += b; d ^= a; d = rol(d, 16); - vadd.vv \a0, \a0, \b0 - vadd.vv \a1, \a1, \b1 - vadd.vv \a2, \a2, \b2 - vadd.vv \a3, \a3, \b3 - vxor.vv \d0, \d0, \a0 - vxor.vv \d1, \d1, \a1 - vxor.vv \d2, \d2, \a2 - vxor.vv \d3, \d3, \a3 - vror.vi \d0, \d0, 32 - 16 - vror.vi \d1, \d1, 32 - 16 - vror.vi \d2, \d2, 32 - 16 - vror.vi \d3, \d3, 32 - 16 - - // c += d; b ^= c; b = rol(b, 12); - vadd.vv \c0, \c0, \d0 - vadd.vv \c1, \c1, \d1 - vadd.vv \c2, \c2, \d2 - vadd.vv \c3, \c3, \d3 - vxor.vv \b0, \b0, \c0 - vxor.vv \b1, \b1, \c1 - vxor.vv \b2, \b2, \c2 - vxor.vv \b3, \b3, \c3 - vror.vi \b0, \b0, 32 - 12 - vror.vi \b1, \b1, 32 - 12 - vror.vi \b2, \b2, 32 - 12 - vror.vi \b3, \b3, 32 - 12 - - // a += b; d ^= a; d = rol(d, 8); - vadd.vv \a0, \a0, \b0 - vadd.vv \a1, \a1, \b1 - vadd.vv \a2, \a2, \b2 - vadd.vv \a3, \a3, \b3 - vxor.vv \d0, \d0, \a0 - vxor.vv \d1, \d1, \a1 - vxor.vv \d2, \d2, \a2 - vxor.vv \d3, \d3, \a3 - vror.vi \d0, \d0, 32 - 8 - vror.vi \d1, \d1, 32 - 8 - vror.vi \d2, \d2, 32 - 8 - vror.vi \d3, \d3, 32 - 8 - - // c += d; b ^= c; b = rol(b, 7); - vadd.vv \c0, \c0, \d0 - vadd.vv \c1, \c1, \d1 - vadd.vv \c2, \c2, \d2 - vadd.vv \c3, \c3, \d3 - vxor.vv \b0, \b0, \c0 - vxor.vv \b1, \b1, \c1 - vxor.vv \b2, \b2, \c2 - vxor.vv \b3, \b3, \c3 - vror.vi \b0, \b0, 32 - 7 - vror.vi \b1, \b1, 32 - 7 - vror.vi \b2, \b2, 32 - 7 - vror.vi \b3, \b3, 32 - 7 -.endm - -// void chacha20_zvkb(const u32 key[8], const u8 *in, u8 *out, size_t len, -// const u32 iv[4]); -// -// |len| must be nonzero and a multiple of 64 (CHACHA_BLOCK_SIZE). -// The counter is treated as 32-bit, following the RFC7539 convention. -SYM_FUNC_START(chacha20_zvkb) - srli LEN, LEN, 6 // Bytes to blocks - - addi sp, sp, -96 - sd s0, 0(sp) - sd s1, 8(sp) - sd s2, 16(sp) - sd s3, 24(sp) - sd s4, 32(sp) - sd s5, 40(sp) - sd s6, 48(sp) - sd s7, 56(sp) - sd s8, 64(sp) - sd s9, 72(sp) - sd s10, 80(sp) - sd s11, 88(sp) - - li STRIDE, 64 - - // Set up the initial state matrix in scalar registers. - li CONSTS0, 0x61707865 // "expa" little endian - li CONSTS1, 0x3320646e // "nd 3" little endian - li CONSTS2, 0x79622d32 // "2-by" little endian - li CONSTS3, 0x6b206574 // "te k" little endian - lw KEY0, 0(KEYP) - lw KEY1, 4(KEYP) - lw KEY2, 8(KEYP) - lw KEY3, 12(KEYP) - lw KEY4, 16(KEYP) - lw KEY5, 20(KEYP) - lw KEY6, 24(KEYP) - lw KEY7, 28(KEYP) - lw COUNTER, 0(IVP) - lw NONCE0, 4(IVP) - lw NONCE1, 8(IVP) - lw NONCE2, 12(IVP) - -.Lblock_loop: - // Set vl to the number of blocks to process in this iteration. - vsetvli VL, LEN, e32, m1, ta, ma - - // Set up the initial state matrix for the next VL blocks in v0-v15. - // v{i} holds the i'th 32-bit word of the state matrix for all blocks. - // Note that only the counter word, at index 12, differs across blocks. - vmv.v.x v0, CONSTS0 - vmv.v.x v1, CONSTS1 - vmv.v.x v2, CONSTS2 - vmv.v.x v3, CONSTS3 - vmv.v.x v4, KEY0 - vmv.v.x v5, KEY1 - vmv.v.x v6, KEY2 - vmv.v.x v7, KEY3 - vmv.v.x v8, KEY4 - vmv.v.x v9, KEY5 - vmv.v.x v10, KEY6 - vmv.v.x v11, KEY7 - vid.v v12 - vadd.vx v12, v12, COUNTER - vmv.v.x v13, NONCE0 - vmv.v.x v14, NONCE1 - vmv.v.x v15, NONCE2 - - // Load the first half of the input data for each block into v16-v23. - // v{16+i} holds the i'th 32-bit word for all blocks. - vlsseg8e32.v v16, (INP), STRIDE - - li NROUNDS, 20 -.Lnext_doubleround: - addi NROUNDS, NROUNDS, -2 - // column round - chacha_round v0, v4, v8, v12, v1, v5, v9, v13, \ - v2, v6, v10, v14, v3, v7, v11, v15 - // diagonal round - chacha_round v0, v5, v10, v15, v1, v6, v11, v12, \ - v2, v7, v8, v13, v3, v4, v9, v14 - bnez NROUNDS, .Lnext_doubleround - - // Load the second half of the input data for each block into v24-v31. - // v{24+i} holds the {8+i}'th 32-bit word for all blocks. - addi TMP, INP, 32 - vlsseg8e32.v v24, (TMP), STRIDE - - // Finalize the first half of the keystream for each block. - vadd.vx v0, v0, CONSTS0 - vadd.vx v1, v1, CONSTS1 - vadd.vx v2, v2, CONSTS2 - vadd.vx v3, v3, CONSTS3 - vadd.vx v4, v4, KEY0 - vadd.vx v5, v5, KEY1 - vadd.vx v6, v6, KEY2 - vadd.vx v7, v7, KEY3 - - // Encrypt/decrypt the first half of the data for each block. - vxor.vv v16, v16, v0 - vxor.vv v17, v17, v1 - vxor.vv v18, v18, v2 - vxor.vv v19, v19, v3 - vxor.vv v20, v20, v4 - vxor.vv v21, v21, v5 - vxor.vv v22, v22, v6 - vxor.vv v23, v23, v7 - - // Store the first half of the output data for each block. - vssseg8e32.v v16, (OUTP), STRIDE - - // Finalize the second half of the keystream for each block. - vadd.vx v8, v8, KEY4 - vadd.vx v9, v9, KEY5 - vadd.vx v10, v10, KEY6 - vadd.vx v11, v11, KEY7 - vid.v v0 - vadd.vx v12, v12, COUNTER - vadd.vx v13, v13, NONCE0 - vadd.vx v14, v14, NONCE1 - vadd.vx v15, v15, NONCE2 - vadd.vv v12, v12, v0 - - // Encrypt/decrypt the second half of the data for each block. - vxor.vv v24, v24, v8 - vxor.vv v25, v25, v9 - vxor.vv v26, v26, v10 - vxor.vv v27, v27, v11 - vxor.vv v29, v29, v13 - vxor.vv v28, v28, v12 - vxor.vv v30, v30, v14 - vxor.vv v31, v31, v15 - - // Store the second half of the output data for each block. - addi TMP, OUTP, 32 - vssseg8e32.v v24, (TMP), STRIDE - - // Update the counter, the remaining number of blocks, and the input and - // output pointers according to the number of blocks processed (VL). - add COUNTER, COUNTER, VL - sub LEN, LEN, VL - slli TMP, VL, 6 - add OUTP, OUTP, TMP - add INP, INP, TMP - bnez LEN, .Lblock_loop - - ld s0, 0(sp) - ld s1, 8(sp) - ld s2, 16(sp) - ld s3, 24(sp) - ld s4, 32(sp) - ld s5, 40(sp) - ld s6, 48(sp) - ld s7, 56(sp) - ld s8, 64(sp) - ld s9, 72(sp) - ld s10, 80(sp) - ld s11, 88(sp) - addi sp, sp, 96 - ret -SYM_FUNC_END(chacha20_zvkb) diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index 312e7891fd0a..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,168 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/ghash.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/linkage.h> -#include <linux/module.h> - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int len; - - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_BLOCK_SIZE - dctx->bytes); - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); - src += GHASH_BLOCK_SIZE - dctx->bytes; - srclen -= GHASH_BLOCK_SIZE - dctx->bytes; - dctx->bytes = 0; - } - - len = round_down(srclen, GHASH_BLOCK_SIZE); - if (len) { - riscv64_ghash_blocks(tctx, dctx, src, len); - src += len; - srclen -= len; - } - - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - - return 0; -} - -static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - int i; - - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++) - dctx->buffer[i] = 0; - - riscv64_ghash_blocks(tctx, dctx, dctx->buffer, - GHASH_BLOCK_SIZE); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .final = riscv64_ghash_final, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/arch/riscv/crypto/ghash-riscv64-zvkg.S deleted file mode 100644 index f2b43fb4d434..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector GCM/GMAC extension ('Zvkg') - -#include <linux/linkage.h> - -.text -.option arch, +zvkg - -#define ACCUMULATOR a0 -#define KEY a1 -#define DATA a2 -#define LEN a3 - -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); -// -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). -SYM_FUNC_START(ghash_zvkg) - vsetivli zero, 4, e32, m1, ta, ma - vle32.v v1, (ACCUMULATOR) - vle32.v v2, (KEY) -.Lnext_block: - vle32.v v3, (DATA) - vghsh.vv v1, v2, v3 - addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block - - vse32.v v1, (ACCUMULATOR) - ret -SYM_FUNC_END(ghash_zvkg) diff --git a/arch/riscv/crypto/sha256-riscv64-glue.c b/arch/riscv/crypto/sha256-riscv64-glue.c deleted file mode 100644 index 71e051e40a64..000000000000 --- a/arch/riscv/crypto/sha256-riscv64-glue.c +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-256 and SHA-224 using the RISC-V vector crypto extensions - * - * Copyright (C) 2022 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sha256_base.h> -#include <linux/linkage.h> -#include <linux/module.h> - -/* - * Note: the asm function only uses the 'state' field of struct sha256_state. - * It is assumed to be the first field. - */ -asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( - struct sha256_state *state, const u8 *data, int num_blocks); - -static int riscv64_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - /* - * Ensure struct sha256_state begins directly with the SHA-256 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha256_base_do_update(desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); - kernel_vector_end(); - } else { - crypto_sha256_update(desc, data, len); - } - return 0; -} - -static int riscv64_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha256_base_do_update( - desc, data, len, - sha256_transform_zvknha_or_zvknhb_zvkb); - sha256_base_do_finalize( - desc, sha256_transform_zvknha_or_zvknhb_zvkb); - kernel_vector_end(); - - return sha256_base_finish(desc, out); - } - - return crypto_sha256_finup(desc, data, len, out); -} - -static int riscv64_sha256_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sha256_finup(desc, NULL, 0, out); -} - -static int riscv64_sha256_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - riscv64_sha256_finup(desc, data, len, out); -} - -static struct shash_alg riscv64_sha256_algs[] = { - { - .init = sha256_base_init, - .update = riscv64_sha256_update, - .final = riscv64_sha256_final, - .finup = riscv64_sha256_finup, - .digest = riscv64_sha256_digest, - .descsize = sizeof(struct sha256_state), - .digestsize = SHA256_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_priority = 300, - .cra_name = "sha256", - .cra_driver_name = "sha256-riscv64-zvknha_or_zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, { - .init = sha224_base_init, - .update = riscv64_sha256_update, - .final = riscv64_sha256_final, - .finup = riscv64_sha256_finup, - .descsize = sizeof(struct sha256_state), - .digestsize = SHA224_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_priority = 300, - .cra_name = "sha224", - .cra_driver_name = "sha224-riscv64-zvknha_or_zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, -}; - -static int __init riscv64_sha256_mod_init(void) -{ - /* Both zvknha and zvknhb provide the SHA-256 instructions. */ - if ((riscv_isa_extension_available(NULL, ZVKNHA) || - riscv_isa_extension_available(NULL, ZVKNHB)) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shashes(riscv64_sha256_algs, - ARRAY_SIZE(riscv64_sha256_algs)); - - return -ENODEV; -} - -static void __exit riscv64_sha256_mod_exit(void) -{ - crypto_unregister_shashes(riscv64_sha256_algs, - ARRAY_SIZE(riscv64_sha256_algs)); -} - -module_init(riscv64_sha256_mod_init); -module_exit(riscv64_sha256_mod_exit); - -MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha224"); diff --git a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S b/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S deleted file mode 100644 index 8ebcc17de4dc..000000000000 --- a/arch/riscv/crypto/sha256-riscv64-zvknha_or_zvknhb-zvkb.S +++ /dev/null @@ -1,225 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector SHA-2 Secure Hash extension ('Zvknha' or 'Zvknhb') -// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') - -#include <linux/cfi_types.h> - -.text -.option arch, +zvknha, +zvkb - -#define STATEP a0 -#define DATA a1 -#define NUM_BLOCKS a2 - -#define STATEP_C a3 - -#define MASK v0 -#define INDICES v1 -#define W0 v2 -#define W1 v3 -#define W2 v4 -#define W3 v5 -#define VTMP v6 -#define FEBA v7 -#define HGDC v8 -#define K0 v10 -#define K1 v11 -#define K2 v12 -#define K3 v13 -#define K4 v14 -#define K5 v15 -#define K6 v16 -#define K7 v17 -#define K8 v18 -#define K9 v19 -#define K10 v20 -#define K11 v21 -#define K12 v22 -#define K13 v23 -#define K14 v24 -#define K15 v25 -#define PREV_FEBA v26 -#define PREV_HGDC v27 - -// Do 4 rounds of SHA-256. w0 contains the current 4 message schedule words. -// -// If not all the message schedule words have been computed yet, then this also -// computes 4 more message schedule words. w1-w3 contain the next 3 groups of 4 -// message schedule words; this macro computes the group after w3 and writes it -// to w0. This means that the next (w0, w1, w2, w3) is the current (w1, w2, w3, -// w0), so the caller must cycle through the registers accordingly. -.macro sha256_4rounds last, k, w0, w1, w2, w3 - vadd.vv VTMP, \k, \w0 - vsha2cl.vv HGDC, FEBA, VTMP - vsha2ch.vv FEBA, HGDC, VTMP -.if !\last - vmerge.vvm VTMP, \w2, \w1, MASK - vsha2ms.vv \w0, VTMP, \w3 -.endif -.endm - -.macro sha256_16rounds last, k0, k1, k2, k3 - sha256_4rounds \last, \k0, W0, W1, W2, W3 - sha256_4rounds \last, \k1, W1, W2, W3, W0 - sha256_4rounds \last, \k2, W2, W3, W0, W1 - sha256_4rounds \last, \k3, W3, W0, W1, W2 -.endm - -// void sha256_transform_zvknha_or_zvknhb_zvkb(u32 state[8], const u8 *data, -// int num_blocks); -SYM_TYPED_FUNC_START(sha256_transform_zvknha_or_zvknhb_zvkb) - - // Load the round constants into K0-K15. - vsetivli zero, 4, e32, m1, ta, ma - la t0, K256 - vle32.v K0, (t0) - addi t0, t0, 16 - vle32.v K1, (t0) - addi t0, t0, 16 - vle32.v K2, (t0) - addi t0, t0, 16 - vle32.v K3, (t0) - addi t0, t0, 16 - vle32.v K4, (t0) - addi t0, t0, 16 - vle32.v K5, (t0) - addi t0, t0, 16 - vle32.v K6, (t0) - addi t0, t0, 16 - vle32.v K7, (t0) - addi t0, t0, 16 - vle32.v K8, (t0) - addi t0, t0, 16 - vle32.v K9, (t0) - addi t0, t0, 16 - vle32.v K10, (t0) - addi t0, t0, 16 - vle32.v K11, (t0) - addi t0, t0, 16 - vle32.v K12, (t0) - addi t0, t0, 16 - vle32.v K13, (t0) - addi t0, t0, 16 - vle32.v K14, (t0) - addi t0, t0, 16 - vle32.v K15, (t0) - - // Setup mask for the vmerge to replace the first word (idx==0) in - // message scheduling. There are 4 words, so an 8-bit mask suffices. - vsetivli zero, 1, e8, m1, ta, ma - vmv.v.i MASK, 0x01 - - // Load the state. The state is stored as {a,b,c,d,e,f,g,h}, but we - // need {f,e,b,a},{h,g,d,c}. The dst vtype is e32m1 and the index vtype - // is e8mf4. We use index-load with the i8 indices {20, 16, 4, 0}, - // loaded using the 32-bit little endian value 0x00041014. - li t0, 0x00041014 - vsetivli zero, 1, e32, m1, ta, ma - vmv.v.x INDICES, t0 - addi STATEP_C, STATEP, 8 - vsetivli zero, 4, e32, m1, ta, ma - vluxei8.v FEBA, (STATEP), INDICES - vluxei8.v HGDC, (STATEP_C), INDICES - -.Lnext_block: - addi NUM_BLOCKS, NUM_BLOCKS, -1 - - // Save the previous state, as it's needed later. - vmv.v.v PREV_FEBA, FEBA - vmv.v.v PREV_HGDC, HGDC - - // Load the next 512-bit message block and endian-swap each 32-bit word. - vle32.v W0, (DATA) - vrev8.v W0, W0 - addi DATA, DATA, 16 - vle32.v W1, (DATA) - vrev8.v W1, W1 - addi DATA, DATA, 16 - vle32.v W2, (DATA) - vrev8.v W2, W2 - addi DATA, DATA, 16 - vle32.v W3, (DATA) - vrev8.v W3, W3 - addi DATA, DATA, 16 - - // Do the 64 rounds of SHA-256. - sha256_16rounds 0, K0, K1, K2, K3 - sha256_16rounds 0, K4, K5, K6, K7 - sha256_16rounds 0, K8, K9, K10, K11 - sha256_16rounds 1, K12, K13, K14, K15 - - // Add the previous state. - vadd.vv FEBA, FEBA, PREV_FEBA - vadd.vv HGDC, HGDC, PREV_HGDC - - // Repeat if more blocks remain. - bnez NUM_BLOCKS, .Lnext_block - - // Store the new state and return. - vsuxei8.v FEBA, (STATEP), INDICES - vsuxei8.v HGDC, (STATEP_C), INDICES - ret -SYM_FUNC_END(sha256_transform_zvknha_or_zvknhb_zvkb) - -.section ".rodata" -.p2align 2 -.type K256, @object -K256: - .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 - .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 - .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 - .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 - .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc - .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da - .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 - .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 - .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 - .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 - .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 - .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 - .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 - .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 - .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 - .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 -.size K256, . - K256 diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c deleted file mode 100644 index 43b56a08aeb5..000000000000 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-512 and SHA-384 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sha512_base.h> -#include <linux/linkage.h> -#include <linux/module.h> - -/* - * Note: the asm function only uses the 'state' field of struct sha512_state. - * It is assumed to be the first field. - */ -asmlinkage void sha512_transform_zvknhb_zvkb( - struct sha512_state *state, const u8 *data, int num_blocks); - -static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - /* - * Ensure struct sha512_state begins directly with the SHA-512 - * 512-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); - kernel_vector_end(); - } else { - crypto_sha512_update(desc, data, len); - } - return 0; -} - -static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sha512_base_do_update(desc, data, len, - sha512_transform_zvknhb_zvkb); - sha512_base_do_finalize(desc, sha512_transform_zvknhb_zvkb); - kernel_vector_end(); - - return sha512_base_finish(desc, out); - } - - return crypto_sha512_finup(desc, data, len, out); -} - -static int riscv64_sha512_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sha512_finup(desc, NULL, 0, out); -} - -static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_base_init(desc) ?: - riscv64_sha512_finup(desc, data, len, out); -} - -static struct shash_alg riscv64_sha512_algs[] = { - { - .init = sha512_base_init, - .update = riscv64_sha512_update, - .final = riscv64_sha512_final, - .finup = riscv64_sha512_finup, - .digest = riscv64_sha512_digest, - .descsize = sizeof(struct sha512_state), - .digestsize = SHA512_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_priority = 300, - .cra_name = "sha512", - .cra_driver_name = "sha512-riscv64-zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, { - .init = sha384_base_init, - .update = riscv64_sha512_update, - .final = riscv64_sha512_final, - .finup = riscv64_sha512_finup, - .descsize = sizeof(struct sha512_state), - .digestsize = SHA384_DIGEST_SIZE, - .base = { - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_priority = 300, - .cra_name = "sha384", - .cra_driver_name = "sha384-riscv64-zvknhb-zvkb", - .cra_module = THIS_MODULE, - }, - }, -}; - -static int __init riscv64_sha512_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKNHB) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); - - return -ENODEV; -} - -static void __exit riscv64_sha512_mod_exit(void) -{ - crypto_unregister_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); -} - -module_init(riscv64_sha512_mod_init); -module_exit(riscv64_sha512_mod_exit); - -MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha384"); diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S deleted file mode 100644 index 3a9ae210f915..000000000000 --- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S +++ /dev/null @@ -1,203 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector SHA-2 Secure Hash extension ('Zvknhb') -// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') - -#include <linux/cfi_types.h> - -.text -.option arch, +zvknhb, +zvkb - -#define STATEP a0 -#define DATA a1 -#define NUM_BLOCKS a2 - -#define STATEP_C a3 -#define K a4 - -#define MASK v0 -#define INDICES v1 -#define W0 v10 // LMUL=2 -#define W1 v12 // LMUL=2 -#define W2 v14 // LMUL=2 -#define W3 v16 // LMUL=2 -#define VTMP v20 // LMUL=2 -#define FEBA v22 // LMUL=2 -#define HGDC v24 // LMUL=2 -#define PREV_FEBA v26 // LMUL=2 -#define PREV_HGDC v28 // LMUL=2 - -// Do 4 rounds of SHA-512. w0 contains the current 4 message schedule words. -// -// If not all the message schedule words have been computed yet, then this also -// computes 4 more message schedule words. w1-w3 contain the next 3 groups of 4 -// message schedule words; this macro computes the group after w3 and writes it -// to w0. This means that the next (w0, w1, w2, w3) is the current (w1, w2, w3, -// w0), so the caller must cycle through the registers accordingly. -.macro sha512_4rounds last, w0, w1, w2, w3 - vle64.v VTMP, (K) - addi K, K, 32 - vadd.vv VTMP, VTMP, \w0 - vsha2cl.vv HGDC, FEBA, VTMP - vsha2ch.vv FEBA, HGDC, VTMP -.if !\last - vmerge.vvm VTMP, \w2, \w1, MASK - vsha2ms.vv \w0, VTMP, \w3 -.endif -.endm - -.macro sha512_16rounds last - sha512_4rounds \last, W0, W1, W2, W3 - sha512_4rounds \last, W1, W2, W3, W0 - sha512_4rounds \last, W2, W3, W0, W1 - sha512_4rounds \last, W3, W0, W1, W2 -.endm - -// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data, -// int num_blocks); -SYM_TYPED_FUNC_START(sha512_transform_zvknhb_zvkb) - - // Setup mask for the vmerge to replace the first word (idx==0) in - // message scheduling. There are 4 words, so an 8-bit mask suffices. - vsetivli zero, 1, e8, m1, ta, ma - vmv.v.i MASK, 0x01 - - // Load the state. The state is stored as {a,b,c,d,e,f,g,h}, but we - // need {f,e,b,a},{h,g,d,c}. The dst vtype is e64m2 and the index vtype - // is e8mf4. We use index-load with the i8 indices {40, 32, 8, 0}, - // loaded using the 32-bit little endian value 0x00082028. - li t0, 0x00082028 - vsetivli zero, 1, e32, m1, ta, ma - vmv.v.x INDICES, t0 - addi STATEP_C, STATEP, 16 - vsetivli zero, 4, e64, m2, ta, ma - vluxei8.v FEBA, (STATEP), INDICES - vluxei8.v HGDC, (STATEP_C), INDICES - -.Lnext_block: - la K, K512 - addi NUM_BLOCKS, NUM_BLOCKS, -1 - - // Save the previous state, as it's needed later. - vmv.v.v PREV_FEBA, FEBA - vmv.v.v PREV_HGDC, HGDC - - // Load the next 1024-bit message block and endian-swap each 64-bit word - vle64.v W0, (DATA) - vrev8.v W0, W0 - addi DATA, DATA, 32 - vle64.v W1, (DATA) - vrev8.v W1, W1 - addi DATA, DATA, 32 - vle64.v W2, (DATA) - vrev8.v W2, W2 - addi DATA, DATA, 32 - vle64.v W3, (DATA) - vrev8.v W3, W3 - addi DATA, DATA, 32 - - // Do the 80 rounds of SHA-512. - sha512_16rounds 0 - sha512_16rounds 0 - sha512_16rounds 0 - sha512_16rounds 0 - sha512_16rounds 1 - - // Add the previous state. - vadd.vv FEBA, FEBA, PREV_FEBA - vadd.vv HGDC, HGDC, PREV_HGDC - - // Repeat if more blocks remain. - bnez NUM_BLOCKS, .Lnext_block - - // Store the new state and return. - vsuxei8.v FEBA, (STATEP), INDICES - vsuxei8.v HGDC, (STATEP_C), INDICES - ret -SYM_FUNC_END(sha512_transform_zvknhb_zvkb) - -.section ".rodata" -.p2align 3 -.type K512, @object -K512: - .dword 0x428a2f98d728ae22, 0x7137449123ef65cd - .dword 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc - .dword 0x3956c25bf348b538, 0x59f111f1b605d019 - .dword 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 - .dword 0xd807aa98a3030242, 0x12835b0145706fbe - .dword 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 - .dword 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 - .dword 0x9bdc06a725c71235, 0xc19bf174cf692694 - .dword 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 - .dword 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 - .dword 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 - .dword 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 - .dword 0x983e5152ee66dfab, 0xa831c66d2db43210 - .dword 0xb00327c898fb213f, 0xbf597fc7beef0ee4 - .dword 0xc6e00bf33da88fc2, 0xd5a79147930aa725 - .dword 0x06ca6351e003826f, 0x142929670a0e6e70 - .dword 0x27b70a8546d22ffc, 0x2e1b21385c26c926 - .dword 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df - .dword 0x650a73548baf63de, 0x766a0abb3c77b2a8 - .dword 0x81c2c92e47edaee6, 0x92722c851482353b - .dword 0xa2bfe8a14cf10364, 0xa81a664bbc423001 - .dword 0xc24b8b70d0f89791, 0xc76c51a30654be30 - .dword 0xd192e819d6ef5218, 0xd69906245565a910 - .dword 0xf40e35855771202a, 0x106aa07032bbd1b8 - .dword 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 - .dword 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 - .dword 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb - .dword 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 - .dword 0x748f82ee5defb2fc, 0x78a5636f43172f60 - .dword 0x84c87814a1f0ab72, 0x8cc702081a6439ec - .dword 0x90befffa23631e28, 0xa4506cebde82bde9 - .dword 0xbef9a3f7b2c67915, 0xc67178f2e372532b - .dword 0xca273eceea26619c, 0xd186b8c721c0c207 - .dword 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 - .dword 0x06f067aa72176fba, 0x0a637dc5a2c898a6 - .dword 0x113f9804bef90dae, 0x1b710b35131c471b - .dword 0x28db77f523047d84, 0x32caab7b40c72493 - .dword 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c - .dword 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a - .dword 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 -.size K512, . - K512 diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c deleted file mode 100644 index e1737a970c7c..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SM3 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih <jerry.shih@sifive.com> - */ - -#include <asm/simd.h> -#include <asm/vector.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <crypto/sm3_base.h> -#include <linux/linkage.h> -#include <linux/module.h> - -/* - * Note: the asm function only uses the 'state' field of struct sm3_state. - * It is assumed to be the first field. - */ -asmlinkage void sm3_transform_zvksh_zvkb( - struct sm3_state *state, const u8 *data, int num_blocks); - -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - /* - * Ensure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sm3_base_do_update(desc, data, len, sm3_transform_zvksh_zvkb); - kernel_vector_end(); - } else { - sm3_update(shash_desc_ctx(desc), data, len); - } - return 0; -} - -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sm3_state *ctx; - - if (crypto_simd_usable()) { - kernel_vector_begin(); - if (len) - sm3_base_do_update(desc, data, len, - sm3_transform_zvksh_zvkb); - sm3_base_do_finalize(desc, sm3_transform_zvksh_zvkb); - kernel_vector_end(); - - return sm3_base_finish(desc, out); - } - - ctx = shash_desc_ctx(desc); - if (len) - sm3_update(ctx, data, len); - sm3_final(ctx, out); - - return 0; -} - -static int riscv64_sm3_final(struct shash_desc *desc, u8 *out) -{ - return riscv64_sm3_finup(desc, NULL, 0, out); -} - -static struct shash_alg riscv64_sm3_alg = { - .init = sm3_base_init, - .update = riscv64_sm3_update, - .final = riscv64_sm3_final, - .finup = riscv64_sm3_finup, - .descsize = sizeof(struct sm3_state), - .digestsize = SM3_DIGEST_SIZE, - .base = { - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_priority = 300, - .cra_name = "sm3", - .cra_driver_name = "sm3-riscv64-zvksh-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_sm3_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKSH) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_sm3_alg); - - return -ENODEV; -} - -static void __exit riscv64_sm3_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_sm3_alg); -} - -module_init(riscv64_sm3_mod_init); -module_exit(riscv64_sm3_mod_exit); - -MODULE_DESCRIPTION("SM3 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sm3"); diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S deleted file mode 100644 index a2b65d961c04..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ -// -// This file is dual-licensed, meaning that you can use it under your -// choice of either of the following two licenses: -// -// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the Apache License 2.0 (the "License"). You can obtain -// a copy in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html -// -// or -// -// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> -// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> -// Copyright 2024 Google LLC -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// The generated code of this file depends on the following RISC-V extensions: -// - RV64I -// - RISC-V Vector ('V') with VLEN >= 128 -// - RISC-V Vector SM3 Secure Hash extension ('Zvksh') -// - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') - -#include <linux/cfi_types.h> - -.text -.option arch, +zvksh, +zvkb - -#define STATEP a0 -#define DATA a1 -#define NUM_BLOCKS a2 - -#define STATE v0 // LMUL=2 -#define PREV_STATE v2 // LMUL=2 -#define W0 v4 // LMUL=2 -#define W1 v6 // LMUL=2 -#define VTMP v8 // LMUL=2 - -.macro sm3_8rounds i, w0, w1 - // Do 4 rounds using W_{0+i}..W_{7+i}. - vsm3c.vi STATE, \w0, \i + 0 - vslidedown.vi VTMP, \w0, 2 - vsm3c.vi STATE, VTMP, \i + 1 - - // Compute W_{4+i}..W_{11+i}. - vslidedown.vi VTMP, \w0, 4 - vslideup.vi VTMP, \w1, 4 - - // Do 4 rounds using W_{4+i}..W_{11+i}. - vsm3c.vi STATE, VTMP, \i + 2 - vslidedown.vi VTMP, VTMP, 2 - vsm3c.vi STATE, VTMP, \i + 3 - -.if \i < 28 - // Compute W_{16+i}..W_{23+i}. - vsm3me.vv \w0, \w1, \w0 -.endif - // For the next 8 rounds, w0 and w1 are swapped. -.endm - -// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); -SYM_TYPED_FUNC_START(sm3_transform_zvksh_zvkb) - - // Load the state and endian-swap each 32-bit word. - vsetivli zero, 8, e32, m2, ta, ma - vle32.v STATE, (STATEP) - vrev8.v STATE, STATE - -.Lnext_block: - addi NUM_BLOCKS, NUM_BLOCKS, -1 - - // Save the previous state, as it's needed later. - vmv.v.v PREV_STATE, STATE - - // Load the next 512-bit message block into W0-W1. - vle32.v W0, (DATA) - addi DATA, DATA, 32 - vle32.v W1, (DATA) - addi DATA, DATA, 32 - - // Do the 64 rounds of SM3. - sm3_8rounds 0, W0, W1 - sm3_8rounds 4, W1, W0 - sm3_8rounds 8, W0, W1 - sm3_8rounds 12, W1, W0 - sm3_8rounds 16, W0, W1 - sm3_8rounds 20, W1, W0 - sm3_8rounds 24, W0, W1 - sm3_8rounds 28, W1, W0 - - // XOR in the previous state. - vxor.vv STATE, STATE, PREV_STATE - - // Repeat if more blocks remain. - bnez NUM_BLOCKS, .Lnext_block - - // Store the new state and return. - vrev8.v STATE, STATE - vse32.v STATE, (STATEP) - ret -SYM_FUNC_END(sm3_transform_zvksh_zvkb) diff --git a/arch/riscv/errata/Makefile b/arch/riscv/errata/Makefile index f0da9d7b39c3..02a7a3335b1d 100644 --- a/arch/riscv/errata/Makefile +++ b/arch/riscv/errata/Makefile @@ -1,5 +1,9 @@ ifdef CONFIG_RELOCATABLE -KBUILD_CFLAGS += -fno-pie +# We can't use PIC/PIE when handling early-boot errata parsing, as the kernel +# doesn't have a GOT setup at that point. So instead just use medany: it's +# usually position-independent, so it should be good enough for the errata +# handling. +KBUILD_CFLAGS += -fno-pie -mcmodel=medany endif ifdef CONFIG_RISCV_ALTERNATIVE_EARLY @@ -9,5 +13,6 @@ endif endif obj-$(CONFIG_ERRATA_ANDES) += andes/ +obj-$(CONFIG_ERRATA_MIPS) += mips/ obj-$(CONFIG_ERRATA_SIFIVE) += sifive/ obj-$(CONFIG_ERRATA_THEAD) += thead/ diff --git a/arch/riscv/errata/mips/Makefile b/arch/riscv/errata/mips/Makefile new file mode 100644 index 000000000000..6278c389b801 --- /dev/null +++ b/arch/riscv/errata/mips/Makefile @@ -0,0 +1,5 @@ +ifdef CONFIG_RISCV_ALTERNATIVE_EARLY +CFLAGS_errata.o := -mcmodel=medany +endif + +obj-y += errata.o diff --git a/arch/riscv/errata/mips/errata.c b/arch/riscv/errata/mips/errata.c new file mode 100644 index 000000000000..2c3dc2259e93 --- /dev/null +++ b/arch/riscv/errata/mips/errata.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 MIPS. + */ + +#include <linux/memory.h> +#include <linux/module.h> +#include <asm/text-patching.h> +#include <asm/alternative.h> +#include <asm/errata_list.h> +#include <asm/vendorid_list.h> +#include <asm/vendor_extensions.h> +#include <asm/vendor_extensions/mips.h> + +static inline bool errata_probe_pause(void) +{ + if (!IS_ENABLED(CONFIG_ERRATA_MIPS_P8700_PAUSE_OPCODE)) + return false; + + if (!riscv_isa_vendor_extension_available(MIPS_VENDOR_ID, XMIPSEXECTL)) + return false; + + return true; +} + +static u32 mips_errata_probe(void) +{ + u32 cpu_req_errata = 0; + + if (errata_probe_pause()) + cpu_req_errata |= BIT(ERRATA_MIPS_P8700_PAUSE_OPCODE); + + return cpu_req_errata; +} + +void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, + unsigned long archid, unsigned long impid, + unsigned int stage) +{ + struct alt_entry *alt; + u32 cpu_req_errata = mips_errata_probe(); + u32 tmp; + + BUILD_BUG_ON(ERRATA_MIPS_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE); + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return; + + for (alt = begin; alt < end; alt++) { + if (alt->vendor_id != MIPS_VENDOR_ID) + continue; + + if (alt->patch_id >= ERRATA_MIPS_NUMBER) { + WARN(1, "MIPS errata id:%d not in kernel errata list\n", + alt->patch_id); + continue; + } + + tmp = (1U << alt->patch_id); + if (cpu_req_errata & tmp) { + mutex_lock(&text_mutex); + patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), + alt->alt_len); + mutex_unlock(&text_mutex); + } + } +} diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index 38aac2c47845..d0c61f86cba3 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -75,26 +75,12 @@ static u32 __init_or_module sifive_errata_probe(unsigned long archid, return cpu_req_errata; } -static void __init_or_module warn_miss_errata(u32 miss_errata) -{ - int i; - - pr_warn("----------------------------------------------------------------\n"); - pr_warn("WARNING: Missing the following errata may cause potential issues\n"); - for (i = 0; i < ERRATA_SIFIVE_NUMBER; i++) - if (miss_errata & 0x1 << i) - pr_warn("\tSiFive Errata[%d]:%s\n", i, errata_list[i].name); - pr_warn("Please enable the corresponding Kconfig to apply them\n"); - pr_warn("----------------------------------------------------------------\n"); -} - void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned long archid, unsigned long impid, unsigned int stage) { struct alt_entry *alt; u32 cpu_req_errata; - u32 cpu_apply_errata = 0; u32 tmp; BUILD_BUG_ON(ERRATA_SIFIVE_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE); @@ -118,10 +104,6 @@ void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt), alt->alt_len); mutex_unlock(&text_mutex); - cpu_apply_errata |= tmp; } } - if (stage != RISCV_ALTERNATIVES_MODULE && - cpu_apply_errata != cpu_req_errata) - warn_miss_errata(cpu_req_errata - cpu_apply_errata); } diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index 0b942183f708..6f1c683f0ec0 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -153,7 +153,7 @@ static bool errata_probe_ghostwrite(unsigned int stage, * target-c9xx cores report arch_id and impid as 0 * * While ghostwrite may not affect all c9xx cores that implement - * xtheadvector, there is no futher granularity than c9xx. Assume + * xtheadvector, there is no further granularity than c9xx. Assume * vulnerable for this entire class of processors when xtheadvector is * enabled. */ diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index bd5fc9403295..7721b63642f4 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -14,5 +14,6 @@ generic-y += ticket_spinlock.h generic-y += qrwlock.h generic-y += qrwlock_types.h generic-y += qspinlock.h +generic-y += ring_buffer.h generic-y += user.h generic-y += vmlinux.lds.h diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 6e13695120bc..26ab37c171bc 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -61,10 +61,6 @@ static inline void arch_fix_phys_package_id(int num, u32 slot) { } void acpi_init_rintc_map(void); struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu); -static inline u32 get_acpi_id_for_cpu(int cpu) -{ - return acpi_cpu_get_madt_rintc(cpu)->uid; -} int acpi_get_riscv_isa(struct acpi_table_header *table, unsigned int cpu, const char **isa); diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 721ec275ce57..9619bd5c8eba 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -4,7 +4,7 @@ #ifdef CONFIG_RISCV_ALTERNATIVE -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ .macro ALT_ENTRY oldptr newptr vendor_id patch_id new_len .4byte \oldptr - . @@ -53,7 +53,7 @@ #define __ALTERNATIVE_CFG(...) ALTERNATIVE_CFG __VA_ARGS__ #define __ALTERNATIVE_CFG_2(...) ALTERNATIVE_CFG_2 __VA_ARGS__ -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ #include <asm/asm.h> #include <linux/stringify.h> @@ -98,7 +98,7 @@ __ALTERNATIVE_CFG(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1) \ ALT_NEW_CONTENT(vendor_id_2, patch_id_2, enable_2, new_c_2) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #define _ALTERNATIVE_CFG(old_c, new_c, vendor_id, patch_id, CONFIG_k) \ __ALTERNATIVE_CFG(old_c, new_c, vendor_id, patch_id, IS_ENABLED(CONFIG_k)) @@ -109,30 +109,25 @@ new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2)) #else /* CONFIG_RISCV_ALTERNATIVE */ -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ .macro ALTERNATIVE_CFG old_c \old_c .endm -#define _ALTERNATIVE_CFG(old_c, ...) \ - ALTERNATIVE_CFG old_c +#define __ALTERNATIVE_CFG(old_c, ...) ALTERNATIVE_CFG old_c +#define __ALTERNATIVE_CFG_2(old_c, ...) ALTERNATIVE_CFG old_c -#define _ALTERNATIVE_CFG_2(old_c, ...) \ - ALTERNATIVE_CFG old_c +#else /* !__ASSEMBLER__ */ -#else /* !__ASSEMBLY__ */ +#define __ALTERNATIVE_CFG(old_c, ...) old_c "\n" +#define __ALTERNATIVE_CFG_2(old_c, ...) old_c "\n" -#define __ALTERNATIVE_CFG(old_c) \ - old_c "\n" +#endif /* __ASSEMBLER__ */ -#define _ALTERNATIVE_CFG(old_c, ...) \ - __ALTERNATIVE_CFG(old_c) +#define _ALTERNATIVE_CFG(old_c, ...) __ALTERNATIVE_CFG(old_c) +#define _ALTERNATIVE_CFG_2(old_c, ...) __ALTERNATIVE_CFG_2(old_c) -#define _ALTERNATIVE_CFG_2(old_c, ...) \ - __ALTERNATIVE_CFG(old_c) - -#endif /* __ASSEMBLY__ */ #endif /* CONFIG_RISCV_ALTERNATIVE */ /* diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 3c2b59b25017..8407d1d535b8 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -8,7 +8,7 @@ #include <asm/alternative-macros.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_RISCV_ALTERNATIVE @@ -48,6 +48,9 @@ struct alt_entry { void andes_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned long archid, unsigned long impid, unsigned int stage); +void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, + unsigned long archid, unsigned long impid, + unsigned int stage); void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned long archid, unsigned long impid, unsigned int stage); diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index 613769b9cdc9..f3c0831beefc 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -19,10 +19,10 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { -#ifdef CONFIG_RISCV_ISA_ZBB - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!(IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))) + return __sw_hweight32(w); asm (".option push\n" ".option arch,+zbb\n" @@ -31,10 +31,6 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) : "=r" (w) : "r" (w) :); return w; - -legacy: -#endif - return __sw_hweight32(w); } static inline unsigned int __arch_hweight16(unsigned int w) @@ -50,10 +46,10 @@ static inline unsigned int __arch_hweight8(unsigned int w) #if BITS_PER_LONG == 64 static __always_inline unsigned long __arch_hweight64(__u64 w) { -# ifdef CONFIG_RISCV_ISA_ZBB - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!(IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))) + return __sw_hweight64(w); asm (".option push\n" ".option arch,+zbb\n" @@ -62,10 +58,6 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) : "=r" (w) : "r" (w) :); return w; - -legacy: -# endif - return __sw_hweight64(w); } #else /* BITS_PER_LONG == 64 */ static inline unsigned long __arch_hweight64(__u64 w) diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h index 0c8bfd54fc4e..37d425d7a762 100644 --- a/arch/riscv/include/asm/asm-extable.h +++ b/arch/riscv/include/asm/asm-extable.h @@ -10,7 +10,7 @@ #ifdef CONFIG_MMU -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ .pushsection __ex_table, "a"; \ @@ -25,7 +25,7 @@ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0) .endm -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ #include <linux/bits.h> #include <linux/stringify.h> @@ -77,7 +77,7 @@ EX_DATA_REG(ADDR, addr) \ ")") -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #else /* CONFIG_MMU */ #define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index cd627ec289f1..5b90ba5314ee 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -12,7 +12,7 @@ long long __ashlti3(long long a, int b); #ifdef CONFIG_RISCV_ISA_V #ifdef CONFIG_MMU -asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n); +asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n, bool enable_sum); #endif /* CONFIG_MMU */ void xor_regs_2_(unsigned long bytes, unsigned long *__restrict p1, @@ -40,6 +40,7 @@ asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs); #define DECLARE_DO_ERROR_INFO(name) asmlinkage void name(struct pt_regs *regs) DECLARE_DO_ERROR_INFO(do_trap_unknown); +DECLARE_DO_ERROR_INFO(do_trap_hardware_error); DECLARE_DO_ERROR_INFO(do_trap_insn_misaligned); DECLARE_DO_ERROR_INFO(do_trap_insn_fault); DECLARE_DO_ERROR_INFO(do_trap_insn_illegal); @@ -51,7 +52,10 @@ DECLARE_DO_ERROR_INFO(do_trap_ecall_u); DECLARE_DO_ERROR_INFO(do_trap_ecall_s); DECLARE_DO_ERROR_INFO(do_trap_ecall_m); DECLARE_DO_ERROR_INFO(do_trap_break); +DECLARE_DO_ERROR_INFO(do_trap_software_check); +asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs); +asmlinkage void ret_from_fork_user(struct pt_regs *regs); asmlinkage void handle_bad_stack(struct pt_regs *regs); asmlinkage void do_page_fault(struct pt_regs *regs); asmlinkage void do_irq(struct pt_regs *regs); diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 776354895b81..e9e8ba83e632 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -6,12 +6,18 @@ #ifndef _ASM_RISCV_ASM_H #define _ASM_RISCV_ASM_H -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define __ASM_STR(x) x #else #define __ASM_STR(x) #x #endif +#ifdef CONFIG_AS_HAS_INSN +#define ASM_INSN_I(__x) ".insn " __x +#else +#define ASM_INSN_I(__x) ".4byte " __x +#endif + #if __riscv_xlen == 64 #define __REG_SEL(a, b) __ASM_STR(a) #elif __riscv_xlen == 32 @@ -27,9 +33,10 @@ #define REG_ASM __REG_SEL(.dword, .word) #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) +#define SRLI __REG_SEL(srliw, srli) #if __SIZEOF_POINTER__ == 8 -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define RISCV_PTR .dword #define RISCV_SZPTR 8 #define RISCV_LGPTR 3 @@ -39,7 +46,7 @@ #define RISCV_LGPTR "3" #endif #elif __SIZEOF_POINTER__ == 4 -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define RISCV_PTR .word #define RISCV_SZPTR 4 #define RISCV_LGPTR 2 @@ -68,7 +75,7 @@ #error "Unexpected __SIZEOF_SHORT__" #endif -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #include <asm/asm-offsets.h> /* Common assembly source macros */ @@ -83,15 +90,9 @@ .endm #ifdef CONFIG_SMP -#ifdef CONFIG_32BIT -#define PER_CPU_OFFSET_SHIFT 2 -#else -#define PER_CPU_OFFSET_SHIFT 3 -#endif - .macro asm_per_cpu dst sym tmp - REG_L \tmp, TASK_TI_CPU_NUM(tp) - slli \tmp, \tmp, PER_CPU_OFFSET_SHIFT + lw \tmp, TASK_TI_CPU_NUM(tp) + slli \tmp, \tmp, RISCV_LGPTR la \dst, __per_cpu_offset add \dst, \dst, \tmp REG_L \tmp, 0(\dst) @@ -193,6 +194,6 @@ #define ASM_NOKPROBE(name) #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_ASM_H */ diff --git a/arch/riscv/include/asm/assembler.h b/arch/riscv/include/asm/assembler.h index 44b1457d3e95..a8df1999118b 100644 --- a/arch/riscv/include/asm/assembler.h +++ b/arch/riscv/include/asm/assembler.h @@ -5,7 +5,7 @@ * Author: Jee Heng Sia <jeeheng.sia@starfivetech.com> */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #error "Only include this from assembly code" #endif @@ -80,3 +80,47 @@ .endm #endif /* __ASM_ASSEMBLER_H */ + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +.macro vdso_lpad, label = 0 +lpad \label +.endm +#else +.macro vdso_lpad, label = 0 +.endm +#endif + +/* + * This macro emits a program property note section identifying + * architecture features which require special handling, mainly for + * use in assembly files included in the VDSO. + */ +#define NT_GNU_PROPERTY_TYPE_0 5 +#define GNU_PROPERTY_RISCV_FEATURE_1_AND 0xc0000000 + +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP BIT(0) +#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS BIT(1) + +#if defined(VDSO_CFI) && (__riscv_xlen == 64) +#define GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT \ + (GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP | GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS) +#endif + +#ifdef GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT +.macro emit_riscv_feature_1_and, feat = GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT + .pushsection .note.gnu.property, "a" + .p2align 3 + .word 4 + .word 16 + .word NT_GNU_PROPERTY_TYPE_0 + .asciz "GNU" + .word GNU_PROPERTY_RISCV_FEATURE_1_AND + .word 4 + .word \feat + .word 0 + .popsection +.endm +#else +.macro emit_riscv_feature_1_and, feat = 0 +.endm +#endif diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 5b96c2f61adb..616b8b332ac5 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -46,7 +46,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) #endif /* - * First, the atomic ops that have no ordering constraints and therefor don't + * First, the atomic ops that have no ordering constraints and therefore don't * have the AQ or RL bits set. These don't return anything, so there's only * one version to worry about. */ @@ -81,7 +81,7 @@ ATOMIC_OPS(xor, xor, i) /* * Atomic ops that have ordered, relaxed, acquire, and release variants. - * There's two flavors of these: the arithmatic ops have both fetch and return + * There's two flavors of these: the arithmetic ops have both fetch and return * versions, while the logical ops only have fetch versions. */ #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ @@ -203,7 +203,7 @@ ATOMIC_OPS(xor, xor, i) " add %[rc], %[p], %[a]\n" \ " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ " bnez %[rc], 0b\n" \ - " fence rw, rw\n" \ + RISCV_FULL_BARRIER \ "1:\n" \ : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ : [a]"r" (_a), [u]"r" (_u) \ @@ -242,7 +242,7 @@ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, " addi %[rc], %[p], 1\n" \ " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ " bnez %[rc], 0b\n" \ - " fence rw, rw\n" \ + RISCV_FULL_BARRIER \ "1:\n" \ : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ : \ @@ -268,7 +268,7 @@ static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v) " addi %[rc], %[p], -1\n" \ " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ " bnez %[rc], 0b\n" \ - " fence rw, rw\n" \ + RISCV_FULL_BARRIER \ "1:\n" \ : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ : \ @@ -294,7 +294,7 @@ static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v) " bltz %[rc], 1f\n" \ " sc." sfx ".rl %[rc], %[rc], %[c]\n" \ " bnez %[rc], 0b\n" \ - " fence rw, rw\n" \ + RISCV_FULL_BARRIER \ "1:\n" \ : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \ : \ diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index e1d9bf1deca6..700ba3f922cb 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -10,15 +10,10 @@ #ifndef _ASM_RISCV_BARRIER_H #define _ASM_RISCV_BARRIER_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/cmpxchg.h> #include <asm/fence.h> -#define nop() __asm__ __volatile__ ("nop") -#define __nops(n) ".rept " #n "\nnop\n.endr\n" -#define nops(n) __asm__ __volatile__ (__nops(n)) - - /* These barriers need to enforce ordering on both devices or memory. */ #define __mb() RISCV_FENCE(iorw, iorw) #define __rmb() RISCV_FENCE(ir, ir) @@ -87,6 +82,6 @@ do { \ #include <asm-generic/barrier.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_BARRIER_H */ diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index c6bd3d8354a9..3c1a15be54d8 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -11,11 +11,10 @@ #endif /* _LINUX_BITOPS_H */ #include <linux/compiler.h> -#include <linux/irqflags.h> #include <asm/barrier.h> #include <asm/bitsperlong.h> -#if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) +#if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) #include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/ffs.h> @@ -45,11 +44,10 @@ #error "Unexpected BITS_PER_LONG" #endif -static __always_inline unsigned long variable__ffs(unsigned long word) +static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return generic___ffs(word); asm volatile (".option push\n" ".option arch,+zbb\n" @@ -58,9 +56,6 @@ static __always_inline unsigned long variable__ffs(unsigned long word) : "=r" (word) : "r" (word) :); return word; - -legacy: - return generic___ffs(word); } /** @@ -74,11 +69,10 @@ legacy: (unsigned long)__builtin_ctzl(word) : \ variable__ffs(word)) -static __always_inline unsigned long variable__fls(unsigned long word) +static __always_inline __attribute_const__ unsigned long variable__fls(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return generic___fls(word); asm volatile (".option push\n" ".option arch,+zbb\n" @@ -87,9 +81,6 @@ static __always_inline unsigned long variable__fls(unsigned long word) : "=r" (word) : "r" (word) :); return BITS_PER_LONG - 1 - word; - -legacy: - return generic___fls(word); } /** @@ -103,11 +94,10 @@ legacy: (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \ variable__fls(word)) -static __always_inline int variable_ffs(int x) +static __always_inline __attribute_const__ int variable_ffs(int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return generic_ffs(x); if (!x) return 0; @@ -119,9 +109,6 @@ static __always_inline int variable_ffs(int x) : "=r" (x) : "r" (x) :); return x + 1; - -legacy: - return generic_ffs(x); } /** @@ -137,9 +124,8 @@ legacy: static __always_inline int variable_fls(unsigned int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return generic_fls(x); if (!x) return 0; @@ -151,9 +137,6 @@ static __always_inline int variable_fls(unsigned int x) : "=r" (x) : "r" (x) :); return 32 - x; - -legacy: - return generic_fls(x); } /** @@ -175,7 +158,7 @@ legacy: variable_fls(x_); \ }) -#endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */ +#endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */ #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/fls64.h> @@ -226,7 +209,7 @@ legacy: * @nr: Bit to set * @addr: Address to count from * - * This operation may be reordered on other architectures than x86. + * This is an atomic fully-ordered operation (implied full memory barrier). */ static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr) { @@ -238,7 +221,7 @@ static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long * @nr: Bit to clear * @addr: Address to count from * - * This operation can be reordered on other architectures other than x86. + * This is an atomic fully-ordered operation (implied full memory barrier). */ static __always_inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr) { diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index 1aaea81fb141..6f581b84d8fc 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -31,52 +31,59 @@ typedef u32 bug_insn_t; #ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS #define __BUG_ENTRY_ADDR RISCV_INT " 1b - ." -#define __BUG_ENTRY_FILE RISCV_INT " %0 - ." +#define __BUG_ENTRY_FILE(file) RISCV_INT " " file " - ." #else #define __BUG_ENTRY_ADDR RISCV_PTR " 1b" -#define __BUG_ENTRY_FILE RISCV_PTR " %0" +#define __BUG_ENTRY_FILE(file) RISCV_PTR " " file #endif #ifdef CONFIG_DEBUG_BUGVERBOSE -#define __BUG_ENTRY \ +#define __BUG_ENTRY(file, line, flags) \ __BUG_ENTRY_ADDR "\n\t" \ - __BUG_ENTRY_FILE "\n\t" \ - RISCV_SHORT " %1\n\t" \ - RISCV_SHORT " %2" + __BUG_ENTRY_FILE(file) "\n\t" \ + RISCV_SHORT " " line "\n\t" \ + RISCV_SHORT " " flags #else -#define __BUG_ENTRY \ - __BUG_ENTRY_ADDR "\n\t" \ - RISCV_SHORT " %2" +#define __BUG_ENTRY(file, line, flags) \ + __BUG_ENTRY_ADDR "\n\t" \ + RISCV_SHORT " " flags #endif #ifdef CONFIG_GENERIC_BUG -#define __BUG_FLAGS(flags) \ -do { \ - __asm__ __volatile__ ( \ + +#define ARCH_WARN_ASM(file, line, flags, size) \ "1:\n\t" \ "ebreak\n" \ ".pushsection __bug_table,\"aw\"\n\t" \ "2:\n\t" \ - __BUG_ENTRY "\n\t" \ - ".org 2b + %3\n\t" \ + __BUG_ENTRY(file, line, flags) "\n\t" \ + ".org 2b + " size "\n\t" \ ".popsection" \ + +#define __BUG_FLAGS(cond_str, flags) \ +do { \ + __asm__ __volatile__ ( \ + ARCH_WARN_ASM("%0", "%1", "%2", "%3") \ : \ - : "i" (__FILE__), "i" (__LINE__), \ + : "i" (WARN_CONDITION_STR(cond_str) __FILE__), "i" (__LINE__), \ "i" (flags), \ "i" (sizeof(struct bug_entry))); \ } while (0) + #else /* CONFIG_GENERIC_BUG */ -#define __BUG_FLAGS(flags) do { \ +#define __BUG_FLAGS(cond_str, flags) do { \ __asm__ __volatile__ ("ebreak\n"); \ } while (0) #endif /* CONFIG_GENERIC_BUG */ #define BUG() do { \ - __BUG_FLAGS(0); \ + __BUG_FLAGS("", 0); \ unreachable(); \ } while (0) -#define __WARN_FLAGS(flags) __BUG_FLAGS(BUGFLAG_WARNING|(flags)) +#define __WARN_FLAGS(cond_str, flags) __BUG_FLAGS(cond_str, BUGFLAG_WARNING|(flags)) + +#define ARCH_WARN_REACHABLE #define HAVE_ARCH_BUG diff --git a/arch/riscv/include/asm/cache.h b/arch/riscv/include/asm/cache.h index 570e9d8acad1..eb42b739d132 100644 --- a/arch/riscv/include/asm/cache.h +++ b/arch/riscv/include/asm/cache.h @@ -24,7 +24,7 @@ #define ARCH_SLAB_MINALIGN 16 #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern int dma_cache_alignment; #ifdef CONFIG_RISCV_DMA_NONCOHERENT @@ -35,6 +35,6 @@ static inline int dma_get_cache_alignment(void) } #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_CACHE_H */ diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 8de73f91bfa3..0092513c3376 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -23,8 +23,8 @@ static inline void local_flush_icache_range(unsigned long start, static inline void flush_dcache_folio(struct folio *folio) { - if (test_bit(PG_dcache_clean, &folio->flags)) - clear_bit(PG_dcache_clean, &folio->flags); + if (test_bit(PG_dcache_clean, &folio->flags.f)) + clear_bit(PG_dcache_clean, &folio->flags.f); } #define flush_dcache_folio flush_dcache_folio #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 @@ -34,11 +34,6 @@ static inline void flush_dcache_page(struct page *page) flush_dcache_folio(page_folio(page)); } -/* - * RISC-V doesn't have an instruction to flush parts of the instruction cache, - * so instead we just flush the whole thing. - */ -#define flush_icache_range(start, end) flush_icache_all() #define flush_icache_user_page(vma, pg, addr, len) \ do { \ if (vma->vm_flags & VM_EXEC) \ @@ -78,8 +73,19 @@ void flush_icache_mm(struct mm_struct *mm, bool local); #endif /* CONFIG_SMP */ +/* + * RISC-V doesn't have an instruction to flush parts of the instruction cache, + * so instead we just flush the whole thing. + */ +#define flush_icache_range flush_icache_range +static inline void flush_icache_range(unsigned long start, unsigned long end) +{ + flush_icache_all(); +} + extern unsigned int riscv_cbom_block_size; extern unsigned int riscv_cboz_block_size; +extern unsigned int riscv_cbop_block_size; void riscv_init_cbo_blocksizes(void); #ifdef CONFIG_RISCV_DMA_NONCOHERENT diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h index fb9696d7a3f2..710aa8192edd 100644 --- a/arch/riscv/include/asm/cfi.h +++ b/arch/riscv/include/asm/cfi.h @@ -11,30 +11,14 @@ struct pt_regs; -#ifdef CONFIG_CFI_CLANG +#ifdef CONFIG_CFI enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); #define __bpfcall -static inline int cfi_get_offset(void) -{ - return 4; -} - -#define cfi_get_offset cfi_get_offset -extern u32 cfi_bpf_hash; -extern u32 cfi_bpf_subprog_hash; -extern u32 cfi_get_func_hash(void *func); #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { return BUG_TRAP_TYPE_NONE; } - -#define cfi_bpf_hash 0U -#define cfi_bpf_subprog_hash 0U -static inline u32 cfi_get_func_hash(void *func) -{ - return 0; -} -#endif /* CONFIG_CFI_CLANG */ +#endif /* CONFIG_CFI */ #endif /* _ASM_RISCV_CFI_H */ diff --git a/arch/riscv/include/asm/checksum.h b/arch/riscv/include/asm/checksum.h index 88e6f1499e88..945cce34be92 100644 --- a/arch/riscv/include/asm/checksum.h +++ b/arch/riscv/include/asm/checksum.h @@ -50,16 +50,10 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) * worth checking if supported without Alternatives. */ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - if (IS_ENABLED(CONFIG_32BIT)) { asm(".option push \n\ .option arch,+zbb \n\ @@ -82,7 +76,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) } return (__force __sum16)(csum >> 16); } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 4cadc56220fe..8712cf9c69dc 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -13,6 +13,8 @@ #include <asm/hwcap.h> #include <asm/insn-def.h> #include <asm/cpufeature-macros.h> +#include <asm/processor.h> +#include <asm/errata_list.h> #define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \ swap_append, r, p, n) \ @@ -37,6 +39,7 @@ \ __asm__ __volatile__ ( \ prepend \ + PREFETCHW_ASM(%5) \ "0: lr.w %0, %2\n" \ " and %1, %0, %z4\n" \ " or %1, %1, %z3\n" \ @@ -44,7 +47,7 @@ " bnez %1, 0b\n" \ sc_append \ : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \ - : "rJ" (__newx), "rJ" (~__mask) \ + : "rJ" (__newx), "rJ" (~__mask), "rJ" (__ptr32b) \ : "memory"); \ \ r = (__typeof__(*(p)))((__retx & __mask) >> __s); \ @@ -131,6 +134,7 @@ ({ \ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \ IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ r = o; \ @@ -178,6 +182,7 @@ r, p, co, o, n) \ ({ \ if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \ + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && \ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \ r = o; \ \ @@ -231,7 +236,7 @@ __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx, \ sc_prepend, sc_append, \ cas_prepend, cas_append, \ - __ret, __ptr, (long), __old, __new); \ + __ret, __ptr, (long)(int)(long), __old, __new); \ break; \ case 8: \ __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx, \ @@ -313,7 +318,7 @@ arch_cmpxchg_release((ptr), (o), (n)); \ }) -#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS) +#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS) && defined(CONFIG_TOOLCHAIN_HAS_ZACAS) #define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS) @@ -349,7 +354,7 @@ union __u128_halves { #define arch_cmpxchg128_local(ptr, o, n) \ __arch_cmpxchg128((ptr), (o), (n), "") -#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS */ +#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS && CONFIG_TOOLCHAIN_HAS_ZACAS */ #ifdef CONFIG_RISCV_ISA_ZAWRS /* @@ -365,16 +370,49 @@ static __always_inline void __cmpwait(volatile void *ptr, { unsigned long tmp; - asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", - 0, RISCV_ISA_EXT_ZAWRS, 1) - : : : : no_zawrs); + u32 *__ptr32b; + ulong __s, __val, __mask; + + if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS)) { + ALT_RISCV_PAUSE(); + return; + } switch (size) { case 1: - fallthrough; + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; case 2: - /* RISC-V doesn't have lr instructions on byte and half-word. */ - goto no_zawrs; + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xffff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; case 4: asm volatile( " lr.w %0, %1\n" @@ -400,11 +438,6 @@ static __always_inline void __cmpwait(volatile void *ptr, default: BUILD_BUG(); } - - return; - -no_zawrs: - asm volatile(RISCV_PAUSE : : : "memory"); } #define __cmpwait_relaxed(ptr, val) \ diff --git a/arch/riscv/include/asm/compat.h b/arch/riscv/include/asm/compat.h index 6081327e55f5..28e115eed218 100644 --- a/arch/riscv/include/asm/compat.h +++ b/arch/riscv/include/asm/compat.h @@ -2,7 +2,7 @@ #ifndef __ASM_COMPAT_H #define __ASM_COMPAT_H -#define COMPAT_UTS_MACHINE "riscv\0\0" +#define COMPAT_UTS_MACHINE "riscv32\0\0" /* * Architecture specific compatibility types diff --git a/arch/riscv/include/asm/cpu_ops_sbi.h b/arch/riscv/include/asm/cpu_ops_sbi.h index d6e4665b3195..776fa55fbaa4 100644 --- a/arch/riscv/include/asm/cpu_ops_sbi.h +++ b/arch/riscv/include/asm/cpu_ops_sbi.h @@ -5,7 +5,7 @@ #ifndef __ASM_CPU_OPS_SBI_H #define __ASM_CPU_OPS_SBI_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/init.h> #include <linux/sched.h> #include <linux/threads.h> diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 569140d6e639..739fcc84bf7b 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -31,6 +31,8 @@ struct riscv_isainfo { DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); +extern const struct seq_operations cpuinfo_op; + /* Per-cpu ISA extensions. */ extern struct riscv_isainfo hart_isa[NR_CPUS]; @@ -56,6 +58,9 @@ void __init riscv_user_isa_enable(void); #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ ARRAY_SIZE(_bundled_exts), NULL) +#define __RISCV_ISA_EXT_BUNDLE_VALIDATE(_name, _bundled_exts, _validate) \ + _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ + ARRAY_SIZE(_bundled_exts), _validate) /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ @@ -63,12 +68,12 @@ void __init riscv_user_isa_enable(void); #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \ _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate) -bool check_unaligned_access_emulated_all_cpus(void); +bool __init check_unaligned_access_emulated_all_cpus(void); +void unaligned_access_init(void); +int cpu_online_unaligned_access_init(unsigned int cpu); #if defined(CONFIG_RISCV_SCALAR_MISALIGNED) -void check_unaligned_access_emulated(struct work_struct *work __always_unused); void unaligned_emulation_finish(void); bool unaligned_ctl_available(void); -DECLARE_PER_CPU(long, misaligned_access_speed); #else static inline bool unaligned_ctl_available(void) { @@ -76,7 +81,17 @@ static inline bool unaligned_ctl_available(void) } #endif -bool check_vector_unaligned_access_emulated_all_cpus(void); +#if defined(CONFIG_RISCV_MISALIGNED) +DECLARE_PER_CPU(long, misaligned_access_speed); +bool misaligned_traps_can_delegate(void); +#else +static inline bool misaligned_traps_can_delegate(void) +{ + return false; +} +#endif + +bool __init check_vector_unaligned_access_emulated_all_cpus(void); #if defined(CONFIG_RISCV_VECTOR_MISALIGNED) void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused); DECLARE_PER_CPU(long, vector_misaligned_access); @@ -137,4 +152,16 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); } +static inline bool cpu_supports_shadow_stack(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS)); +} + +static inline bool cpu_supports_indirect_br_lp_instr(void) +{ + return (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFILP)); +} + #endif diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 6fed42e37705..31b8988f4488 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -18,6 +18,15 @@ #define SR_MPP _AC(0x00001800, UL) /* Previously Machine */ #define SR_SUM _AC(0x00040000, UL) /* Supervisor User Memory Access */ +/* zicfilp landing pad status bit */ +#define SR_SPELP _AC(0x00800000, UL) +#define SR_MPELP _AC(0x020000000000, UL) +#ifdef CONFIG_RISCV_M_MODE +#define SR_ELP SR_MPELP +#else +#define SR_ELP SR_SPELP +#endif + #define SR_FS _AC(0x00006000, UL) /* Floating-point Status */ #define SR_FS_OFF _AC(0x00000000, UL) #define SR_FS_INITIAL _AC(0x00002000, UL) @@ -212,6 +221,8 @@ #define ENVCFG_PMM_PMLEN_16 (_AC(0x3, ULL) << 32) #define ENVCFG_CBZE (_AC(1, UL) << 7) #define ENVCFG_CBCFE (_AC(1, UL) << 6) +#define ENVCFG_LPE (_AC(1, UL) << 2) +#define ENVCFG_SSE (_AC(1, UL) << 3) #define ENVCFG_CBIE_SHIFT 4 #define ENVCFG_CBIE (_AC(0x3, UL) << ENVCFG_CBIE_SHIFT) #define ENVCFG_CBIE_ILL _AC(0x0, UL) @@ -321,6 +332,9 @@ #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D +/* zicfiss user mode csr. CSR_SSP holds current shadow stack pointer */ +#define CSR_SSP 0x011 + /* xtheadvector symbolic CSR names */ #define CSR_VXSAT 0x9 #define CSR_VXRM 0xa @@ -444,6 +458,23 @@ #define CSR_VTYPE 0xc21 #define CSR_VLENB 0xc22 +#define VTYPE_VLMUL _AC(7, UL) +#define VTYPE_VLMUL_FRAC _AC(4, UL) +#define VTYPE_VSEW_SHIFT 3 +#define VTYPE_VSEW (_AC(7, UL) << VTYPE_VSEW_SHIFT) +#define VTYPE_VTA_SHIFT 6 +#define VTYPE_VTA (_AC(1, UL) << VTYPE_VTA_SHIFT) +#define VTYPE_VMA_SHIFT 7 +#define VTYPE_VMA (_AC(1, UL) << VTYPE_VMA_SHIFT) +#define VTYPE_VILL_SHIFT (__riscv_xlen - 1) +#define VTYPE_VILL (_AC(1, UL) << VTYPE_VILL_SHIFT) + +#define VTYPE_VLMUL_THEAD _AC(3, UL) +#define VTYPE_VSEW_THEAD_SHIFT 2 +#define VTYPE_VSEW_THEAD (_AC(7, UL) << VTYPE_VSEW_THEAD_SHIFT) +#define VTYPE_VEDIV_THEAD_SHIFT 5 +#define VTYPE_VEDIV_THEAD (_AC(3, UL) << VTYPE_VEDIV_THEAD_SHIFT) + /* Scalar Crypto Extension - Entropy */ #define CSR_SEED 0x015 #define SEED_OPST_MASK _AC(0xC0000000, UL) @@ -513,7 +544,7 @@ #define IE_TIE (_AC(0x1, UL) << RV_IRQ_TIMER) #define IE_EIE (_AC(0x1, UL) << RV_IRQ_EXT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define csr_swap(csr, val) \ ({ \ @@ -575,6 +606,6 @@ : "memory"); \ }) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_CSR_H */ diff --git a/arch/riscv/include/asm/current.h b/arch/riscv/include/asm/current.h index 21774d868c65..ba5aa72aff63 100644 --- a/arch/riscv/include/asm/current.h +++ b/arch/riscv/include/asm/current.h @@ -13,7 +13,7 @@ #include <linux/bug.h> #include <linux/compiler.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct task_struct; @@ -35,6 +35,6 @@ static __always_inline struct task_struct *get_current(void) register unsigned long current_stack_pointer __asm__("sp"); -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_CURRENT_H */ diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index c7aea7886d22..aa4961b0e208 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -59,8 +59,8 @@ extern bool compat_elf_check_arch(Elf32_Ehdr *hdr); #endif /* - * Provides information on the availiable set of ISA extensions to userspace, - * via a bitmap that coorespends to each single-letter ISA extension. This is + * Provides information on the available set of ISA extensions to userspace, + * via a bitmap that corresponds to each single-letter ISA extension. This is * essentially defunct, but will remain for compatibility with userspace. */ #define ELF_HWCAP riscv_get_elf_hwcap() diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h index b28ccc6cdeea..34ed149af5d1 100644 --- a/arch/riscv/include/asm/entry-common.h +++ b/arch/riscv/include/asm/entry-common.h @@ -40,4 +40,6 @@ static inline int handle_misaligned_store(struct pt_regs *regs) } #endif +bool handle_user_cfi_violation(struct pt_regs *regs); + #endif /* _ASM_RISCV_ENTRY_COMMON_H */ diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 6e426ed7919a..6694b5ccdcf8 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -5,31 +5,14 @@ #ifndef ASM_ERRATA_LIST_H #define ASM_ERRATA_LIST_H -#include <asm/alternative.h> #include <asm/csr.h> #include <asm/insn-def.h> #include <asm/hwcap.h> #include <asm/vendorid_list.h> +#include <asm/errata_list_vendors.h> +#include <asm/vendor_extensions/mips.h> -#ifdef CONFIG_ERRATA_ANDES -#define ERRATA_ANDES_NO_IOCP 0 -#define ERRATA_ANDES_NUMBER 1 -#endif - -#ifdef CONFIG_ERRATA_SIFIVE -#define ERRATA_SIFIVE_CIP_453 0 -#define ERRATA_SIFIVE_CIP_1200 1 -#define ERRATA_SIFIVE_NUMBER 2 -#endif - -#ifdef CONFIG_ERRATA_THEAD -#define ERRATA_THEAD_MAE 0 -#define ERRATA_THEAD_PMU 1 -#define ERRATA_THEAD_GHOSTWRITE 2 -#define ERRATA_THEAD_NUMBER 3 -#endif - -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define ALT_INSN_FAULT(x) \ ALTERNATIVE(__stringify(RISCV_PTR do_trap_insn_fault), \ @@ -42,7 +25,7 @@ ALTERNATIVE(__stringify(RISCV_PTR do_page_fault), \ __stringify(RISCV_PTR sifive_cip_453_page_fault_trp), \ SIFIVE_VENDOR_ID, ERRATA_SIFIVE_CIP_453, \ CONFIG_ERRATA_SIFIVE_CIP_453) -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ #define ALT_SFENCE_VMA_ASID(asid) \ asm(ALTERNATIVE("sfence.vma x0, %0", "sfence.vma", SIFIVE_VENDOR_ID, \ @@ -59,6 +42,17 @@ asm(ALTERNATIVE("sfence.vma %0, %1", "sfence.vma", SIFIVE_VENDOR_ID, \ ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \ : : "r" (addr), "r" (asid) : "memory") +#define ALT_RISCV_PAUSE() \ +asm(ALTERNATIVE( \ + RISCV_PAUSE, /* Original RISC‑V pause insn */ \ + MIPS_PAUSE, /* Replacement for MIPS P8700 */ \ + MIPS_VENDOR_ID, /* Vendor ID to match */ \ + ERRATA_MIPS_P8700_PAUSE_OPCODE, /* patch_id */ \ + CONFIG_ERRATA_MIPS_P8700_PAUSE_OPCODE) \ + : /* no outputs */ \ + : /* no inputs */ \ + : "memory") + /* * _val is marked as "will be overwritten", so need to set it to 0 * in the default case. @@ -123,6 +117,6 @@ asm volatile(ALTERNATIVE( \ #define THEAD_C9XX_RV_IRQ_PMU 17 #define THEAD_C9XX_CSR_SCOUNTEROF 0x5c5 -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/arch/riscv/include/asm/errata_list_vendors.h b/arch/riscv/include/asm/errata_list_vendors.h new file mode 100644 index 000000000000..ec7eba373437 --- /dev/null +++ b/arch/riscv/include/asm/errata_list_vendors.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef ASM_ERRATA_LIST_VENDORS_H +#define ASM_ERRATA_LIST_VENDORS_H + +#ifdef CONFIG_ERRATA_ANDES +#define ERRATA_ANDES_NO_IOCP 0 +#define ERRATA_ANDES_NUMBER 1 +#endif + +#ifdef CONFIG_ERRATA_SIFIVE +#define ERRATA_SIFIVE_CIP_453 0 +#define ERRATA_SIFIVE_CIP_1200 1 +#define ERRATA_SIFIVE_NUMBER 2 +#endif + +#ifdef CONFIG_ERRATA_THEAD +#define ERRATA_THEAD_MAE 0 +#define ERRATA_THEAD_PMU 1 +#define ERRATA_THEAD_GHOSTWRITE 2 +#define ERRATA_THEAD_NUMBER 3 +#endif + +#ifdef CONFIG_ERRATA_MIPS +#define ERRATA_MIPS_P8700_PAUSE_OPCODE 0 +#define ERRATA_MIPS_NUMBER 1 +#endif + +#endif /* ASM_ERRATA_LIST_VENDORS_H */ diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index c4721ce44ca4..e5026cd8f022 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -13,17 +13,16 @@ #endif #define ARCH_SUPPORTS_FTRACE_OPS 1 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern void *return_address(unsigned int level); #define ftrace_return_address(n) return_address(n) void _mcount(void); -static inline unsigned long ftrace_call_adjust(unsigned long addr) -{ - return addr; -} +unsigned long ftrace_call_adjust(unsigned long addr); +unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip); +#define ftrace_get_symaddr(fentry_ip) arch_ftrace_get_symaddr(fentry_ip) /* * Let's do like x86/arm64 and ignore the compat syscalls. @@ -57,12 +56,21 @@ struct dyn_arch_ftrace { * 2) jalr: setting low-12 offset to ra, jump to ra, and set ra to * return address (original pc + 4) * + * The first 2 instructions for each tracable function is compiled to 2 nop + * instructions. Then, the kernel initializes the first instruction to auipc at + * boot time (<ftrace disable>). The second instruction is patched to jalr to + * start the trace. + * + *<Image>: + * 0: nop + * 4: nop + * *<ftrace enable>: - * 0: auipc t0/ra, 0x? - * 4: jalr t0/ra, ?(t0/ra) + * 0: auipc t0, 0x? + * 4: jalr t0, ?(t0) * *<ftrace disable>: - * 0: nop + * 0: auipc t0, 0x? * 4: nop * * Dynamic ftrace generates probes to call sites, so we must deal with @@ -75,11 +83,9 @@ struct dyn_arch_ftrace { #define AUIPC_OFFSET_MASK (0xfffff000) #define AUIPC_PAD (0x00001000) #define JALR_SHIFT 20 -#define JALR_RA (0x000080e7) -#define AUIPC_RA (0x00000097) #define JALR_T0 (0x000282e7) #define AUIPC_T0 (0x00000297) -#define NOP4 (0x00000013) +#define JALR_RANGE (JALR_SIGN_MASK - 1) #define to_jalr_t0(offset) \ (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_T0) @@ -92,33 +98,21 @@ struct dyn_arch_ftrace { #define make_call_t0(caller, callee, call) \ do { \ unsigned int offset = \ - (unsigned long) callee - (unsigned long) caller; \ + (unsigned long) (callee) - (unsigned long) (caller); \ call[0] = to_auipc_t0(offset); \ call[1] = to_jalr_t0(offset); \ } while (0) -#define to_jalr_ra(offset) \ - (((offset & JALR_OFFSET_MASK) << JALR_SHIFT) | JALR_RA) - -#define to_auipc_ra(offset) \ - ((offset & JALR_SIGN_MASK) ? \ - (((offset & AUIPC_OFFSET_MASK) + AUIPC_PAD) | AUIPC_RA) : \ - ((offset & AUIPC_OFFSET_MASK) | AUIPC_RA)) - -#define make_call_ra(caller, callee, call) \ -do { \ - unsigned int offset = \ - (unsigned long) callee - (unsigned long) caller; \ - call[0] = to_auipc_ra(offset); \ - call[1] = to_jalr_ra(offset); \ -} while (0) - /* - * Let auipc+jalr be the basic *mcount unit*, so we make it 8 bytes here. + * Only the jalr insn in the auipc+jalr is patched, so we make it 4 + * bytes here. */ -#define MCOUNT_INSN_SIZE 8 +#define MCOUNT_INSN_SIZE 4 +#define MCOUNT_AUIPC_SIZE 4 +#define MCOUNT_JALR_SIZE 4 +#define MCOUNT_NOP4_SIZE 4 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct dyn_ftrace; int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); #define ftrace_init_nop ftrace_init_nop @@ -136,6 +130,9 @@ struct __arch_ftrace_regs { unsigned long sp; unsigned long s0; unsigned long t1; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + unsigned long direct_tramp; +#endif union { unsigned long args[8]; struct { @@ -147,6 +144,13 @@ struct __arch_ftrace_regs { unsigned long a5; unsigned long a6; unsigned long a7; +#ifdef CONFIG_CC_IS_CLANG + unsigned long t2; + unsigned long t3; + unsigned long t4; + unsigned long t5; + unsigned long t6; +#endif }; }; }; @@ -207,7 +211,7 @@ ftrace_partial_regs(const struct ftrace_regs *fregs, struct pt_regs *regs) { struct __arch_ftrace_regs *afregs = arch_ftrace_regs(fregs); - memcpy(®s->a0, afregs->args, sizeof(afregs->args)); + memcpy(®s->a_regs, afregs->args, sizeof(afregs->args)); regs->epc = afregs->epc; regs->ra = afregs->ra; regs->sp = afregs->sp; @@ -222,13 +226,16 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs); #define ftrace_graph_func ftrace_graph_func +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr) { arch_ftrace_regs(fregs)->t1 = addr; } +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* CONFIG_DYNAMIC_FTRACE */ diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 72be100afa23..90c86b115e00 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -93,7 +93,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %[r]) \ _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %[r]) \ : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), [t] "=&r" (tmp) - : [ov] "Jr" (oldval), [nv] "Jr" (newval) + : [ov] "Jr" ((long)(int)oldval), [nv] "Jr" (newval) : "memory"); __disable_user_access(); diff --git a/arch/riscv/include/asm/gpr-num.h b/arch/riscv/include/asm/gpr-num.h index efeb5edf8a3a..b499cf832734 100644 --- a/arch/riscv/include/asm/gpr-num.h +++ b/arch/riscv/include/asm/gpr-num.h @@ -2,7 +2,7 @@ #ifndef __ASM_GPR_NUM_H #define __ASM_GPR_NUM_H -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 .equ .L__gpr_num_x\num, \num @@ -41,7 +41,7 @@ .equ .L__gpr_num_t5, 30 .equ .L__gpr_num_t6, 31 -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ #define __DEFINE_ASM_GPR_NUMS \ " .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ @@ -80,6 +80,6 @@ " .equ .L__gpr_num_t5, 30\n" \ " .equ .L__gpr_num_t6, 31\n" -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GPR_NUM_H */ diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h index faf3624d8057..0872d43fc0c0 100644 --- a/arch/riscv/include/asm/hugetlb.h +++ b/arch/riscv/include/asm/hugetlb.h @@ -7,7 +7,7 @@ static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &folio->flags); + clear_bit(PG_dcache_clean, &folio->flags.f); } #define arch_clear_hugetlb_flags arch_clear_hugetlb_flags @@ -28,7 +28,8 @@ void set_huge_pte_at(struct mm_struct *mm, #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep); + unsigned long addr, pte_t *ptep, + unsigned long sz); #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 869da082252a..7ef8e5f55c8d 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -100,6 +100,18 @@ #define RISCV_ISA_EXT_ZICCRSE 91 #define RISCV_ISA_EXT_SVADE 92 #define RISCV_ISA_EXT_SVADU 93 +#define RISCV_ISA_EXT_ZFBFMIN 94 +#define RISCV_ISA_EXT_ZVFBFMIN 95 +#define RISCV_ISA_EXT_ZVFBFWMA 96 +#define RISCV_ISA_EXT_ZAAMO 97 +#define RISCV_ISA_EXT_ZALRSC 98 +#define RISCV_ISA_EXT_ZICBOP 99 +#define RISCV_ISA_EXT_SVRSW60T59B 100 +#define RISCV_ISA_EXT_ZALASR 101 +#define RISCV_ISA_EXT_ZILSD 102 +#define RISCV_ISA_EXT_ZCLSD 103 +#define RISCV_ISA_EXT_ZICFILP 104 +#define RISCV_ISA_EXT_ZICFISS 105 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h index dd624523981c..8b9f5e1cf4cb 100644 --- a/arch/riscv/include/asm/hwprobe.h +++ b/arch/riscv/include/asm/hwprobe.h @@ -8,7 +8,7 @@ #include <uapi/asm/hwprobe.h> -#define RISCV_HWPROBE_MAX_KEY 11 +#define RISCV_HWPROBE_MAX_KEY 16 static inline bool riscv_hwprobe_key_is_valid(__s64 key) { @@ -20,8 +20,11 @@ static inline bool hwprobe_key_is_bitmask(__s64 key) switch (key) { case RISCV_HWPROBE_KEY_BASE_BEHAVIOR: case RISCV_HWPROBE_KEY_IMA_EXT_0: + case RISCV_HWPROBE_KEY_IMA_EXT_1: case RISCV_HWPROBE_KEY_CPUPERF_0: case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: + case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: + case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0: return true; } @@ -40,4 +43,11 @@ static inline bool riscv_hwprobe_pair_cmp(struct riscv_hwprobe *pair, return pair->value == other_pair->value; } +#ifdef CONFIG_MMU +void riscv_hwprobe_register_async_probe(void); +void riscv_hwprobe_complete_async_probe(void); +#else +static inline void riscv_hwprobe_register_async_probe(void) {} +static inline void riscv_hwprobe_complete_async_probe(void) {} +#endif #endif diff --git a/arch/riscv/include/asm/image.h b/arch/riscv/include/asm/image.h index e0b319af3681..899254966e85 100644 --- a/arch/riscv/include/asm/image.h +++ b/arch/riscv/include/asm/image.h @@ -29,7 +29,9 @@ #define RISCV_HEADER_VERSION (RISCV_HEADER_VERSION_MAJOR << 16 | \ RISCV_HEADER_VERSION_MINOR) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +#define riscv_image_flag_field(flags, field)\ + (((flags) >> field##_SHIFT) & field##_MASK) /** * struct riscv_image_header - riscv kernel image header * @code0: Executable code @@ -61,5 +63,5 @@ struct riscv_image_header { u32 magic2; u32 res3; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_IMAGE_H */ diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 9a913010cdd9..7c6daf116756 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -18,7 +18,14 @@ #define INSN_I_RD_SHIFT 7 #define INSN_I_OPCODE_SHIFT 0 -#ifdef __ASSEMBLY__ +#define INSN_S_SIMM7_SHIFT 25 +#define INSN_S_RS2_SHIFT 20 +#define INSN_S_RS1_SHIFT 15 +#define INSN_S_FUNC3_SHIFT 12 +#define INSN_S_SIMM5_SHIFT 7 +#define INSN_S_OPCODE_SHIFT 0 + +#ifdef __ASSEMBLER__ #ifdef CONFIG_AS_HAS_INSN @@ -30,6 +37,10 @@ .insn i \opcode, \func3, \rd, \rs1, \simm12 .endm + .macro insn_s, opcode, func3, rs2, simm12, rs1 + .insn s \opcode, \func3, \rs2, \simm12(\rs1) + .endm + #else #include <asm/gpr-num.h> @@ -51,12 +62,22 @@ (\simm12 << INSN_I_SIMM12_SHIFT)) .endm + .macro insn_s, opcode, func3, rs2, simm12, rs1 + .4byte ((\opcode << INSN_S_OPCODE_SHIFT) | \ + (\func3 << INSN_S_FUNC3_SHIFT) | \ + (.L__gpr_num_\rs2 << INSN_S_RS2_SHIFT) | \ + (.L__gpr_num_\rs1 << INSN_S_RS1_SHIFT) | \ + ((\simm12 & 0x1f) << INSN_S_SIMM5_SHIFT) | \ + (((\simm12 >> 5) & 0x7f) << INSN_S_SIMM7_SHIFT)) + .endm + #endif #define __INSN_R(...) insn_r __VA_ARGS__ #define __INSN_I(...) insn_i __VA_ARGS__ +#define __INSN_S(...) insn_s __VA_ARGS__ -#else /* ! __ASSEMBLY__ */ +#else /* ! __ASSEMBLER__ */ #ifdef CONFIG_AS_HAS_INSN @@ -66,6 +87,9 @@ #define __INSN_I(opcode, func3, rd, rs1, simm12) \ ".insn i " opcode ", " func3 ", " rd ", " rs1 ", " simm12 "\n" +#define __INSN_S(opcode, func3, rs2, simm12, rs1) \ + ".insn s " opcode ", " func3 ", " rs2 ", " simm12 "(" rs1 ")\n" + #else #include <linux/stringify.h> @@ -92,12 +116,26 @@ " (\\simm12 << " __stringify(INSN_I_SIMM12_SHIFT) "))\n" \ " .endm\n" +#define DEFINE_INSN_S \ + __DEFINE_ASM_GPR_NUMS \ +" .macro insn_s, opcode, func3, rs2, simm12, rs1\n" \ +" .4byte ((\\opcode << " __stringify(INSN_S_OPCODE_SHIFT) ") |" \ +" (\\func3 << " __stringify(INSN_S_FUNC3_SHIFT) ") |" \ +" (.L__gpr_num_\\rs2 << " __stringify(INSN_S_RS2_SHIFT) ") |" \ +" (.L__gpr_num_\\rs1 << " __stringify(INSN_S_RS1_SHIFT) ") |" \ +" ((\\simm12 & 0x1f) << " __stringify(INSN_S_SIMM5_SHIFT) ") |" \ +" (((\\simm12 >> 5) & 0x7f) << " __stringify(INSN_S_SIMM7_SHIFT) "))\n" \ +" .endm\n" + #define UNDEFINE_INSN_R \ " .purgem insn_r\n" #define UNDEFINE_INSN_I \ " .purgem insn_i\n" +#define UNDEFINE_INSN_S \ +" .purgem insn_s\n" + #define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \ DEFINE_INSN_R \ "insn_r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" \ @@ -108,9 +146,14 @@ "insn_i " opcode ", " func3 ", " rd ", " rs1 ", " simm12 "\n" \ UNDEFINE_INSN_I +#define __INSN_S(opcode, func3, rs2, simm12, rs1) \ + DEFINE_INSN_S \ + "insn_s " opcode ", " func3 ", " rs2 ", " simm12 ", " rs1 "\n" \ + UNDEFINE_INSN_S + #endif -#endif /* ! __ASSEMBLY__ */ +#endif /* ! __ASSEMBLER__ */ #define INSN_R(opcode, func3, func7, rd, rs1, rs2) \ __INSN_R(RV_##opcode, RV_##func3, RV_##func7, \ @@ -120,6 +163,10 @@ __INSN_I(RV_##opcode, RV_##func3, RV_##rd, \ RV_##rs1, RV_##simm12) +#define INSN_S(opcode, func3, rs2, simm12, rs1) \ + __INSN_S(RV_##opcode, RV_##func3, RV_##rs2, \ + RV_##simm12, RV_##rs1) + #define RV_OPCODE(v) __ASM_STR(v) #define RV_FUNC3(v) __ASM_STR(v) #define RV_FUNC7(v) __ASM_STR(v) @@ -132,7 +179,9 @@ #define RV___RS1(v) __RV_REG(v) #define RV___RS2(v) __RV_REG(v) +#define RV_OPCODE_AMO RV_OPCODE(47) #define RV_OPCODE_MISC_MEM RV_OPCODE(15) +#define RV_OPCODE_OP_IMM RV_OPCODE(19) #define RV_OPCODE_SYSTEM RV_OPCODE(115) #define HFENCE_VVMA(vaddr, asid) \ @@ -160,6 +209,84 @@ __ASM_STR(.error "hlv.d requires 64-bit support") #endif +#define LB_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LB_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LH_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LH_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LW_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LW_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define SB_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SB_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#define SH_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SH_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#define SW_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SW_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#ifdef CONFIG_64BIT +#define LD_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LD_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define SD_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SD_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) +#else +#define LD_AQ(dest, addr) \ + __ASM_STR(.error "ld.aq requires 64-bit support") + +#define LD_AQRL(dest, addr) \ + __ASM_STR(.error "ld.aqrl requires 64-bit support") + +#define SD_RL(dest, addr) \ + __ASM_STR(.error "sd.rl requires 64-bit support") + +#define SD_AQRL(dest, addr) \ + __ASM_STR(.error "sd.aqrl requires 64-bit support") +#endif + #define SINVAL_VMA(vaddr, asid) \ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(11), \ __RD(0), RS1(vaddr), RS2(asid)) @@ -196,8 +323,29 @@ INSN_I(OPCODE_MISC_MEM, FUNC3(2), __RD(0), \ RS1(base), SIMM12(4)) -#define RISCV_PAUSE ".4byte 0x100000f" -#define ZAWRS_WRS_NTO ".4byte 0x00d00073" -#define ZAWRS_WRS_STO ".4byte 0x01d00073" +#define PREFETCH_I(base, offset) \ + INSN_S(OPCODE_OP_IMM, FUNC3(6), __RS2(0), \ + SIMM12((offset) & 0xfe0), RS1(base)) + +#define PREFETCH_R(base, offset) \ + INSN_S(OPCODE_OP_IMM, FUNC3(6), __RS2(1), \ + SIMM12((offset) & 0xfe0), RS1(base)) + +#define PREFETCH_W(base, offset) \ + INSN_S(OPCODE_OP_IMM, FUNC3(6), __RS2(3), \ + SIMM12((offset) & 0xfe0), RS1(base)) + +#define RISCV_PAUSE ASM_INSN_I("0x100000f") +#define ZAWRS_WRS_NTO ASM_INSN_I("0x00d00073") +#define ZAWRS_WRS_STO ASM_INSN_I("0x01d00073") +#define RISCV_NOP4 ASM_INSN_I("0x00000013") + +#define RISCV_INSN_NOP4 _AC(0x00000013, U) + +#ifndef __ASSEMBLER__ +#define nop() __asm__ __volatile__ ("nop") +#define __nops(n) ".rept " #n "\nnop\n.endr\n" +#define nops(n) __asm__ __volatile__ (__nops(n)) +#endif #endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h index 09fde95a5e8f..c3005573e8c9 100644 --- a/arch/riscv/include/asm/insn.h +++ b/arch/riscv/include/asm/insn.h @@ -64,6 +64,7 @@ #define RVG_RS2_OPOFF 20 #define RVG_RD_OPOFF 7 #define RVG_RS1_MASK GENMASK(4, 0) +#define RVG_RS2_MASK GENMASK(4, 0) #define RVG_RD_MASK GENMASK(4, 0) /* The bit field of immediate value in RVC J instruction */ @@ -286,45 +287,216 @@ static __always_inline bool riscv_insn_is_c_jalr(u32 code) (code & RVC_INSN_J_RS1_MASK) != 0; } -#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) -#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) -#define RV_X(X, s, mask) (((X) >> (s)) & (mask)) -#define RVC_X(X, s, mask) RV_X(X, s, mask) +#define INSN_MATCH_LB 0x3 +#define INSN_MASK_LB 0x707f +#define INSN_MATCH_LH 0x1003 +#define INSN_MASK_LH 0x707f +#define INSN_MATCH_LW 0x2003 +#define INSN_MASK_LW 0x707f +#define INSN_MATCH_LD 0x3003 +#define INSN_MASK_LD 0x707f +#define INSN_MATCH_LBU 0x4003 +#define INSN_MASK_LBU 0x707f +#define INSN_MATCH_LHU 0x5003 +#define INSN_MASK_LHU 0x707f +#define INSN_MATCH_LWU 0x6003 +#define INSN_MASK_LWU 0x707f +#define INSN_MATCH_SB 0x23 +#define INSN_MASK_SB 0x707f +#define INSN_MATCH_SH 0x1023 +#define INSN_MASK_SH 0x707f +#define INSN_MATCH_SW 0x2023 +#define INSN_MASK_SW 0x707f +#define INSN_MATCH_SD 0x3023 +#define INSN_MASK_SD 0x707f + +#define INSN_MATCH_C_LD 0x6000 +#define INSN_MASK_C_LD 0xe003 +#define INSN_MATCH_C_SD 0xe000 +#define INSN_MASK_C_SD 0xe003 +#define INSN_MATCH_C_LW 0x4000 +#define INSN_MASK_C_LW 0xe003 +#define INSN_MATCH_C_SW 0xc000 +#define INSN_MASK_C_SW 0xe003 +#define INSN_MATCH_C_LDSP 0x6002 +#define INSN_MASK_C_LDSP 0xe003 +#define INSN_MATCH_C_SDSP 0xe002 +#define INSN_MASK_C_SDSP 0xe003 +#define INSN_MATCH_C_LWSP 0x4002 +#define INSN_MASK_C_LWSP 0xe003 +#define INSN_MATCH_C_SWSP 0xc002 +#define INSN_MASK_C_SWSP 0xe003 + +#define INSN_OPCODE_MASK 0x007c +#define INSN_OPCODE_SHIFT 2 +#define INSN_OPCODE_SYSTEM 28 + +#define INSN_MASK_WFI 0xffffffff +#define INSN_MATCH_WFI 0x10500073 + +#define INSN_MASK_WRS 0xffffffff +#define INSN_MATCH_WRS 0x00d00073 + +#define INSN_MATCH_CSRRW 0x1073 +#define INSN_MASK_CSRRW 0x707f +#define INSN_MATCH_CSRRS 0x2073 +#define INSN_MASK_CSRRS 0x707f +#define INSN_MATCH_CSRRC 0x3073 +#define INSN_MASK_CSRRC 0x707f +#define INSN_MATCH_CSRRWI 0x5073 +#define INSN_MASK_CSRRWI 0x707f +#define INSN_MATCH_CSRRSI 0x6073 +#define INSN_MASK_CSRRSI 0x707f +#define INSN_MATCH_CSRRCI 0x7073 +#define INSN_MASK_CSRRCI 0x707f + +#define INSN_MATCH_FLW 0x2007 +#define INSN_MASK_FLW 0x707f +#define INSN_MATCH_FLD 0x3007 +#define INSN_MASK_FLD 0x707f +#define INSN_MATCH_FLQ 0x4007 +#define INSN_MASK_FLQ 0x707f +#define INSN_MATCH_FSW 0x2027 +#define INSN_MASK_FSW 0x707f +#define INSN_MATCH_FSD 0x3027 +#define INSN_MASK_FSD 0x707f +#define INSN_MATCH_FSQ 0x4027 +#define INSN_MASK_FSQ 0x707f + +#define INSN_MATCH_C_FLD 0x2000 +#define INSN_MASK_C_FLD 0xe003 +#define INSN_MATCH_C_FLW 0x6000 +#define INSN_MASK_C_FLW 0xe003 +#define INSN_MATCH_C_FSD 0xa000 +#define INSN_MASK_C_FSD 0xe003 +#define INSN_MATCH_C_FSW 0xe000 +#define INSN_MASK_C_FSW 0xe003 +#define INSN_MATCH_C_FLDSP 0x2002 +#define INSN_MASK_C_FLDSP 0xe003 +#define INSN_MATCH_C_FSDSP 0xa002 +#define INSN_MASK_C_FSDSP 0xe003 +#define INSN_MATCH_C_FLWSP 0x6002 +#define INSN_MASK_C_FLWSP 0xe003 +#define INSN_MATCH_C_FSWSP 0xe002 +#define INSN_MASK_C_FSWSP 0xe003 + +#define INSN_MATCH_C_LHU 0x8400 +#define INSN_MASK_C_LHU 0xfc43 +#define INSN_MATCH_C_LH 0x8440 +#define INSN_MASK_C_LH 0xfc43 +#define INSN_MATCH_C_SH 0x8c00 +#define INSN_MASK_C_SH 0xfc43 + +#define INSN_16BIT_MASK 0x3 +#define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK) +#define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4) + +#define SHIFT_RIGHT(x, y) \ + ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) + +#define REG_MASK \ + ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) + +#define REG_OFFSET(insn, pos) \ + (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) + +#define REG_PTR(insn, pos, regs) \ + ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))) + +#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) +#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) +#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) +#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) +#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) +#define GET_SP(regs) (*REG_PTR(2, 0, regs)) +#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) +#define IMM_I(insn) ((s32)(insn) >> 20) +#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ + (s32)(((insn) >> 7) & 0x1f)) + +#define SH_RD 7 +#define SH_RS1 15 +#define SH_RS2 20 +#define SH_RS2C 2 +#define MASK_RX 0x1f + +#if defined(CONFIG_64BIT) +#define LOG_REGBYTES 3 +#else +#define LOG_REGBYTES 2 +#endif + +#define MASK_FUNCT3 0x7000 + +#define GET_FUNCT3(insn) (((insn) >> 12) & 7) + +#define RV_IMM_SIGN(x) (-(((x) >> 31) & 1)) +#define RVC_IMM_SIGN(x) (-(((x) >> 12) & 1)) +#define RV_X_MASK(X, s, mask) (((X) >> (s)) & (mask)) +#define RV_X(X, s, n) RV_X_MASK(X, s, ((1 << (n)) - 1)) +#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ + (RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 1) << 6)) +#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 5, 2) << 6)) +#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 2) << 6)) +#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ + (RV_X(x, 12, 1) << 5) | \ + (RV_X(x, 2, 3) << 6)) +#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ + (RV_X(x, 7, 2) << 6)) +#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ + (RV_X(x, 7, 3) << 6)) +#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) +#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) +#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) +#define RVC_X(X, s, mask) RV_X_MASK(X, s, mask) + +#define RV_EXTRACT_FUNCT3(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RV_INSN_FUNCT3_OPOFF, \ + RV_INSN_FUNCT3_MASK >> RV_INSN_FUNCT3_OPOFF)); }) #define RV_EXTRACT_RS1_REG(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RVG_RS1_OPOFF, RVG_RS1_MASK)); }) + (RV_X_MASK(x_, RVG_RS1_OPOFF, RVG_RS1_MASK)); }) + +#define RV_EXTRACT_RS2_REG(x) \ + ({typeof(x) x_ = (x); \ + (RV_X_MASK(x_, RVG_RS2_OPOFF, RVG_RS2_MASK)); }) #define RV_EXTRACT_RD_REG(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RVG_RD_OPOFF, RVG_RD_MASK)); }) + (RV_X_MASK(x_, RVG_RD_OPOFF, RVG_RD_MASK)); }) #define RV_EXTRACT_UTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) + (RV_X_MASK(x_, RV_U_IMM_31_12_OPOFF, RV_U_IMM_31_12_MASK)); }) #define RV_EXTRACT_JTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ - (RV_X(x_, RV_J_IMM_11_OPOFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OFF) | \ - (RV_X(x_, RV_J_IMM_19_12_OPOFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OFF) | \ + (RV_X_MASK(x_, RV_J_IMM_10_1_OPOFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OFF) | \ + (RV_X_MASK(x_, RV_J_IMM_11_OPOFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OFF) | \ + (RV_X_MASK(x_, RV_J_IMM_19_12_OPOFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OFF) | \ (RV_IMM_SIGN(x_) << RV_J_IMM_SIGN_OFF); }) #define RV_EXTRACT_ITYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RV_I_IMM_11_0_OPOFF, RV_I_IMM_11_0_MASK)) | \ + (RV_X_MASK(x_, RV_I_IMM_11_0_OPOFF, RV_I_IMM_11_0_MASK)) | \ (RV_IMM_SIGN(x_) << RV_I_IMM_SIGN_OFF); }) #define RV_EXTRACT_BTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RV_B_IMM_4_1_OPOFF, RV_B_IMM_4_1_MASK) << RV_B_IMM_4_1_OFF) | \ - (RV_X(x_, RV_B_IMM_10_5_OPOFF, RV_B_IMM_10_5_MASK) << RV_B_IMM_10_5_OFF) | \ - (RV_X(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \ + (RV_X_MASK(x_, RV_B_IMM_4_1_OPOFF, RV_B_IMM_4_1_MASK) << RV_B_IMM_4_1_OFF) | \ + (RV_X_MASK(x_, RV_B_IMM_10_5_OPOFF, RV_B_IMM_10_5_MASK) << RV_B_IMM_10_5_OFF) | \ + (RV_X_MASK(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \ (RV_IMM_SIGN(x_) << RV_B_IMM_SIGN_OFF); }) #define RVC_EXTRACT_C2_RS1_REG(x) \ ({typeof(x) x_ = (x); \ - (RV_X(x_, RVC_C2_RS1_OPOFF, RVC_C2_RS1_MASK)); }) + (RV_X_MASK(x_, RVC_C2_RS1_OPOFF, RVC_C2_RS1_MASK)); }) #define RVC_EXTRACT_JTYPE_IMM(x) \ ({typeof(x) x_ = (x); \ @@ -346,13 +518,13 @@ static __always_inline bool riscv_insn_is_c_jalr(u32 code) (RVC_IMM_SIGN(x_) << RVC_B_IMM_SIGN_OFF); }) #define RVG_EXTRACT_SYSTEM_CSR(x) \ - ({typeof(x) x_ = (x); RV_X(x_, RVG_SYSTEM_CSR_OFF, RVG_SYSTEM_CSR_MASK); }) + ({typeof(x) x_ = (x); RV_X_MASK(x_, RVG_SYSTEM_CSR_OFF, RVG_SYSTEM_CSR_MASK); }) #define RVFDQ_EXTRACT_FL_FS_WIDTH(x) \ - ({typeof(x) x_ = (x); RV_X(x_, RVFDQ_FL_FS_WIDTH_OFF, \ + ({typeof(x) x_ = (x); RV_X_MASK(x_, RVFDQ_FL_FS_WIDTH_OFF, \ RVFDQ_FL_FS_WIDTH_MASK); }) -#define RVV_EXRACT_VL_VS_WIDTH(x) RVFDQ_EXTRACT_FL_FS_WIDTH(x) +#define RVV_EXTRACT_VL_VS_WIDTH(x) RVFDQ_EXTRACT_FL_FS_WIDTH(x) /* * Get the immediate from a J-type instruction. @@ -375,10 +547,10 @@ static inline void riscv_insn_insert_jtype_imm(u32 *insn, s32 imm) { /* drop the old IMMs, all jal IMM bits sit at 31:12 */ *insn &= ~GENMASK(31, 12); - *insn |= (RV_X(imm, RV_J_IMM_10_1_OFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OPOFF) | - (RV_X(imm, RV_J_IMM_11_OFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OPOFF) | - (RV_X(imm, RV_J_IMM_19_12_OFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OPOFF) | - (RV_X(imm, RV_J_IMM_SIGN_OFF, 1) << RV_J_IMM_SIGN_OPOFF); + *insn |= (RV_X_MASK(imm, RV_J_IMM_10_1_OFF, RV_J_IMM_10_1_MASK) << RV_J_IMM_10_1_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_11_OFF, RV_J_IMM_11_MASK) << RV_J_IMM_11_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_19_12_OFF, RV_J_IMM_19_12_MASK) << RV_J_IMM_19_12_OPOFF) | + (RV_X_MASK(imm, RV_J_IMM_SIGN_OFF, 1) << RV_J_IMM_SIGN_OPOFF); } /* diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 1c5c641075d2..09bb5f57a9d3 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -28,6 +28,10 @@ #ifdef CONFIG_MMU #define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) #define PCI_IOBASE ((void __iomem *)PCI_IO_START) + +#define ioremap_wc(addr, size) \ + ioremap_prot((addr), (size), __pgprot(_PAGE_KERNEL_NC)) + #endif /* CONFIG_MMU */ /* @@ -136,8 +140,8 @@ __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) #include <asm-generic/io.h> #ifdef CONFIG_MMU -#define arch_memremap_wb(addr, size) \ - ((__force void *)ioremap_prot((addr), (size), _PAGE_KERNEL)) +#define arch_memremap_wb(addr, size, flags) \ + ((__force void *)ioremap_prot((addr), (size), __pgprot(_PAGE_KERNEL))) #endif #endif /* _ASM_RISCV_IO_H */ diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 7b038f3b7cb0..e29ded3416b4 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -22,6 +22,8 @@ void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); void riscv_set_intc_hwnode_fn(struct fwnode_handle *(*fn)(void)); struct fwnode_handle *riscv_get_intc_hwnode(void); +int riscv_get_hart_index(struct fwnode_handle *fwnode, u32 logical_index, + u32 *hart_index); #ifdef CONFIG_ACPI @@ -30,6 +32,7 @@ enum riscv_irqchip_type { ACPI_RISCV_IRQCHIP_IMSIC = 0x01, ACPI_RISCV_IRQCHIP_PLIC = 0x02, ACPI_RISCV_IRQCHIP_APLIC = 0x03, + ACPI_RISCV_IRQCHIP_SMSI = 0x04, }; int riscv_acpi_get_gsi_info(struct fwnode_handle *fwnode, u32 *gsi_base, @@ -40,6 +43,7 @@ unsigned long acpi_rintc_ext_parent_to_hartid(unsigned int plic_id, unsigned int unsigned int acpi_rintc_get_plic_nr_contexts(unsigned int plic_id); unsigned int acpi_rintc_get_plic_context(unsigned int plic_id, unsigned int ctxt_idx); int __init acpi_rintc_get_imsic_mmio_info(u32 index, struct resource *res); +int riscv_acpi_update_gsi_range(u32 gsi_base, u32 nr_irqs); #else static inline int riscv_acpi_get_gsi_info(struct fwnode_handle *fwnode, u32 *gsi_base, @@ -74,6 +78,10 @@ static inline int __init acpi_rintc_get_imsic_mmio_info(u32 index, struct resour return 0; } +static inline int riscv_acpi_update_gsi_range(u32 gsi_base, u32 nr_irqs) +{ + return -ENODEV; +} #endif /* CONFIG_ACPI */ #endif /* _ASM_RISCV_IRQ_H */ diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 87a71cc6d146..3ab5f2e3212b 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -7,7 +7,7 @@ #ifndef __ASM_JUMP_LABEL_H #define __ASM_JUMP_LABEL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <asm/asm.h> @@ -66,5 +66,5 @@ label: return true; } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index e6a0071bdb56..60af6691f903 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -4,7 +4,7 @@ #ifndef __ASM_KASAN_H #define __ASM_KASAN_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The following comment was copied from arm64: diff --git a/arch/riscv/include/asm/kexec.h b/arch/riscv/include/asm/kexec.h index 2b56769cb530..b9ee8346cc8c 100644 --- a/arch/riscv/include/asm/kexec.h +++ b/arch/riscv/include/asm/kexec.h @@ -56,6 +56,7 @@ extern riscv_kexec_method riscv_kexec_norelocate; #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops elf_kexec_ops; +extern const struct kexec_file_ops image_kexec_ops; struct purgatory_info; int arch_kexec_apply_relocations_add(struct purgatory_info *pi, @@ -67,6 +68,11 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, struct kimage; int arch_kimage_file_post_load_cleanup(struct kimage *image); #define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup + +int load_extra_segments(struct kimage *image, unsigned long kernel_start, + unsigned long kernel_len, char *initrd, + unsigned long initrd_len, char *cmdline, + unsigned long cmdline_len); #endif #endif diff --git a/arch/riscv/include/asm/kgdb.h b/arch/riscv/include/asm/kgdb.h index 46677daf708b..78b18e2fd771 100644 --- a/arch/riscv/include/asm/kgdb.h +++ b/arch/riscv/include/asm/kgdb.h @@ -3,33 +3,30 @@ #ifndef __ASM_KGDB_H_ #define __ASM_KGDB_H_ +#include <linux/build_bug.h> + #ifdef __KERNEL__ #define GDB_SIZEOF_REG sizeof(unsigned long) -#define DBG_MAX_REG_NUM (36) -#define NUMREGBYTES ((DBG_MAX_REG_NUM) * GDB_SIZEOF_REG) +#define DBG_MAX_REG_NUM 36 +#define NUMREGBYTES (DBG_MAX_REG_NUM * GDB_SIZEOF_REG) #define CACHE_FLUSH_IS_SAFE 1 #define BUFMAX 2048 +static_assert(BUFMAX > NUMREGBYTES, + "As per KGDB documentation, BUFMAX must be larger than NUMREGBYTES"); #ifdef CONFIG_RISCV_ISA_C #define BREAK_INSTR_SIZE 2 #else #define BREAK_INSTR_SIZE 4 #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +void arch_kgdb_breakpoint(void); extern unsigned long kgdb_compiled_break; -static inline void arch_kgdb_breakpoint(void) -{ - asm(".global kgdb_compiled_break\n" - ".option norvc\n" - "kgdb_compiled_break: ebreak\n" - ".option rvc\n"); -} - -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #define DBG_REG_ZERO "zero" #define DBG_REG_RA "ra" @@ -104,6 +101,7 @@ static inline void arch_kgdb_breakpoint(void) #define DBG_REG_STATUS_OFF 33 #define DBG_REG_BADADDR_OFF 34 #define DBG_REG_CAUSE_OFF 35 +/* NOTE: increase DBG_MAX_REG_NUM if you add more values here. */ extern const char riscv_gdb_stub_feature[64]; diff --git a/arch/riscv/include/asm/kvm_aia.h b/arch/riscv/include/asm/kvm_aia.h index 1f37b600ca47..b04ecdd1a860 100644 --- a/arch/riscv/include/asm/kvm_aia.h +++ b/arch/riscv/include/asm/kvm_aia.h @@ -63,9 +63,6 @@ struct kvm_vcpu_aia { /* CPU AIA CSR context of Guest VCPU */ struct kvm_vcpu_aia_csr guest_csr; - /* CPU AIA CSR context upon Guest VCPU reset */ - struct kvm_vcpu_aia_csr guest_reset_csr; - /* Guest physical address of IMSIC for this VCPU */ gpa_t imsic_addr; @@ -90,6 +87,9 @@ DECLARE_STATIC_KEY_FALSE(kvm_riscv_aia_available); extern struct kvm_device_ops kvm_riscv_aia_device_ops; +bool kvm_riscv_vcpu_aia_imsic_has_interrupt(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_aia_imsic_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_riscv_vcpu_aia_imsic_put(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu); int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu); @@ -150,7 +150,7 @@ int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu); -int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu); int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index, @@ -164,7 +164,6 @@ void kvm_riscv_aia_destroy_vm(struct kvm *kvm); int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, void __iomem **hgei_va, phys_addr_t *hgei_pa); void kvm_riscv_aia_free_hgei(int cpu, int hgei); -void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable); void kvm_riscv_aia_enable(void); void kvm_riscv_aia_disable(void); diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h new file mode 100644 index 000000000000..9c908432bc17 --- /dev/null +++ b/arch/riscv/include/asm/kvm_gstage.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#ifndef __RISCV_KVM_GSTAGE_H_ +#define __RISCV_KVM_GSTAGE_H_ + +#include <linux/kvm_types.h> + +struct kvm_gstage { + struct kvm *kvm; + unsigned long flags; +#define KVM_GSTAGE_FLAGS_LOCAL BIT(0) + unsigned long vmid; + pgd_t *pgd; + unsigned long pgd_levels; +}; + +struct kvm_gstage_mapping { + gpa_t addr; + pte_t pte; + u32 level; +}; + +#ifdef CONFIG_64BIT +#define kvm_riscv_gstage_index_bits 9 +#else +#define kvm_riscv_gstage_index_bits 10 +#endif + +extern unsigned long kvm_riscv_gstage_max_pgd_levels; + +#define kvm_riscv_gstage_pgd_xbits 2 +#define kvm_riscv_gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + kvm_riscv_gstage_pgd_xbits)) + +static inline unsigned long kvm_riscv_gstage_gpa_bits(unsigned long pgd_levels) +{ + return (HGATP_PAGE_SHIFT + + pgd_levels * kvm_riscv_gstage_index_bits + + kvm_riscv_gstage_pgd_xbits); +} + +static inline gpa_t kvm_riscv_gstage_gpa_size(unsigned long pgd_levels) +{ + return BIT_ULL(kvm_riscv_gstage_gpa_bits(pgd_levels)); +} + +bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, + pte_t **ptepp, u32 *ptep_level); + +int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + const struct kvm_gstage_mapping *map); + +int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t gpa, phys_addr_t hpa, unsigned long page_size, + bool page_rdonly, bool page_exec, + struct kvm_gstage_mapping *out_map); + +int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush); + +enum kvm_riscv_gstage_op { + GSTAGE_OP_NOP = 0, /* Nothing */ + GSTAGE_OP_CLEAR, /* Clear/Unmap */ + GSTAGE_OP_WP, /* Write-protect */ +}; + +void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, + pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op); + +void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, + gpa_t start, gpa_t size, bool may_block); + +void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end); + +void kvm_riscv_gstage_mode_detect(void); + +static inline unsigned long kvm_riscv_gstage_mode(unsigned long pgd_levels) +{ + switch (pgd_levels) { + case 2: + return HGATP_MODE_SV32X4; + case 3: + return HGATP_MODE_SV39X4; + case 4: + return HGATP_MODE_SV48X4; + case 5: + return HGATP_MODE_SV57X4; + default: + WARN_ON_ONCE(1); + return HGATP_MODE_OFF; + } +} + +static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *kvm) +{ + gstage->kvm = kvm; + gstage->flags = 0; + gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid); + gstage->pgd = kvm->arch.pgd; + gstage->pgd_levels = kvm->arch.pgd_levels; +} + +#endif diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index cc33e35cd628..75b0a951c1bc 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -16,9 +16,13 @@ #include <asm/hwcap.h> #include <asm/kvm_aia.h> #include <asm/ptrace.h> +#include <asm/kvm_tlb.h> +#include <asm/kvm_vmid.h> +#include <asm/kvm_vcpu_config.h> #include <asm/kvm_vcpu_fp.h> #include <asm/kvm_vcpu_insn.h> #include <asm/kvm_vcpu_sbi.h> +#include <asm/kvm_vcpu_sbi_fwft.h> #include <asm/kvm_vcpu_timer.h> #include <asm/kvm_vcpu_pmu.h> @@ -36,41 +40,16 @@ #define KVM_REQ_UPDATE_HGATP KVM_ARCH_REQ(2) #define KVM_REQ_FENCE_I \ KVM_ARCH_REQ_FLAGS(3, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) -#define KVM_REQ_HFENCE_GVMA_VMID_ALL KVM_REQ_TLB_FLUSH #define KVM_REQ_HFENCE_VVMA_ALL \ KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HFENCE \ KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6) -#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \ - BIT(EXC_BREAKPOINT) | \ - BIT(EXC_SYSCALL) | \ - BIT(EXC_INST_PAGE_FAULT) | \ - BIT(EXC_LOAD_PAGE_FAULT) | \ - BIT(EXC_STORE_PAGE_FAULT)) - -#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \ - BIT(IRQ_VS_TIMER) | \ - BIT(IRQ_VS_EXT)) - -enum kvm_riscv_hfence_type { - KVM_RISCV_HFENCE_UNKNOWN = 0, - KVM_RISCV_HFENCE_GVMA_VMID_GPA, - KVM_RISCV_HFENCE_VVMA_ASID_GVA, - KVM_RISCV_HFENCE_VVMA_ASID_ALL, - KVM_RISCV_HFENCE_VVMA_GVA, -}; - -struct kvm_riscv_hfence { - enum kvm_riscv_hfence_type type; - unsigned long asid; - unsigned long order; - gpa_t addr; - gpa_t size; -}; +#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define KVM_RISCV_VCPU_MAX_HFENCE 64 +#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ + KVM_DIRTY_LOG_INITIALLY_SET) struct kvm_vm_stat { struct kvm_vm_stat_generic generic; @@ -97,15 +76,6 @@ struct kvm_vcpu_stat { struct kvm_arch_memory_slot { }; -struct kvm_vmid { - /* - * Writes to vmid_version and vmid happen with vmid_lock held - * whereas reads happen without any lock held. - */ - unsigned long vmid_version; - unsigned long vmid; -}; - struct kvm_arch { /* G-stage vmid */ struct kvm_vmid vmid; @@ -113,12 +83,16 @@ struct kvm_arch { /* G-stage page table */ pgd_t *pgd; phys_addr_t pgd_phys; + unsigned long pgd_levels; /* Guest Timer */ struct kvm_guest_timer timer; /* AIA Guest/VM context */ struct kvm_aia aia; + + /* KVM_CAP_RISCV_MP_STATE_RESET */ + bool mp_state_reset; }; struct kvm_cpu_trap { @@ -183,16 +157,16 @@ struct kvm_vcpu_csr { unsigned long senvcfg; }; -struct kvm_vcpu_config { - u64 henvcfg; - u64 hstateen0; - unsigned long hedeleg; -}; - struct kvm_vcpu_smstateen_csr { unsigned long sstateen0; }; +struct kvm_vcpu_reset_state { + spinlock_t lock; + unsigned long pc; + unsigned long a1; +}; + struct kvm_vcpu_arch { /* VCPU ran at least once */ bool ran_atleast_once; @@ -227,12 +201,8 @@ struct kvm_vcpu_arch { /* CPU Smstateen CSR context of Guest VCPU */ struct kvm_vcpu_smstateen_csr smstateen_csr; - /* CPU context upon Guest VCPU reset */ - struct kvm_cpu_context guest_reset_context; - spinlock_t reset_cntx_lock; - - /* CPU CSR context upon Guest VCPU reset */ - struct kvm_vcpu_csr guest_reset_csr; + /* CPU reset state of Guest VCPU */ + struct kvm_vcpu_reset_state reset_state; /* * VCPU interrupts @@ -281,9 +251,15 @@ struct kvm_vcpu_arch { /* Performance monitoring context */ struct kvm_pmu pmu_context; + /* Firmware feature SBI extension context */ + struct kvm_sbi_fwft fwft_context; + /* 'static' configurations which are set only once */ struct kvm_vcpu_config cfg; + /* Indicates modified guest CSRs */ + bool csr_dirty; + /* SBI steal-time accounting */ struct { gpa_t shmem; @@ -301,78 +277,8 @@ static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; } -static inline void kvm_arch_sync_events(struct kvm *kvm) {} - -#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 - -void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, - gpa_t gpa, gpa_t gpsz, - unsigned long order); -void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid); -void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, - unsigned long order); -void kvm_riscv_local_hfence_gvma_all(void); -void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, - unsigned long asid, - unsigned long gva, - unsigned long gvsz, - unsigned long order); -void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, - unsigned long asid); -void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, - unsigned long gva, unsigned long gvsz, - unsigned long order); -void kvm_riscv_local_hfence_vvma_all(unsigned long vmid); - -void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu); - -void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu); -void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu); -void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu); -void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu); - -void kvm_riscv_fence_i(struct kvm *kvm, - unsigned long hbase, unsigned long hmask); -void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, - unsigned long hbase, unsigned long hmask, - gpa_t gpa, gpa_t gpsz, - unsigned long order); -void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, - unsigned long hbase, unsigned long hmask); -void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, - unsigned long hbase, unsigned long hmask, - unsigned long gva, unsigned long gvsz, - unsigned long order, unsigned long asid); -void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, - unsigned long hbase, unsigned long hmask, - unsigned long asid); -void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, - unsigned long hbase, unsigned long hmask, - unsigned long gva, unsigned long gvsz, - unsigned long order); -void kvm_riscv_hfence_vvma_all(struct kvm *kvm, - unsigned long hbase, unsigned long hmask); - -int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa, - phys_addr_t hpa, unsigned long size, - bool writable, bool in_atomic); -void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa, - unsigned long size); -int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, - struct kvm_memory_slot *memslot, - gpa_t gpa, unsigned long hva, bool is_write); -int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm); -void kvm_riscv_gstage_free_pgd(struct kvm *kvm); -void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu); -void __init kvm_riscv_gstage_mode_detect(void); -unsigned long __init kvm_riscv_gstage_mode(void); -int kvm_riscv_gstage_gpa_bits(void); - -void __init kvm_riscv_gstage_vmid_detect(void); -unsigned long kvm_riscv_gstage_vmid_bits(void); -int kvm_riscv_gstage_vmid_init(struct kvm *kvm); -bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid); -void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu); +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines); @@ -409,7 +315,9 @@ void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu); bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu); -void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu); +/* Flags representing implementation specific details */ +DECLARE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); + #endif /* __RISCV_KVM_HOST_H__ */ diff --git a/arch/riscv/include/asm/kvm_isa.h b/arch/riscv/include/asm/kvm_isa.h new file mode 100644 index 000000000000..bc4b956d5f17 --- /dev/null +++ b/arch/riscv/include/asm/kvm_isa.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_RISCV_ISA_H +#define __KVM_RISCV_ISA_H + +#include <linux/types.h> + +unsigned long kvm_riscv_base2isa_ext(unsigned long base_ext); + +int __kvm_riscv_isa_check_host(unsigned long ext, unsigned long *base_ext); +#define kvm_riscv_isa_check_host(ext) \ + __kvm_riscv_isa_check_host(KVM_RISCV_ISA_EXT_##ext, NULL) + +bool kvm_riscv_isa_enable_allowed(unsigned long ext); +bool kvm_riscv_isa_disable_allowed(unsigned long ext); + +#endif diff --git a/arch/riscv/include/asm/kvm_mmu.h b/arch/riscv/include/asm/kvm_mmu.h new file mode 100644 index 000000000000..5439e76f0a96 --- /dev/null +++ b/arch/riscv/include/asm/kvm_mmu.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#ifndef __RISCV_KVM_MMU_H_ +#define __RISCV_KVM_MMU_H_ + +#include <asm/kvm_gstage.h> + +int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, + unsigned long size, bool writable, bool in_atomic); +void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size); +int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, + gpa_t gpa, unsigned long hva, bool is_write, + struct kvm_gstage_mapping *out_map); +int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm); +void kvm_riscv_mmu_free_pgd(struct kvm *kvm); +void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/riscv/include/asm/kvm_tlb.h b/arch/riscv/include/asm/kvm_tlb.h new file mode 100644 index 000000000000..a0e7099bcb85 --- /dev/null +++ b/arch/riscv/include/asm/kvm_tlb.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#ifndef __RISCV_KVM_TLB_H_ +#define __RISCV_KVM_TLB_H_ + +#include <linux/kvm_types.h> + +enum kvm_riscv_hfence_type { + KVM_RISCV_HFENCE_UNKNOWN = 0, + KVM_RISCV_HFENCE_GVMA_VMID_GPA, + KVM_RISCV_HFENCE_GVMA_VMID_ALL, + KVM_RISCV_HFENCE_VVMA_ASID_GVA, + KVM_RISCV_HFENCE_VVMA_ASID_ALL, + KVM_RISCV_HFENCE_VVMA_GVA, + KVM_RISCV_HFENCE_VVMA_ALL +}; + +struct kvm_riscv_hfence { + enum kvm_riscv_hfence_type type; + unsigned long asid; + unsigned long vmid; + unsigned long order; + gpa_t addr; + gpa_t size; +}; + +#define KVM_RISCV_VCPU_MAX_HFENCE 64 + +#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 + +void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, + gpa_t gpa, gpa_t gpsz, + unsigned long order); +void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid); +void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, + unsigned long order); +void kvm_riscv_local_hfence_gvma_all(void); +void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, + unsigned long asid, + unsigned long gva, + unsigned long gvsz, + unsigned long order); +void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, + unsigned long asid); +void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, + unsigned long gva, unsigned long gvsz, + unsigned long order); +void kvm_riscv_local_hfence_vvma_all(unsigned long vmid); +void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu); + +void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu); + +void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu); + +void kvm_riscv_fence_i(struct kvm *kvm, + unsigned long hbase, unsigned long hmask); +void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + gpa_t gpa, gpa_t gpsz, + unsigned long order, unsigned long vmid); +void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long vmid); +void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order, unsigned long asid, + unsigned long vmid); +void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long asid, unsigned long vmid); +void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order, unsigned long vmid); +void kvm_riscv_hfence_vvma_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long vmid); + +#endif diff --git a/arch/riscv/include/asm/kvm_vcpu_config.h b/arch/riscv/include/asm/kvm_vcpu_config.h new file mode 100644 index 000000000000..fcc15a0296b3 --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_config.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#ifndef __KVM_VCPU_RISCV_CONFIG_H +#define __KVM_VCPU_RISCV_CONFIG_H + +#include <linux/types.h> + +struct kvm_vcpu; + +struct kvm_vcpu_config { + u64 henvcfg; + u64 hstateen0; + unsigned long hedeleg; + unsigned long hideleg; +}; + +void kvm_riscv_vcpu_config_init(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_guest_debug(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_ran_once(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_config_load(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/riscv/include/asm/kvm_vcpu_pmu.h b/arch/riscv/include/asm/kvm_vcpu_pmu.h index 1d85b6617508..9a930afc8f57 100644 --- a/arch/riscv/include/asm/kvm_vcpu_pmu.h +++ b/arch/riscv/include/asm/kvm_vcpu_pmu.h @@ -98,6 +98,9 @@ void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu); int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low, unsigned long saddr_high, unsigned long flags, struct kvm_vcpu_sbi_return *retdata); +int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low, + unsigned long saddr_high, unsigned long num_events, + unsigned long flags, struct kvm_vcpu_sbi_return *retdata); void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu); diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index 4ed6203cdd30..c1a7e3b40d9c 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -11,7 +11,7 @@ #define KVM_SBI_IMPID 3 -#define KVM_SBI_VERSION_MAJOR 2 +#define KVM_SBI_VERSION_MAJOR 3 #define KVM_SBI_VERSION_MINOR 0 enum kvm_riscv_sbi_ext_status { @@ -49,31 +49,50 @@ struct kvm_vcpu_sbi_extension { /* Extension specific probe function */ unsigned long (*probe)(struct kvm_vcpu *vcpu); + + /* + * Init/deinit function called once during VCPU init/destroy. These + * might be use if the SBI extensions need to allocate or do specific + * init time only configuration. + */ + int (*init)(struct kvm_vcpu *vcpu); + void (*deinit)(struct kvm_vcpu *vcpu); + + void (*reset)(struct kvm_vcpu *vcpu); + + unsigned long state_reg_subtype; + unsigned long (*get_state_reg_count)(struct kvm_vcpu *vcpu); + int (*get_state_reg_id)(struct kvm_vcpu *vcpu, int index, u64 *reg_id); + int (*get_state_reg)(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, void *reg_val); + int (*set_state_reg)(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, const void *reg_val); }; -void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_riscv_vcpu_sbi_forward_handler(struct kvm_vcpu *vcpu, + struct kvm_run *run, + struct kvm_vcpu_sbi_return *retdata); void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, struct kvm_run *run, u32 type, u64 flags); +void kvm_riscv_vcpu_sbi_request_reset(struct kvm_vcpu *vcpu, + unsigned long pc, unsigned long a1); +void kvm_riscv_vcpu_sbi_load_reset_state(struct kvm_vcpu *vcpu); int kvm_riscv_vcpu_sbi_return(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_riscv_vcpu_reg_indices_sbi_ext(struct kvm_vcpu *vcpu, u64 __user *uindices); int kvm_riscv_vcpu_set_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, - const struct kvm_one_reg *reg); -int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, - const struct kvm_one_reg *reg); +int kvm_riscv_vcpu_reg_indices_sbi(struct kvm_vcpu *vcpu, u64 __user *uindices); +int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext( struct kvm_vcpu *vcpu, unsigned long extid); -bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx); int kvm_riscv_vcpu_sbi_ecall(struct kvm_vcpu *vcpu, struct kvm_run *run); void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu); - -int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, - unsigned long *reg_val); -int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, unsigned long reg_num, - unsigned long reg_val); +void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu); +void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu); #ifdef CONFIG_RISCV_SBI_V01 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; @@ -87,6 +106,8 @@ extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_susp; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_mpxy; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi_fwft.h b/arch/riscv/include/asm/kvm_vcpu_sbi_fwft.h new file mode 100644 index 000000000000..5604cec79902 --- /dev/null +++ b/arch/riscv/include/asm/kvm_vcpu_sbi_fwft.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Rivos Inc. + * + * Authors: + * Clément Léger <cleger@rivosinc.com> + */ + +#ifndef __KVM_VCPU_RISCV_FWFT_H +#define __KVM_VCPU_RISCV_FWFT_H + +#include <asm/sbi.h> + +struct kvm_sbi_fwft_feature; + +struct kvm_sbi_fwft_config { + const struct kvm_sbi_fwft_feature *feature; + bool supported; + bool enabled; + unsigned long flags; +}; + +/* FWFT data structure per vcpu */ +struct kvm_sbi_fwft { + struct kvm_sbi_fwft_config *configs; +#ifndef CONFIG_32BIT + bool have_vs_pmlen_7; + bool have_vs_pmlen_16; +#endif +}; + +#define vcpu_to_fwft(vcpu) (&(vcpu)->arch.fwft_context) + +#endif /* !__KVM_VCPU_RISCV_FWFT_H */ diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 27f5bccdd8b0..57a798a4cb0d 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -33,8 +33,7 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, unsigned long *isa); void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx); void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx); -int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, - struct kvm_cpu_context *cntx); +int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu); void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu); #else @@ -62,8 +61,7 @@ static inline void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cn { } -static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, - struct kvm_cpu_context *cntx) +static inline int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) { return 0; } diff --git a/arch/riscv/include/asm/kvm_vmid.h b/arch/riscv/include/asm/kvm_vmid.h new file mode 100644 index 000000000000..db61b0525a8d --- /dev/null +++ b/arch/riscv/include/asm/kvm_vmid.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#ifndef __RISCV_KVM_VMID_H_ +#define __RISCV_KVM_VMID_H_ + +#include <linux/kvm_types.h> + +struct kvm_vmid { + /* + * Writes to vmid_version and vmid happen with vmid_lock held + * whereas reads happen without any lock held. + */ + unsigned long vmid_version; + unsigned long vmid; +}; + +void __init kvm_riscv_gstage_vmid_detect(void); +unsigned long kvm_riscv_gstage_vmid_bits(void); +int kvm_riscv_gstage_vmid_init(struct kvm *kvm); +bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid); +void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/riscv/include/asm/mman.h b/arch/riscv/include/asm/mman.h new file mode 100644 index 000000000000..0ad1d19832eb --- /dev/null +++ b/arch/riscv/include/asm/mman.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMAN_H__ +#define __ASM_MMAN_H__ + +#include <linux/compiler.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <uapi/asm/mman.h> + +static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, + unsigned long pkey __always_unused) +{ + unsigned long ret = 0; + + /* + * If PROT_WRITE was specified, force it to VM_READ | VM_WRITE. + * Only VM_WRITE means shadow stack. + */ + if (prot & PROT_WRITE) + ret = (VM_READ | VM_WRITE); + return ret; +} + +#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) + +#endif /* ! __ASM_MMAN_H__ */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index 1cc90465d75b..cf8e6eac77d5 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -7,7 +7,7 @@ #ifndef _ASM_RISCV_MMU_H #define _ASM_RISCV_MMU_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { #ifndef CONFIG_MMU @@ -40,6 +40,6 @@ typedef struct { void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, pgprot_t prot); -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_MMU_H */ diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 8c4bc49a3a0f..dbf27a78df6c 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -48,6 +48,13 @@ static inline unsigned long mm_untag_mask(struct mm_struct *mm) } #endif +#define deactivate_mm deactivate_mm +static inline void deactivate_mm(struct task_struct *tsk, + struct mm_struct *mm) +{ + shstk_release(tsk); +} + #include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 125f5ecd9565..709a36fb4323 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -24,23 +24,20 @@ * When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#ifdef CONFIG_64BIT #ifdef CONFIG_MMU -#define PAGE_OFFSET kernel_map.page_offset -#else -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) -#endif -/* - * By default, CONFIG_PAGE_OFFSET value corresponds to SV57 address space so - * define the PAGE_OFFSET value for SV48 and SV39. - */ +#ifdef CONFIG_64BIT +#define PAGE_OFFSET_L5 _AC(0xff60000000000000, UL) #define PAGE_OFFSET_L4 _AC(0xffffaf8000000000, UL) #define PAGE_OFFSET_L3 _AC(0xffffffd600000000, UL) +#define PAGE_OFFSET kernel_map.page_offset #else -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET _AC(0xc0000000, UL) #endif /* CONFIG_64BIT */ +#else +#define PAGE_OFFSET ((unsigned long)phys_ram_base) +#endif /* CONFIG_MMU */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_RISCV_ISA_ZICBOZ void clear_page(void *page); @@ -49,9 +46,7 @@ void clear_page(void *page); #endif #define copy_page(to, from) memcpy((to), (from), PAGE_SIZE) -#define clear_user_page(pgaddr, vaddr, page) clear_page(pgaddr) -#define copy_user_page(vto, vfrom, vaddr, topg) \ - memcpy((vto), (vfrom), PAGE_SIZE) +#define copy_user_page(vto, vfrom, vaddr, topg) copy_page(vto, vfrom) /* * Use struct definitions to apply C type checking @@ -95,14 +90,9 @@ typedef struct page *pgtable_t; #define MIN_MEMBLOCK_ADDR 0 #endif -#ifdef CONFIG_MMU #define ARCH_PFN_OFFSET (PFN_DOWN((unsigned long)phys_ram_base)) -#else -#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) -#endif /* CONFIG_MMU */ struct kernel_mapping { - unsigned long page_offset; unsigned long virt_addr; unsigned long virt_offset; uintptr_t phys_addr; @@ -110,14 +100,8 @@ struct kernel_mapping { /* Offset between linear mapping virtual address and kernel load address */ unsigned long va_pa_offset; /* Offset between kernel mapping virtual address and kernel load address */ -#ifdef CONFIG_XIP_KERNEL - unsigned long va_kernel_xip_text_pa_offset; - unsigned long va_kernel_xip_data_pa_offset; - uintptr_t xiprom; - uintptr_t xiprom_sz; -#else + unsigned long page_offset; unsigned long va_kernel_pa_offset; -#endif }; extern struct kernel_mapping kernel_map; @@ -136,16 +120,7 @@ extern unsigned long vmemmap_start_pfn; void *linear_mapping_pa_to_va(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_pa_to_va(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < phys_ram_base) ? \ - (void *)(_y + kernel_map.va_kernel_xip_text_pa_offset) : \ - (void *)(_y + kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_pa_to_va(y) ((void *)((unsigned long)(y) + kernel_map.va_kernel_pa_offset)) -#endif #define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x) @@ -155,16 +130,7 @@ void *linear_mapping_pa_to_va(unsigned long x); phys_addr_t linear_mapping_va_to_pa(unsigned long x); #endif -#ifdef CONFIG_XIP_KERNEL -#define kernel_mapping_va_to_pa(y) ({ \ - unsigned long _y = (unsigned long)(y); \ - (_y < kernel_map.virt_addr + kernel_map.xiprom_sz) ? \ - (_y - kernel_map.va_kernel_xip_text_pa_offset) : \ - (_y - kernel_map.va_kernel_xip_data_pa_offset); \ - }) -#else #define kernel_mapping_va_to_pa(y) ((unsigned long)(y) - kernel_map.va_kernel_pa_offset) -#endif #define __va_to_pa_nodebug(x) ({ \ unsigned long _x = x; \ @@ -195,21 +161,24 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) -unsigned long kaslr_offset(void); +static inline unsigned long kaslr_offset(void) +{ + return kernel_map.virt_offset; +} static __always_inline void *pfn_to_kaddr(unsigned long pfn) { return __va(pfn << PAGE_SHIFT); } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #define virt_addr_valid(vaddr) ({ \ unsigned long _addr = (unsigned long)vaddr; \ (unsigned long)(_addr) >= PAGE_OFFSET && pfn_valid(virt_to_pfn(_addr)); \ }) -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC +#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_NON_EXEC #include <asm-generic/memory_model.h> #include <asm-generic/getorder.h> diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h index c0abde70fc2c..c49c55b266f3 100644 --- a/arch/riscv/include/asm/paravirt.h +++ b/arch/riscv/include/asm/paravirt.h @@ -3,20 +3,6 @@ #define _ASM_RISCV_PARAVIRT_H #ifdef CONFIG_PARAVIRT -#include <linux/static_call_types.h> - -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - -u64 dummy_steal_clock(int cpu); - -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); - -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} int __init pv_time_init(void); diff --git a/arch/riscv/include/asm/paravirt_api_clock.h b/arch/riscv/include/asm/paravirt_api_clock.h deleted file mode 100644 index 65ac7cee0dad..000000000000 --- a/arch/riscv/include/asm/paravirt_api_clock.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm/paravirt.h> diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 3e2aebea6312..770ce18a7328 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -15,24 +15,6 @@ #define __HAVE_ARCH_PUD_FREE #include <asm-generic/pgalloc.h> -/* - * While riscv platforms with riscv_ipi_for_rfence as true require an IPI to - * perform TLB shootdown, some platforms with riscv_ipi_for_rfence as false use - * SBI to perform TLB shootdown. To keep software pagetable walkers safe in this - * case we switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the - * comment below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h - * for more details. - */ -static inline void riscv_tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) -{ - if (riscv_use_sbi_for_rfence()) { - tlb_remove_ptdesc(tlb, pt); - } else { - pagetable_dtor(pt); - tlb_remove_page_ptdesc(tlb, pt); - } -} - static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) { @@ -108,14 +90,14 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long addr) { if (pgtable_l4_enabled) - riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); } static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr) { if (pgtable_l5_enabled) - riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); } #endif /* __PAGETABLE_PMD_FOLDED */ @@ -143,7 +125,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr) { - riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(pmd)); + tlb_remove_ptdesc(tlb, virt_to_ptdesc(pmd)); } #endif /* __PAGETABLE_PMD_FOLDED */ @@ -151,7 +133,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - riscv_tlb_remove_ptdesc(tlb, page_ptdesc(pte)); + tlb_remove_ptdesc(tlb, page_ptdesc(pte)); } #endif /* CONFIG_MMU */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 0897dd99ab8d..6e789fa58514 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -69,6 +69,8 @@ typedef struct { #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) +#define MAX_POSSIBLE_PHYSMEM_BITS 56 + /* * rv64 PTE format: * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 @@ -184,7 +186,7 @@ static inline int pud_none(pud_t pud) static inline int pud_bad(pud_t pud) { - return !pud_present(pud); + return !pud_present(pud) || (pud_val(pud) & _PAGE_LEAF); } #define pud_leaf pud_leaf @@ -262,8 +264,6 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) return __page_val_to_pfn(pmd_val(pmd)); } -#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) - #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) @@ -399,23 +399,8 @@ static inline struct page *pgd_page(pgd_t pgd) p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); #ifdef CONFIG_TRANSPARENT_HUGEPAGE -static inline int pte_devmap(pte_t pte); static inline pte_t pmd_pte(pmd_t pmd); - -static inline int pmd_devmap(pmd_t pmd) -{ - return pte_devmap(pmd_pte(pmd)); -} - -static inline int pud_devmap(pud_t pud) -{ - return 0; -} - -static inline int pgd_devmap(pgd_t pgd) -{ - return 0; -} +static inline pte_t pud_pte(pud_t pud); #endif #endif /* _ASM_RISCV_PGTABLE_64_H */ diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index a8f5205cea54..b422d9691e60 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -19,7 +19,43 @@ #define _PAGE_SOFT (3 << 8) /* Reserved for software */ #define _PAGE_SPECIAL (1 << 8) /* RSW: 0x1 */ -#define _PAGE_DEVMAP (1 << 9) /* RSW, devmap */ + +#ifdef CONFIG_MEM_SOFT_DIRTY + +/* ext_svrsw60t59b: bit 59 for soft-dirty tracking */ +#define _PAGE_SOFT_DIRTY \ + ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ + (1UL << 59) : 0) +/* + * Bit 3 is always zero for swap entry computation, so we + * can borrow it for swap page soft-dirty tracking. + */ +#define _PAGE_SWP_SOFT_DIRTY \ + ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ + _PAGE_EXEC : 0) +#else +#define _PAGE_SOFT_DIRTY 0 +#define _PAGE_SWP_SOFT_DIRTY 0 +#endif /* CONFIG_MEM_SOFT_DIRTY */ + +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP + +/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */ +#define _PAGE_UFFD_WP \ + ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ + (1UL << 60) : 0) +/* + * Bit 4 is not involved into swap entry computation, so we + * can borrow it for swap page uffd-wp tracking. + */ +#define _PAGE_SWP_UFFD_WP \ + ((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ? \ + _PAGE_USER : 0) +#else +#define _PAGE_UFFD_WP 0 +#define _PAGE_SWP_UFFD_WP 0 +#endif + #define _PAGE_TABLE _PAGE_PRESENT /* diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 050fdc49b5ad..a1a7c6520a09 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -12,7 +12,11 @@ #include <asm/pgtable-bits.h> #ifndef CONFIG_MMU -#define KERNEL_LINK_ADDR PAGE_OFFSET +#ifdef CONFIG_RELOCATABLE +#define KERNEL_LINK_ADDR UL(0) +#else +#define KERNEL_LINK_ADDR _AC(CONFIG_PHYS_RAM_BASE, UL) +#endif #define KERN_VIRT_SIZE (UL(-1)) #else @@ -107,7 +111,7 @@ #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/page.h> #include <asm/tlbflush.h> @@ -120,10 +124,6 @@ #ifdef CONFIG_64BIT #include <asm/pgtable-64.h> -#define VA_USER_SV39 (UL(1) << (VA_BITS_SV39 - 1)) -#define VA_USER_SV48 (UL(1) << (VA_BITS_SV48 - 1)) -#define VA_USER_SV57 (UL(1) << (VA_BITS_SV57 - 1)) - #define MMAP_VA_BITS_64 ((VA_BITS >= VA_BITS_SV48) ? VA_BITS_SV48 : VA_BITS) #define MMAP_MIN_VA_BITS_64 (VA_BITS_SV39) #define MMAP_VA_BITS (is_compat_task() ? VA_BITS_SV32 : MMAP_VA_BITS_64) @@ -134,21 +134,6 @@ #include <linux/page_table_check.h> -#ifdef CONFIG_XIP_KERNEL -#define XIP_FIXUP(addr) ({ \ - extern char _sdata[], _start[], _end[]; \ - uintptr_t __rom_start_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_sdata - (uintptr_t)&_start; \ - uintptr_t __rom_end_data = CONFIG_XIP_PHYS_ADDR \ - + (uintptr_t)&_end - (uintptr_t)&_start; \ - uintptr_t __a = (uintptr_t)(addr); \ - (__a >= __rom_start_data && __a < __rom_end_data) ? \ - __a - __rom_start_data + CONFIG_PHYS_RAM_BASE : __a; \ - }) -#else -#define XIP_FIXUP(addr) (addr) -#endif /* CONFIG_XIP_KERNEL */ - struct pt_alloc_ops { pte_t *(*get_pte_virt)(phys_addr_t pa); phys_addr_t (*alloc_pte)(uintptr_t va); @@ -178,6 +163,7 @@ extern struct pt_alloc_ops pt_ops __meminitdata; #define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) #define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \ _PAGE_EXEC | _PAGE_WRITE) +#define PAGE_SHADOWSTACK __pgprot(_PAGE_BASE | _PAGE_WRITE) #define PAGE_COPY PAGE_READ #define PAGE_COPY_EXEC PAGE_READ_EXEC @@ -199,6 +185,7 @@ extern struct pt_alloc_ops pt_ops __meminitdata; #define PAGE_TABLE __pgprot(_PAGE_TABLE) +#define _PAGE_KERNEL_NC ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_NOCACHE) #define _PAGE_IOREMAP ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO) #define PAGE_KERNEL_IO __pgprot(_PAGE_IOREMAP) @@ -339,7 +326,13 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val); } -#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) +#define pte_pgprot pte_pgprot +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} static inline int pte_present(pte_t pte) { @@ -399,32 +392,69 @@ static inline int pte_special(pte_t pte) return pte_val(pte) & _PAGE_SPECIAL; } -#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP -static inline int pte_devmap(pte_t pte) +/* static inline pte_t pte_rdprotect(pte_t pte) */ + +static inline pte_t pte_wrprotect(pte_t pte) { - return pte_val(pte) & _PAGE_DEVMAP; + return __pte((pte_val(pte) & ~(_PAGE_WRITE)) | (_PAGE_READ)); } -#endif -/* static inline pte_t pte_rdprotect(pte_t pte) */ +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +#define pgtable_supports_uffd_wp() \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B) -static inline pte_t pte_wrprotect(pte_t pte) +static inline bool pte_uffd_wp(pte_t pte) { - return __pte(pte_val(pte) & ~(_PAGE_WRITE)); + return !!(pte_val(pte) & _PAGE_UFFD_WP); } +static inline pte_t pte_mkuffd_wp(pte_t pte) +{ + return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP)); +} + +static inline pte_t pte_clear_uffd_wp(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP)); +} + +static inline bool pte_swp_uffd_wp(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP); +} + +static inline pte_t pte_swp_mkuffd_wp(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP); +} + +static inline pte_t pte_swp_clear_uffd_wp(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP)); +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + /* static inline pte_t pte_mkread(pte_t pte) */ +struct vm_area_struct; +pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma); +#define pte_mkwrite pte_mkwrite + static inline pte_t pte_mkwrite_novma(pte_t pte) { return __pte(pte_val(pte) | _PAGE_WRITE); } +static inline pte_t pte_mkwrite_shstk(pte_t pte) +{ + return __pte((pte_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + /* static inline pte_t pte_mkexec(pte_t pte) */ static inline pte_t pte_mkdirty(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_DIRTY); + return __pte(pte_val(pte) | _PAGE_DIRTY | _PAGE_SOFT_DIRTY); } static inline pte_t pte_mkclean(pte_t pte) @@ -447,16 +477,47 @@ static inline pte_t pte_mkspecial(pte_t pte) return __pte(pte_val(pte) | _PAGE_SPECIAL); } -static inline pte_t pte_mkdevmap(pte_t pte) +static inline pte_t pte_mkhuge(pte_t pte) { - return __pte(pte_val(pte) | _PAGE_DEVMAP); + return pte; } -static inline pte_t pte_mkhuge(pte_t pte) +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +#define pgtable_supports_soft_dirty() \ + (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && \ + riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) + +static inline bool pte_soft_dirty(pte_t pte) { - return pte; + return !!(pte_val(pte) & _PAGE_SOFT_DIRTY); +} + +static inline pte_t pte_mksoft_dirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY); +} + +static inline pte_t pte_clear_soft_dirty(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_SOFT_DIRTY)); +} + +static inline bool pte_swp_soft_dirty(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_SWP_SOFT_DIRTY); +} + +static inline pte_t pte_swp_mksoft_dirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY); } +static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_SWP_SOFT_DIRTY)); +} +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + #ifdef CONFIG_RISCV_ISA_SVNAPOT #define pte_leaf_size(pte) (pte_napot(pte) ? \ napot_cont_size(napot_cont_order(pte)) :\ @@ -497,8 +558,13 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + /* + * Svvptc guarantees that the new valid pte will be visible within + * a bounded timeframe, so when the uarch does not cache invalid + * entries, we don't have to do anything. + */ + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) + return; /* * The kernel assumes that TLBs don't cache invalid entries, but @@ -510,12 +576,6 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, while (nr--) local_flush_tlb_page(address + nr * PAGE_SIZE); -svvptc:; - /* - * Svvptc guarantees that the new valid pte will be visible within - * a bounded timeframe, so when the uarch does not cache invalid - * entries, we don't have to do anything. - */ } #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) @@ -562,7 +622,7 @@ static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval, unsigned int nr) { - page_table_check_ptes_set(mm, ptep, pteval, nr); + page_table_check_ptes_set(mm, addr, ptep, pteval, nr); for (;;) { __set_pte_at(mm, ptep, pteval); @@ -584,16 +644,22 @@ static inline void pte_clear(struct mm_struct *mm, extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty); #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG /* defined in mm/pgtable.c */ -extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep); +bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - pte_t pte = __pte(atomic_long_xchg((atomic_long_t *)ptep, 0)); +#ifdef CONFIG_SMP + pte_t pte = __pte(xchg(&ptep->pte, 0)); +#else + pte_t pte = *ptep; + + set_pte(ptep, __pte(0)); +#endif - page_table_check_pte_clear(mm, pte); + page_table_check_pte_clear(mm, address, pte); return pte; } @@ -602,12 +668,20 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep); + pte_t read_pte = READ_ONCE(*ptep); + /* + * ptep_set_wrprotect can be called for shadow stack ranges too. + * shadow stack memory is XWR = 010 and thus clearing _PAGE_WRITE will lead to + * encoding 000b which is wrong encoding with V = 1. This should lead to page fault + * but we dont want this wrong configuration to be set in page tables. + */ + atomic_long_set((atomic_long_t *)ptep, + ((pte_val(read_pte) & ~(unsigned long)_PAGE_WRITE) | _PAGE_READ)); } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -static inline int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep) +static inline bool ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { /* * This comment is borrowed from x86, but applies equally to RISC-V: @@ -655,6 +729,8 @@ static inline pgprot_t pgprot_writecombine(pgprot_t _prot) return __pgprot(prot); } +#define pgprot_dmacoherent pgprot_writecombine + /* * Both Svade and Svadu control the hardware behavior when the PTE A/D bits need to be set. By * default the M-mode firmware enables the hardware updating scheme when only Svadu is present in @@ -674,6 +750,11 @@ static inline pmd_t pte_pmd(pte_t pte) return __pmd(pte_val(pte)); } +static inline pud_t pte_pud(pte_t pte) +{ + return __pud(pte_val(pte)); +} + static inline pmd_t pmd_mkhuge(pmd_t pmd) { return pmd; @@ -699,6 +780,18 @@ static inline unsigned long pud_pfn(pud_t pud) return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT); } +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t pmd) +{ + return pte_pgprot(pmd_pte(pmd)); +} + +#define pud_pgprot pud_pgprot +static inline pgprot_t pud_pgprot(pud_t pud) +{ + return pte_pgprot(pud_pte(pud)); +} + static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); @@ -743,11 +836,19 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) return pte_pmd(pte_mkyoung(pmd_pte(pmd))); } +pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); +#define pmd_mkwrite pmd_mkwrite + static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) { return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd))); } +static inline pmd_t pmd_mkwrite_shstk(pmd_t pte) +{ + return __pmd((pmd_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE); +} + static inline pmd_t pmd_wrprotect(pmd_t pmd) { return pte_pmd(pte_wrprotect(pmd_pte(pmd))); @@ -763,37 +864,122 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pte_pmd(pte_mkdirty(pmd_pte(pmd))); } -static inline pmd_t pmd_mkdevmap(pmd_t pmd) +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP +static inline bool pmd_special(pmd_t pmd) { - return pte_pmd(pte_mkdevmap(pmd_pte(pmd))); + return pte_special(pmd_pte(pmd)); } +static inline pmd_t pmd_mkspecial(pmd_t pmd) +{ + return pte_pmd(pte_mkspecial(pmd_pte(pmd))); +} +#endif + +#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP +static inline bool pud_special(pud_t pud) +{ + return pte_special(pud_pte(pud)); +} + +static inline pud_t pud_mkspecial(pud_t pud) +{ + return pte_pud(pte_mkspecial(pud_pte(pud))); +} +#endif + +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP +static inline bool pmd_uffd_wp(pmd_t pmd) +{ + return pte_uffd_wp(pmd_pte(pmd)); +} + +static inline pmd_t pmd_mkuffd_wp(pmd_t pmd) +{ + return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd))); +} + +static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd) +{ + return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd))); +} + +static inline bool pmd_swp_uffd_wp(pmd_t pmd) +{ + return pte_swp_uffd_wp(pmd_pte(pmd)); +} + +static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd) +{ + return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd))); +} + +static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd) +{ + return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd))); +} +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +static inline bool pmd_soft_dirty(pmd_t pmd) +{ + return pte_soft_dirty(pmd_pte(pmd)); +} + +static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) +{ + return pte_pmd(pte_mksoft_dirty(pmd_pte(pmd))); +} + +static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) +{ + return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd))); +} + +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +static inline bool pmd_swp_soft_dirty(pmd_t pmd) +{ + return pte_swp_soft_dirty(pmd_pte(pmd)); +} + +static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) +{ + return pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd))); +} + +static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) +{ + return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd))); +} +#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(mm, pmdp, pmd); + page_table_check_pmd_set(mm, addr, pmdp, pmd); return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd)); } static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) { - page_table_check_pud_set(mm, pudp, pud); + page_table_check_pud_set(mm, addr, pudp, pud); return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud)); } #ifdef CONFIG_PAGE_TABLE_CHECK -static inline bool pte_user_accessible_page(pte_t pte) +static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte) { return pte_present(pte) && pte_user(pte); } -static inline bool pmd_user_accessible_page(pmd_t pmd) +static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd) { return pmd_leaf(pmd) && pmd_user(pmd); } -static inline bool pud_user_accessible_page(pud_t pud) +static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud) { return pud_leaf(pud) && pud_user(pud); } @@ -814,8 +1000,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, } #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); } @@ -824,9 +1010,15 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pmd_t *pmdp) { - pmd_t pmd = __pmd(atomic_long_xchg((atomic_long_t *)pmdp, 0)); +#ifdef CONFIG_SMP + pmd_t pmd = __pmd(xchg(&pmdp->pmd, 0)); +#else + pmd_t pmd = *pmdp; + + pmd_clear(pmdp); +#endif - page_table_check_pmd_clear(mm, pmd); + page_table_check_pmd_clear(mm, address, pmd); return pmd; } @@ -842,13 +1034,122 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { - page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd); return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); } #define pmdp_collapse_flush pmdp_collapse_flush extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); + +static inline pud_t pud_wrprotect(pud_t pud) +{ + return pte_pud(pte_wrprotect(pud_pte(pud))); +} + +static inline int pud_trans_huge(pud_t pud) +{ + return pud_leaf(pud); +} + +static inline int pud_dirty(pud_t pud) +{ + return pte_dirty(pud_pte(pud)); +} + +static inline pud_t pud_mkyoung(pud_t pud) +{ + return pte_pud(pte_mkyoung(pud_pte(pud))); +} + +static inline pud_t pud_mkold(pud_t pud) +{ + return pte_pud(pte_mkold(pud_pte(pud))); +} + +static inline pud_t pud_mkdirty(pud_t pud) +{ + return pte_pud(pte_mkdirty(pud_pte(pud))); +} + +static inline pud_t pud_mkclean(pud_t pud) +{ + return pte_pud(pte_mkclean(pud_pte(pud))); +} + +static inline pud_t pud_mkwrite(pud_t pud) +{ + return pte_pud(pte_mkwrite_novma(pud_pte(pud))); +} + +static inline pud_t pud_mkhuge(pud_t pud) +{ + return pud; +} + +static inline int pudp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp, + pud_t entry, int dirty) +{ + return ptep_set_access_flags(vma, address, (pte_t *)pudp, pud_pte(entry), dirty); +} + +static inline bool pudp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp) +{ + return ptep_test_and_clear_young(vma, address, (pte_t *)pudp); +} + +#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR +static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pud_t *pudp) +{ +#ifdef CONFIG_SMP + pud_t pud = __pud(xchg(&pudp->pud, 0)); +#else + pud_t pud = *pudp; + + pud_clear(pudp); +#endif + + page_table_check_pud_clear(mm, address, pud); + + return pud; +} + +static inline int pud_young(pud_t pud) +{ + return pte_young(pud_pte(pud)); +} + +static inline void update_mmu_cache_pud(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp) +{ + pte_t *ptep = (pte_t *)pudp; + + update_mmu_cache(vma, address, ptep); +} + +static inline pud_t pudp_establish(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp, pud_t pud) +{ + page_table_check_pud_set(vma->vm_mm, address, pudp, pud); + return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud))); +} + +static inline pud_t pud_mkinvalid(pud_t pud) +{ + return __pud(pud_val(pud) & ~(_PAGE_PRESENT | _PAGE_PROT_NONE)); +} + +extern pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, + pud_t *pudp); + +static inline pud_t pud_modify(pud_t pud, pgprot_t newprot) +{ + return pte_pud(pte_modify(pud_pte(pud), newprot)); +} + #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ /* @@ -857,7 +1158,9 @@ extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, * * Format of swap PTE: * bit 0: _PAGE_PRESENT (zero) - * bit 1 to 3: _PAGE_LEAF (zero) + * bit 1 to 2: (zero) + * bit 3: _PAGE_SWP_SOFT_DIRTY + * bit 4: _PAGE_SWP_UFFD_WP * bit 5: _PAGE_PROT_NONE (zero) * bit 6: exclusive marker * bits 7 to 11: swap type @@ -880,7 +1183,7 @@ extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) -static inline int pte_swp_exclusive(pte_t pte) +static inline bool pte_swp_exclusive(pte_t pte) { return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; } @@ -927,7 +1230,6 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) */ #ifdef CONFIG_64BIT #define TASK_SIZE_64 (PGDIR_SIZE * PTRS_PER_PGD / 2) -#define TASK_SIZE_MAX LONG_MAX #ifdef CONFIG_COMPAT #define TASK_SIZE_32 (_AC(0x80000000, UL) - PAGE_SIZE) @@ -955,26 +1257,14 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) extern char _start[]; extern void *_dtb_early_va; extern uintptr_t _dtb_early_pa; -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_MMU) -#define dtb_early_va (*(void **)XIP_FIXUP(&_dtb_early_va)) -#define dtb_early_pa (*(uintptr_t *)XIP_FIXUP(&_dtb_early_pa)) -#else #define dtb_early_va _dtb_early_va #define dtb_early_pa _dtb_early_pa -#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; void paging_init(void); void misc_mem_init(void); /* - * ZERO_PAGE is a global shared page that is always zero, - * used for zero-mapped memory areas, etc. - */ -extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) - -/* * Use set_p*_safe(), and elide TLB flushing, when confident that *no* * TLB flush will be required as a result of the "set". For example, use * in scenarios where it is known ahead of time that the routine is @@ -993,6 +1283,6 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ set_pgd(pgdp, pgd); \ }) -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _ASM_RISCV_PGTABLE_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 5f56eb9d114a..812517b2cec1 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -13,6 +13,10 @@ #include <vdso/processor.h> #include <asm/ptrace.h> +#include <asm/insn-def.h> +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> +#include <asm/usercfi.h> #define arch_get_mmap_end(addr, len, flags) \ ({ \ @@ -51,8 +55,7 @@ #define TASK_UNMAPPED_BASE PAGE_ALIGN(TASK_SIZE / 3) #endif -#ifndef __ASSEMBLY__ -#include <linux/cpumask.h> +#ifndef __ASSEMBLER__ struct task_struct; struct pt_regs; @@ -79,7 +82,11 @@ struct pt_regs; * Thus, the task does not own preempt_v. Any use of Vector will have to * save preempt_v, if dirty, and fallback to non-preemptible kernel-mode * Vector. - * - bit 30: The in-kernel preempt_v context is saved, and requries to be + * - bit 29: The thread voluntarily calls schedule() while holding an active + * preempt_v. All preempt_v context should be dropped in such case because + * V-regs are caller-saved. Only sstatus.VS=ON is persisted across a + * schedule() call. + * - bit 30: The in-kernel preempt_v context is saved, and is required to be * restored when returning to the context that owns the preempt_v. * - bit 31: The in-kernel preempt_v context is dirty, as signaled by the * trap entry code. Any context switches out-of current task need to save @@ -93,6 +100,7 @@ struct pt_regs; #define RISCV_PREEMPT_V 0x00000100 #define RISCV_PREEMPT_V_DIRTY 0x80000000 #define RISCV_PREEMPT_V_NEED_RESTORE 0x40000000 +#define RISCV_PREEMPT_V_IN_SCHEDULE 0x20000000 /* CPU-specific state of a task */ struct thread_struct { @@ -103,6 +111,7 @@ struct thread_struct { struct __riscv_d_ext_state fstate; unsigned long bad_cause; unsigned long envcfg; + unsigned long sum; u32 riscv_v_flags; u32 vstate_ctrl; struct __riscv_v_ext_state vstate; @@ -136,6 +145,27 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset, #define KSTK_EIP(tsk) (task_pt_regs(tsk)->epc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->sp) +#define PREFETCH_ASM(x) \ + ALTERNATIVE(__nops(1), PREFETCH_R(x, 0), 0, \ + RISCV_ISA_EXT_ZICBOP, CONFIG_RISCV_ISA_ZICBOP) + +#define PREFETCHW_ASM(x) \ + ALTERNATIVE(__nops(1), PREFETCH_W(x, 0), 0, \ + RISCV_ISA_EXT_ZICBOP, CONFIG_RISCV_ISA_ZICBOP) + +#ifdef CONFIG_RISCV_ISA_ZICBOP +#define ARCH_HAS_PREFETCH +static inline void prefetch(const void *x) +{ + __asm__ __volatile__(PREFETCH_ASM(%0) : : "r" (x) : "memory"); +} + +#define ARCH_HAS_PREFETCHW +static inline void prefetchw(const void *x) +{ + __asm__ __volatile__(PREFETCHW_ASM(%0) : : "r" (x) : "memory"); +} +#endif /* CONFIG_RISCV_ISA_ZICBOP */ /* Do necessary setup to start up a newly executed thread. */ extern void start_thread(struct pt_regs *regs, @@ -186,6 +216,6 @@ long get_tagged_addr_ctrl(struct task_struct *task); #define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current) #endif -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index b5b0adcc85c1..addc8188152f 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -10,7 +10,7 @@ #include <asm/csr.h> #include <linux/compiler.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct pt_regs { unsigned long epc; @@ -23,14 +23,16 @@ struct pt_regs { unsigned long t2; unsigned long s0; unsigned long s1; - unsigned long a0; - unsigned long a1; - unsigned long a2; - unsigned long a3; - unsigned long a4; - unsigned long a5; - unsigned long a6; - unsigned long a7; + struct_group(a_regs, + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + ); unsigned long s2; unsigned long s3; unsigned long s4; @@ -173,11 +175,11 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, return 0; } -static inline int regs_irqs_disabled(struct pt_regs *regs) +static __always_inline bool regs_irqs_disabled(struct pt_regs *regs) { return !(regs->status & SR_PIE); } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/asm/runtime-const.h b/arch/riscv/include/asm/runtime-const.h new file mode 100644 index 000000000000..900db0a103d0 --- /dev/null +++ b/arch/riscv/include/asm/runtime-const.h @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_RUNTIME_CONST_H +#define _ASM_RISCV_RUNTIME_CONST_H + +#ifdef MODULE + #error "Cannot use runtime-const infrastructure from modules" +#endif + +#include <asm/asm.h> +#include <asm/alternative.h> +#include <asm/cacheflush.h> +#include <asm/insn-def.h> +#include <linux/memory.h> +#include <asm/text-patching.h> + +#include <linux/uaccess.h> + +#ifdef CONFIG_32BIT +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + "lui %[__ret],0x89abd\n\t" \ + "addi %[__ret],%[__ret],-0x211\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret)); \ + __ret; \ +}) +#else +/* + * Loading 64-bit constants into a register from immediates is a non-trivial + * task on riscv64. To get it somewhat performant, load 32 bits into two + * different registers and then combine the results. + * + * If the processor supports the Zbkb extension, we can combine the final + * "slli,slli,srli,add" into the single "pack" instruction. If the processor + * doesn't support Zbkb but does support the Zbb extension, we can + * combine the final "slli,srli,add" into one instruction "add.uw". + */ +#define RISCV_RUNTIME_CONST_64_PREAMBLE \ + ".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + "lui %[__ret],0x89abd\n\t" \ + "lui %[__tmp],0x1234\n\t" \ + "addiw %[__ret],%[__ret],-0x211\n\t" \ + "addiw %[__tmp],%[__tmp],0x567\n\t" \ + +#define RISCV_RUNTIME_CONST_64_BASE \ + "slli %[__tmp],%[__tmp],32\n\t" \ + "slli %[__ret],%[__ret],32\n\t" \ + "srli %[__ret],%[__ret],32\n\t" \ + "add %[__ret],%[__ret],%[__tmp]\n\t" \ + +#define RISCV_RUNTIME_CONST_64_ZBA \ + ".option push\n\t" \ + ".option arch,+zba\n\t" \ + ".option norvc\n\t" \ + "slli %[__tmp],%[__tmp],32\n\t" \ + "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".option pop\n\t" \ + +#define RISCV_RUNTIME_CONST_64_ZBKB \ + ".option push\n\t" \ + ".option arch,+zbkb\n\t" \ + ".option norvc\n\t" \ + "pack %[__ret],%[__ret],%[__tmp]\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".option pop\n\t" \ + +#define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + ".option pop\n\t" \ + ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + +#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) \ + && defined(CONFIG_RISCV_ISA_ZBKB) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE_2( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBA, \ + 0, RISCV_ISA_EXT_ZBA, 1, \ + RISCV_RUNTIME_CONST_64_ZBKB, \ + 0, RISCV_ISA_EXT_ZBKB, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#elif defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBA, \ + 0, RISCV_ISA_EXT_ZBA, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#elif defined(CONFIG_RISCV_ISA_ZBKB) +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + ALTERNATIVE( \ + RISCV_RUNTIME_CONST_64_BASE, \ + RISCV_RUNTIME_CONST_64_ZBKB, \ + 0, RISCV_ISA_EXT_ZBKB, 1 \ + ) \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#else +#define runtime_const_ptr(sym) \ +({ \ + typeof(sym) __ret, __tmp; \ + asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \ + RISCV_RUNTIME_CONST_64_BASE \ + RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \ + : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \ + __ret; \ +}) +#endif +#endif + +#define runtime_const_shift_right_32(val, sym) \ +({ \ + u32 __ret; \ + asm_inline(".option push\n\t" \ + ".option norvc\n\t" \ + "1:\t" \ + SRLI " %[__ret],%[__val],12\n\t" \ + ".option pop\n\t" \ + ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \ + ".long 1b - .\n\t" \ + ".popsection" \ + : [__ret] "=r" (__ret) \ + : [__val] "r" (val)); \ + __ret; \ +}) + +#define runtime_const_init(type, sym) do { \ + extern s32 __start_runtime_##type##_##sym[]; \ + extern s32 __stop_runtime_##type##_##sym[]; \ + \ + runtime_const_fixup(__runtime_fixup_##type, \ + (unsigned long)(sym), \ + __start_runtime_##type##_##sym, \ + __stop_runtime_##type##_##sym); \ +} while (0) + +static inline void __runtime_fixup_caches(void *where, unsigned int insns) +{ + /* On riscv there are currently only cache-wide flushes so va is ignored. */ + __always_unused uintptr_t va = (uintptr_t)where; + + flush_icache_range(va, va + 4 * insns); +} + +/* + * The 32-bit immediate is stored in a lui+addi pairing. + * lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction. + * addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction. + */ +static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val) +{ + unsigned int lower_immediate, upper_immediate; + u32 lui_insn, addi_insn, addi_insn_mask; + __le32 lui_res, addi_res; + + /* Mask out upper 12 bit of addi */ + addi_insn_mask = 0x000fffff; + + lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16; + addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16; + + lower_immediate = sign_extend32(val, 11); + upper_immediate = (val - lower_immediate); + + if (upper_immediate & 0xfffff000) { + /* replace upper 20 bits of lui with upper immediate */ + lui_insn &= 0x00000fff; + lui_insn |= upper_immediate & 0xfffff000; + } else { + /* replace lui with nop if immediate is small enough to fit in addi */ + lui_insn = RISCV_INSN_NOP4; + /* + * lui is being skipped, so do a load instead of an add. A load + * is performed by adding with the x0 register. Setting rs to + * zero with the following mask will accomplish this goal. + */ + addi_insn_mask &= 0x07fff; + } + + if (lower_immediate & 0x00000fff || lui_insn == RISCV_INSN_NOP4) { + /* replace upper 12 bits of addi with lower 12 bits of val */ + addi_insn &= addi_insn_mask; + addi_insn |= (lower_immediate & 0x00000fff) << 20; + } else { + /* replace addi with nop if lower_immediate is empty */ + addi_insn = RISCV_INSN_NOP4; + } + + addi_res = cpu_to_le32(addi_insn); + lui_res = cpu_to_le32(lui_insn); + mutex_lock(&text_mutex); + patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res)); + patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res)); + mutex_unlock(&text_mutex); +} + +static inline void __runtime_fixup_ptr(void *where, unsigned long val) +{ +#ifdef CONFIG_32BIT + __runtime_fixup_32(where, where + 4, val); + __runtime_fixup_caches(where, 2); +#else + __runtime_fixup_32(where, where + 8, val); + __runtime_fixup_32(where + 4, where + 12, val >> 32); + __runtime_fixup_caches(where, 4); +#endif +} + +/* + * Replace the least significant 5 bits of the srli/srliw immediate that is + * located at bits 20-24 + */ +static inline void __runtime_fixup_shift(void *where, unsigned long val) +{ + __le16 *parcel = where; + __le32 res; + u32 insn; + + insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16; + + insn &= 0xfe0fffff; + insn |= (val & 0b11111) << 20; + + res = cpu_to_le32(insn); + mutex_lock(&text_mutex); + patch_text_nosync(where, &res, sizeof(insn)); + mutex_unlock(&text_mutex); +} + +static inline void runtime_const_fixup(void (*fn)(void *, unsigned long), + unsigned long val, s32 *start, s32 *end) +{ + while (start < end) { + fn(*start + (void *)start, val); + start++; + } +} + +#endif /* _ASM_RISCV_RUNTIME_CONST_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 3d250824178b..5725e0ca4dda 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -35,6 +35,9 @@ enum sbi_ext_id { SBI_EXT_DBCN = 0x4442434E, SBI_EXT_STA = 0x535441, SBI_EXT_NACL = 0x4E41434C, + SBI_EXT_FWFT = 0x46574654, + SBI_EXT_MPXY = 0x4D505859, + SBI_EXT_DBTR = 0x44425452, /* Experimentals extensions must lie within this range */ SBI_EXT_EXPERIMENTAL_START = 0x08000000, @@ -135,6 +138,7 @@ enum sbi_ext_pmu_fid { SBI_EXT_PMU_COUNTER_FW_READ, SBI_EXT_PMU_COUNTER_FW_READ_HI, SBI_EXT_PMU_SNAPSHOT_SET_SHMEM, + SBI_EXT_PMU_EVENT_GET_INFO, }; union sbi_pmu_ctr_info { @@ -158,9 +162,20 @@ struct riscv_pmu_snapshot_data { u64 reserved[447]; }; +struct riscv_pmu_event_info { + u32 event_idx; + u32 output; + u64 event_data; +}; + +#define RISCV_PMU_EVENT_INFO_OUTPUT_MASK 0x01 + #define RISCV_PMU_RAW_EVENT_MASK GENMASK_ULL(47, 0) #define RISCV_PMU_PLAT_FW_EVENT_MASK GENMASK_ULL(61, 0) +/* SBI v3.0 allows extended hpmeventX width value */ +#define RISCV_PMU_RAW_EVENT_V2_MASK GENMASK_ULL(55, 0) #define RISCV_PMU_RAW_EVENT_IDX 0x20000 +#define RISCV_PMU_RAW_EVENT_V2_IDX 0x30000 #define RISCV_PLAT_FW_EVENT 0xFFFF /** General pmu event codes specified in SBI PMU extension */ @@ -218,6 +233,7 @@ enum sbi_pmu_event_type { SBI_PMU_EVENT_TYPE_HW = 0x0, SBI_PMU_EVENT_TYPE_CACHE = 0x1, SBI_PMU_EVENT_TYPE_RAW = 0x2, + SBI_PMU_EVENT_TYPE_RAW_V2 = 0x3, SBI_PMU_EVENT_TYPE_FW = 0xf, }; @@ -402,6 +418,122 @@ enum sbi_ext_nacl_feature { #define SBI_NACL_SHMEM_SRET_X(__i) ((__riscv_xlen / 8) * (__i)) #define SBI_NACL_SHMEM_SRET_X_LAST 31 +/* SBI function IDs for FW feature extension */ +#define SBI_EXT_FWFT_SET 0x0 +#define SBI_EXT_FWFT_GET 0x1 + +enum sbi_fwft_feature_t { + SBI_FWFT_MISALIGNED_EXC_DELEG = 0x0, + SBI_FWFT_LANDING_PAD = 0x1, + SBI_FWFT_SHADOW_STACK = 0x2, + SBI_FWFT_DOUBLE_TRAP = 0x3, + SBI_FWFT_PTE_AD_HW_UPDATING = 0x4, + SBI_FWFT_POINTER_MASKING_PMLEN = 0x5, + SBI_FWFT_LOCAL_RESERVED_START = 0x6, + SBI_FWFT_LOCAL_RESERVED_END = 0x3fffffff, + SBI_FWFT_LOCAL_PLATFORM_START = 0x40000000, + SBI_FWFT_LOCAL_PLATFORM_END = 0x7fffffff, + + SBI_FWFT_GLOBAL_RESERVED_START = 0x80000000, + SBI_FWFT_GLOBAL_RESERVED_END = 0xbfffffff, + SBI_FWFT_GLOBAL_PLATFORM_START = 0xc0000000, + SBI_FWFT_GLOBAL_PLATFORM_END = 0xffffffff, +}; + +#define SBI_FWFT_PLATFORM_FEATURE_BIT BIT(30) +#define SBI_FWFT_GLOBAL_FEATURE_BIT BIT(31) + +#define SBI_FWFT_SET_FLAG_LOCK BIT(0) + +enum sbi_ext_mpxy_fid { + SBI_EXT_MPXY_GET_SHMEM_SIZE, + SBI_EXT_MPXY_SET_SHMEM, + SBI_EXT_MPXY_GET_CHANNEL_IDS, + SBI_EXT_MPXY_READ_ATTRS, + SBI_EXT_MPXY_WRITE_ATTRS, + SBI_EXT_MPXY_SEND_MSG_WITH_RESP, + SBI_EXT_MPXY_SEND_MSG_WITHOUT_RESP, + SBI_EXT_MPXY_GET_NOTIFICATION_EVENTS, +}; + +enum sbi_mpxy_attribute_id { + /* Standard channel attributes managed by MPXY framework */ + SBI_MPXY_ATTR_MSG_PROT_ID = 0x00000000, + SBI_MPXY_ATTR_MSG_PROT_VER = 0x00000001, + SBI_MPXY_ATTR_MSG_MAX_LEN = 0x00000002, + SBI_MPXY_ATTR_MSG_SEND_TIMEOUT = 0x00000003, + SBI_MPXY_ATTR_MSG_COMPLETION_TIMEOUT = 0x00000004, + SBI_MPXY_ATTR_CHANNEL_CAPABILITY = 0x00000005, + SBI_MPXY_ATTR_SSE_EVENT_ID = 0x00000006, + SBI_MPXY_ATTR_MSI_CONTROL = 0x00000007, + SBI_MPXY_ATTR_MSI_ADDR_LO = 0x00000008, + SBI_MPXY_ATTR_MSI_ADDR_HI = 0x00000009, + SBI_MPXY_ATTR_MSI_DATA = 0x0000000A, + SBI_MPXY_ATTR_EVENTS_STATE_CONTROL = 0x0000000B, + SBI_MPXY_ATTR_STD_ATTR_MAX_IDX, + /* + * Message protocol specific attributes, managed by + * the message protocol specification. + */ + SBI_MPXY_ATTR_MSGPROTO_ATTR_START = 0x80000000, + SBI_MPXY_ATTR_MSGPROTO_ATTR_END = 0xffffffff +}; + +/* Possible values of MSG_PROT_ID attribute as-per SBI v3.0 (or higher) */ +enum sbi_mpxy_msgproto_id { + SBI_MPXY_MSGPROTO_RPMI_ID = 0x0, +}; + +/* RPMI message protocol specific MPXY attributes */ +enum sbi_mpxy_rpmi_attribute_id { + SBI_MPXY_RPMI_ATTR_SERVICEGROUP_ID = SBI_MPXY_ATTR_MSGPROTO_ATTR_START, + SBI_MPXY_RPMI_ATTR_SERVICEGROUP_VERSION, + SBI_MPXY_RPMI_ATTR_IMPL_ID, + SBI_MPXY_RPMI_ATTR_IMPL_VERSION, + SBI_MPXY_RPMI_ATTR_MAX_ID +}; + +/* Encoding of MSG_PROT_VER attribute */ +#define SBI_MPXY_MSG_PROT_VER_MAJOR(__ver) upper_16_bits(__ver) +#define SBI_MPXY_MSG_PROT_VER_MINOR(__ver) lower_16_bits(__ver) +#define SBI_MPXY_MSG_PROT_MKVER(__maj, __min) (((u32)(__maj) << 16) | (u16)(__min)) + +/* Capabilities available through CHANNEL_CAPABILITY attribute */ +#define SBI_MPXY_CHAN_CAP_MSI BIT(0) +#define SBI_MPXY_CHAN_CAP_SSE BIT(1) +#define SBI_MPXY_CHAN_CAP_EVENTS_STATE BIT(2) +#define SBI_MPXY_CHAN_CAP_SEND_WITH_RESP BIT(3) +#define SBI_MPXY_CHAN_CAP_SEND_WITHOUT_RESP BIT(4) +#define SBI_MPXY_CHAN_CAP_GET_NOTIFICATIONS BIT(5) + +/* SBI debug triggers function IDs */ +enum sbi_ext_dbtr_fid { + SBI_EXT_DBTR_NUM_TRIGGERS = 0, + SBI_EXT_DBTR_SETUP_SHMEM, + SBI_EXT_DBTR_TRIG_READ, + SBI_EXT_DBTR_TRIG_INSTALL, + SBI_EXT_DBTR_TRIG_UPDATE, + SBI_EXT_DBTR_TRIG_UNINSTALL, + SBI_EXT_DBTR_TRIG_ENABLE, + SBI_EXT_DBTR_TRIG_DISABLE, +}; + +struct sbi_dbtr_data_msg { + unsigned long tstate; + unsigned long tdata1; + unsigned long tdata2; + unsigned long tdata3; +}; + +struct sbi_dbtr_id_msg { + unsigned long idx; +}; + +union sbi_dbtr_shmem_entry { + struct sbi_dbtr_data_msg data; + struct sbi_dbtr_id_msg id; +}; + /* SBI spec version fields */ #define SBI_SPEC_VERSION_DEFAULT 0x1 #define SBI_SPEC_VERSION_MAJOR_SHIFT 24 @@ -419,6 +551,11 @@ enum sbi_ext_nacl_feature { #define SBI_ERR_ALREADY_STARTED -7 #define SBI_ERR_ALREADY_STOPPED -8 #define SBI_ERR_NO_SHMEM -9 +#define SBI_ERR_INVALID_STATE -10 +#define SBI_ERR_BAD_RANGE -11 +#define SBI_ERR_TIMEOUT -12 +#define SBI_ERR_IO -13 +#define SBI_ERR_DENIED_LOCKED -14 extern unsigned long sbi_spec_version; struct sbiret { @@ -470,6 +607,23 @@ int sbi_remote_hfence_vvma_asid(const struct cpumask *cpu_mask, unsigned long asid); long sbi_probe_extension(int ext); +int sbi_fwft_set(u32 feature, unsigned long value, unsigned long flags); +int sbi_fwft_set_cpumask(const cpumask_t *mask, u32 feature, + unsigned long value, unsigned long flags); +/** + * sbi_fwft_set_online_cpus() - Set a feature on all online cpus + * @feature: The feature to be set + * @value: The feature value to be set + * @flags: FWFT feature set flags + * + * Return: 0 on success, appropriate linux error code otherwise. + */ +static inline int sbi_fwft_set_online_cpus(u32 feature, unsigned long value, + unsigned long flags) +{ + return sbi_fwft_set_cpumask(cpu_online_mask, feature, value, flags); +} + /* Check if current SBI specification version is 0.1 or not */ static inline int sbi_spec_is_0_1(void) { @@ -503,11 +657,21 @@ static inline int sbi_err_map_linux_errno(int err) case SBI_SUCCESS: return 0; case SBI_ERR_DENIED: + case SBI_ERR_DENIED_LOCKED: return -EPERM; case SBI_ERR_INVALID_PARAM: + case SBI_ERR_INVALID_STATE: return -EINVAL; + case SBI_ERR_BAD_RANGE: + return -ERANGE; case SBI_ERR_INVALID_ADDRESS: return -EFAULT; + case SBI_ERR_NO_SHMEM: + return -ENOMEM; + case SBI_ERR_TIMEOUT: + return -ETIMEDOUT; + case SBI_ERR_IO: + return -EIO; case SBI_ERR_NOT_SUPPORTED: case SBI_ERR_FAILURE: default: diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 0e45db78b24b..023a412fe38d 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -2,7 +2,7 @@ #ifndef _ASM_SCS_H #define _ASM_SCS_H -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #include <asm/asm-offsets.h> #ifdef CONFIG_SHADOW_CALL_STACK @@ -10,7 +10,6 @@ /* Load init_shadow_call_stack to gp. */ .macro scs_load_init_stack la gp, init_shadow_call_stack - XIP_FIXUP_OFFSET gp .endm /* Load the per-CPU IRQ shadow call stack to gp. */ @@ -49,6 +48,6 @@ _skip_scs: .endm #endif /* CONFIG_SHADOW_CALL_STACK */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _ASM_SCS_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ea263d3683ef..ef59e1716a2c 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -6,7 +6,7 @@ #ifndef _ASM_RISCV_SET_MEMORY_H #define _ASM_RISCV_SET_MEMORY_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * Functions to change memory attributes. */ @@ -45,9 +45,9 @@ int set_direct_map_default_noflush(struct page *page); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ -#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_STRICT_KERNEL_RWX) #ifdef CONFIG_64BIT #define SECTION_ALIGN (1 << 21) #else diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 7ac80e9f2288..0ecc67641b09 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -105,7 +105,7 @@ static inline void riscv_ipi_set_virq_range(int virq, int nr) #endif /* CONFIG_SMP */ -#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) +#if defined(CONFIG_HOTPLUG_CPU) bool cpu_has_hotplug(unsigned int cpu); #else static inline bool cpu_has_hotplug(unsigned int cpu) diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 5ba77f60bf0b..764ffe8f6479 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -28,6 +28,15 @@ extern asmlinkage __kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP extern asmlinkage int strncmp(const char *cs, const char *ct, size_t count); + +#define __HAVE_ARCH_STRNLEN +extern asmlinkage __kernel_size_t strnlen(const char *, size_t); + +#define __HAVE_ARCH_STRCHR +extern asmlinkage char *strchr(const char *, int); + +#define __HAVE_ARCH_STRRCHR +extern asmlinkage char *strrchr(const char *, int); #endif /* For those files which don't want to check by kasan. */ diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h index 4ffb022b097f..dc5782b5fbad 100644 --- a/arch/riscv/include/asm/suspend.h +++ b/arch/riscv/include/asm/suspend.h @@ -18,6 +18,10 @@ struct suspend_context { unsigned long ie; #ifdef CONFIG_MMU unsigned long satp; + unsigned long stimecmp; +#if __riscv_xlen < 64 + unsigned long stimecmph; +#endif #endif }; diff --git a/arch/riscv/include/asm/swab.h b/arch/riscv/include/asm/swab.h new file mode 100644 index 000000000000..c1da22aa1326 --- /dev/null +++ b/arch/riscv/include/asm/swab.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_RISCV_SWAB_H +#define _ASM_RISCV_SWAB_H + +#include <linux/types.h> +#include <linux/compiler.h> +#include <asm/cpufeature-macros.h> +#include <asm/hwcap.h> +#include <asm-generic/swab.h> + +#if defined(CONFIG_TOOLCHAIN_HAS_ZBB) && defined(CONFIG_RISCV_ISA_ZBB) && !defined(NO_ALTERNATIVE) + +// Duplicated from include/uapi/linux/swab.h +#define ___constant_swab16(x) ((__u16)( \ + (((__u16)(x) & (__u16)0x00ffU) << 8) | \ + (((__u16)(x) & (__u16)0xff00U) >> 8))) + +#define ___constant_swab32(x) ((__u32)( \ + (((__u32)(x) & (__u32)0x000000ffUL) << 24) | \ + (((__u32)(x) & (__u32)0x0000ff00UL) << 8) | \ + (((__u32)(x) & (__u32)0x00ff0000UL) >> 8) | \ + (((__u32)(x) & (__u32)0xff000000UL) >> 24))) + +#define ___constant_swab64(x) ((__u64)( \ + (((__u64)(x) & (__u64)0x00000000000000ffULL) << 56) | \ + (((__u64)(x) & (__u64)0x000000000000ff00ULL) << 40) | \ + (((__u64)(x) & (__u64)0x0000000000ff0000ULL) << 24) | \ + (((__u64)(x) & (__u64)0x00000000ff000000ULL) << 8) | \ + (((__u64)(x) & (__u64)0x000000ff00000000ULL) >> 8) | \ + (((__u64)(x) & (__u64)0x0000ff0000000000ULL) >> 24) | \ + (((__u64)(x) & (__u64)0x00ff000000000000ULL) >> 40) | \ + (((__u64)(x) & (__u64)0xff00000000000000ULL) >> 56))) + +#define ARCH_SWAB(size, value) \ +({ \ + unsigned long x = value; \ + \ + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { \ + asm volatile (".option push\n" \ + ".option arch,+zbb\n" \ + "rev8 %0, %1\n" \ + ".option pop\n" \ + : "=r" (x) : "r" (x)); \ + x = x >> (BITS_PER_LONG - size); \ + } else { \ + x = ___constant_swab##size(value); \ + } \ + x; \ +}) + +static __always_inline __u16 __arch_swab16(__u16 value) +{ + return ARCH_SWAB(16, value); +} + +static __always_inline __u32 __arch_swab32(__u32 value) +{ + return ARCH_SWAB(32, value); +} + +#ifdef CONFIG_64BIT +static __always_inline __u64 __arch_swab64(__u64 value) +{ + return ARCH_SWAB(64, value); +} +#else +static __always_inline __u64 __arch_swab64(__u64 value) +{ + __u32 h = value >> 32; + __u32 l = value & ((1ULL << 32) - 1); + + return ((__u64)(__arch_swab32(l)) << 32) | ((__u64)(__arch_swab32(h))); +} +#endif + +#define __arch_swab64 __arch_swab64 +#define __arch_swab32 __arch_swab32 +#define __arch_swab16 __arch_swab16 + +#undef ___constant_swab16 +#undef ___constant_swab32 +#undef ___constant_swab64 + +#undef ARCH_SWAB + +#endif /* defined(CONFIG_TOOLCHAIN_HAS_ZBB) && defined(CONFIG_RISCV_ISA_ZBB) && !defined(NO_ALTERNATIVE) */ +#endif /* _ASM_RISCV_SWAB_H */ diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h index 121fff429dce..8067e666a4ca 100644 --- a/arch/riscv/include/asm/syscall.h +++ b/arch/riscv/include/asm/syscall.h @@ -20,7 +20,7 @@ extern void * const sys_call_table[]; extern void * const compat_sys_call_table[]; /* - * Only the low 32 bits of orig_r0 are meaningful, so we return int. + * Only the low 32 bits of orig_a0 are meaningful, so we return int. * This importantly ignores the high bits on 64-bit, so comparisons * sign-extend the low 32 bits. */ @@ -30,6 +30,13 @@ static inline int syscall_get_nr(struct task_struct *task, return regs->a7; } +static inline void syscall_set_nr(struct task_struct *task, + struct pt_regs *regs, + int nr) +{ + regs->a7 = nr; +} + static inline void syscall_rollback(struct task_struct *task, struct pt_regs *regs) { @@ -62,8 +69,23 @@ static inline void syscall_get_arguments(struct task_struct *task, unsigned long *args) { args[0] = regs->orig_a0; - args++; - memcpy(args, ®s->a1, 5 * sizeof(args[0])); + args[1] = regs->a1; + args[2] = regs->a2; + args[3] = regs->a3; + args[4] = regs->a4; + args[5] = regs->a5; +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + regs->orig_a0 = args[0]; + regs->a1 = args[1]; + regs->a2 = args[2]; + regs->a3 = args[3]; + regs->a4 = args[4]; + regs->a5 = args[5]; } static inline int syscall_get_arch(struct task_struct *task) diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h index ac80216549ff..226289c3b5c8 100644 --- a/arch/riscv/include/asm/syscall_wrapper.h +++ b/arch/riscv/include/asm/syscall_wrapper.h @@ -32,6 +32,10 @@ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *); __diag_push(); \ __diag_ignore(GCC, 8, "-Wattribute-alias", \ "Type aliasing is used to sanitize syscall arguments"); \ + __diag_ignore(clang, 23, "-Wunknown-warning-option", \ + "Avoid breaking versions without -Wattribute-alias"); \ + __diag_ignore(clang, 23, "-Wattribute-alias", \ + "Type aliasing is used to sanitize syscall arguments"); \ static long __se_##prefix##name(ulong, ulong, ulong, ulong, ulong, ulong, \ ulong) \ __attribute__((alias(__stringify(___se_##prefix##name)))); \ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index f5916a70879a..55019fdfa9ec 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -37,7 +37,7 @@ #define IRQ_STACK_SIZE THREAD_SIZE -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/processor.h> #include <asm/csr.h> @@ -73,6 +73,9 @@ struct thread_info { */ unsigned long a0, a1, a2; #endif +#ifdef CONFIG_RISCV_USER_CFI + struct cfi_state user_cfi_state; +#endif }; #ifdef CONFIG_SHADOW_CALL_STACK @@ -98,7 +101,7 @@ struct thread_info { void arch_release_task_struct(struct task_struct *tsk); int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ /* * thread information flags @@ -107,23 +110,18 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); * - pending work-to-be-done flags are in lowest half-word * - other flags in upper half-word(s) */ -#define TIF_NEED_RESCHED 0 /* rescheduling necessary */ -#define TIF_NEED_RESCHED_LAZY 1 /* Lazy rescheduling needed */ -#define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ -#define TIF_SIGPENDING 3 /* signal pending */ -#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ -#define TIF_MEMDIE 5 /* is terminating due to OOM killer */ -#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ -#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */ -#define TIF_32BIT 11 /* compat-mode 32bit process */ -#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */ - -#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) -#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) -#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) -#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) -#define _TIF_UPROBE (1 << TIF_UPROBE) -#define _TIF_RISCV_V_DEFER_RESTORE (1 << TIF_RISCV_V_DEFER_RESTORE) + +/* + * Tell the generic TIF infrastructure which bits riscv supports + */ +#define HAVE_TIF_NEED_RESCHED_LAZY +#define HAVE_TIF_RESTORE_SIGMASK + +#include <asm-generic/thread_info_tif.h> + +#define TIF_32BIT 16 /* compat-mode 32bit process */ +#define TIF_RISCV_V_DEFER_RESTORE 17 /* restore Vector before returning to user */ + +#define _TIF_RISCV_V_DEFER_RESTORE BIT(TIF_RISCV_V_DEFER_RESTORE) #endif /* _ASM_RISCV_THREAD_INFO_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index 72e559934952..eed0abc40514 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -56,13 +56,13 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); #endif bool arch_tlbbatch_should_defer(struct mm_struct *mm); void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm, - unsigned long uaddr); -void arch_flush_tlb_batched_pending(struct mm_struct *mm); + struct mm_struct *mm, unsigned long start, unsigned long end); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); extern unsigned long tlb_flush_all_threshold; diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index fee56b0c8058..11c9886c3b70 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -62,6 +62,19 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigne __asm__ __volatile__ ("csrc sstatus, %0" : : "r" (SR_SUM) : "memory") /* + * This is the smallest unsigned integer type that can fit a value + * (up to 'long long') + */ +#define __inttype(x) __typeof__( \ + __typefits(x, char, \ + __typefits(x, short, \ + __typefits(x, int, \ + __typefits(x, long, 0ULL))))) + +#define __typefits(x, type, not) \ + __builtin_choose_expr(sizeof(x) <= sizeof(type), (unsigned type)0, not) + +/* * The exception table consists of pairs of addresses: the first is the * address of an instruction that is allowed to fault, and the second is * the address at which the program should continue. No registers are @@ -83,27 +96,69 @@ static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigne * call. */ -#define __get_user_asm(insn, x, ptr, err) \ +#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT +/* + * Use a temporary variable for the output of the asm goto to avoid a + * triggering an LLVM assertion due to sign extending the output when + * it is used in later function calls: + * https://github.com/llvm/llvm-project/issues/143795 + */ +#define __get_user_asm(insn, x, ptr, label) \ +do { \ + u64 __tmp; \ + asm_goto_output( \ + "1:\n" \ + " " insn " %0, %1\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, %l2, %0) \ + : "=&r" (__tmp) \ + : "m" (*(ptr)) : : label); \ + (x) = (__typeof__(x))(unsigned long)__tmp; \ +} while (0) +#else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ +#define __get_user_asm(insn, x, ptr, label) \ do { \ - __typeof__(x) __x; \ + long __gua_err = 0; \ __asm__ __volatile__ ( \ "1:\n" \ " " insn " %1, %2\n" \ "2:\n" \ _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %0, %1) \ - : "+r" (err), "=&r" (__x) \ + : "+r" (__gua_err), "=&r" (x) \ : "m" (*(ptr))); \ - (x) = __x; \ + if (__gua_err) \ + goto label; \ } while (0) +#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ #ifdef CONFIG_64BIT -#define __get_user_8(x, ptr, err) \ - __get_user_asm("ld", x, ptr, err) +#define __get_user_8(x, ptr, label) \ + __get_user_asm("ld", x, ptr, label) #else /* !CONFIG_64BIT */ -#define __get_user_8(x, ptr, err) \ + +#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT +#define __get_user_8(x, ptr, label) \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u32 __lo, __hi; \ + asm_goto_output( \ + "1:\n" \ + " lw %0, %2\n" \ + "2:\n" \ + " lw %1, %3\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, %l4, %0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, %l4, %0) \ + : "=&r" (__lo), "=r" (__hi) \ + : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]) \ + : : label); \ + (x) = (__typeof__(x))((__typeof__((x) - (x)))( \ + (((u64)__hi << 32) | __lo))); \ +} while (0) +#else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ +#define __get_user_8(x, ptr, label) \ +do { \ + u32 __user *__ptr = (u32 __user *)(ptr); \ + u32 __lo, __hi; \ + long __gu8_err = 0; \ __asm__ __volatile__ ( \ "1:\n" \ " lw %1, %3\n" \ @@ -112,35 +167,62 @@ do { \ "3:\n" \ _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 3b, %0, %1) \ _ASM_EXTABLE_UACCESS_ERR_ZERO(2b, 3b, %0, %1) \ - : "+r" (err), "=&r" (__lo), "=r" (__hi) \ + : "+r" (__gu8_err), "=&r" (__lo), "=r" (__hi) \ : "m" (__ptr[__LSW]), "m" (__ptr[__MSW])); \ - if (err) \ + if (__gu8_err) { \ __hi = 0; \ - (x) = (__typeof__(x))((__typeof__((x)-(x)))( \ + goto label; \ + } \ + (x) = (__typeof__(x))((__typeof__((x) - (x)))( \ (((u64)__hi << 32) | __lo))); \ } while (0) +#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ + #endif /* CONFIG_64BIT */ -#define __get_user_nocheck(x, __gu_ptr, __gu_err) \ +unsigned long __must_check __asm_copy_to_user_sum_enabled(void __user *to, + const void *from, unsigned long n); +unsigned long __must_check __asm_copy_from_user_sum_enabled(void *to, + const void __user *from, unsigned long n); + +#define __get_user_nocheck(x, __gu_ptr, label) \ do { \ + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && \ + !IS_ALIGNED((uintptr_t)__gu_ptr, sizeof(*__gu_ptr))) { \ + if (__asm_copy_from_user_sum_enabled(&(x), __gu_ptr, sizeof(*__gu_ptr))) \ + goto label; \ + break; \ + } \ switch (sizeof(*__gu_ptr)) { \ case 1: \ - __get_user_asm("lb", (x), __gu_ptr, __gu_err); \ + __get_user_asm("lb", (x), __gu_ptr, label); \ break; \ case 2: \ - __get_user_asm("lh", (x), __gu_ptr, __gu_err); \ + __get_user_asm("lh", (x), __gu_ptr, label); \ break; \ case 4: \ - __get_user_asm("lw", (x), __gu_ptr, __gu_err); \ + __get_user_asm("lw", (x), __gu_ptr, label); \ break; \ case 8: \ - __get_user_8((x), __gu_ptr, __gu_err); \ + __get_user_8((x), __gu_ptr, label); \ break; \ default: \ BUILD_BUG(); \ } \ } while (0) +#define __get_user_error(x, ptr, err) \ +do { \ + __label__ __gu_failed; \ + \ + __get_user_nocheck(x, ptr, __gu_failed); \ + err = 0; \ + break; \ +__gu_failed: \ + x = (__typeof__(x))0; \ + err = -EFAULT; \ +} while (0) + /** * __get_user: - Get a simple variable from user space, with less checking. * @x: Variable to store result. @@ -165,13 +247,16 @@ do { \ ({ \ const __typeof__(*(ptr)) __user *__gu_ptr = untagged_addr(ptr); \ long __gu_err = 0; \ + __typeof__(x) __gu_val; \ \ __chk_user_ptr(__gu_ptr); \ \ __enable_user_access(); \ - __get_user_nocheck(x, __gu_ptr, __gu_err); \ + __get_user_error(__gu_val, __gu_ptr, __gu_err); \ __disable_user_access(); \ \ + (x) = __gu_val; \ + \ __gu_err; \ }) @@ -201,61 +286,73 @@ do { \ ((x) = (__force __typeof__(x))0, -EFAULT); \ }) -#define __put_user_asm(insn, x, ptr, err) \ +#define __put_user_asm(insn, x, ptr, label) \ do { \ __typeof__(*(ptr)) __x = x; \ - __asm__ __volatile__ ( \ + asm goto( \ "1:\n" \ - " " insn " %z2, %1\n" \ - "2:\n" \ - _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %0) \ - : "+r" (err), "=m" (*(ptr)) \ - : "rJ" (__x)); \ + " " insn " %z0, %1\n" \ + _ASM_EXTABLE(1b, %l2) \ + : : "rJ" (__x), "m"(*(ptr)) : : label); \ } while (0) #ifdef CONFIG_64BIT -#define __put_user_8(x, ptr, err) \ - __put_user_asm("sd", x, ptr, err) +#define __put_user_8(x, ptr, label) \ + __put_user_asm("sd", x, ptr, label) #else /* !CONFIG_64BIT */ -#define __put_user_8(x, ptr, err) \ +#define __put_user_8(x, ptr, label) \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u64 __x = (__typeof__((x)-(x)))(x); \ - __asm__ __volatile__ ( \ + asm goto( \ "1:\n" \ - " sw %z3, %1\n" \ + " sw %z0, %2\n" \ "2:\n" \ - " sw %z4, %2\n" \ - "3:\n" \ - _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) \ - _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) \ - : "+r" (err), \ - "=m" (__ptr[__LSW]), \ - "=m" (__ptr[__MSW]) \ - : "rJ" (__x), "rJ" (__x >> 32)); \ + " sw %z1, %3\n" \ + _ASM_EXTABLE(1b, %l4) \ + _ASM_EXTABLE(2b, %l4) \ + : : "rJ" (__x), "rJ" (__x >> 32), \ + "m" (__ptr[__LSW]), \ + "m" (__ptr[__MSW]) : : label); \ } while (0) #endif /* CONFIG_64BIT */ -#define __put_user_nocheck(x, __gu_ptr, __pu_err) \ +#define __put_user_nocheck(x, __gu_ptr, label) \ do { \ + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && \ + !IS_ALIGNED((uintptr_t)__gu_ptr, sizeof(*__gu_ptr))) { \ + __typeof__(*(__gu_ptr)) ___val = (x); \ + if (__asm_copy_to_user_sum_enabled(__gu_ptr, &(___val), sizeof(*__gu_ptr))) \ + goto label; \ + break; \ + } \ switch (sizeof(*__gu_ptr)) { \ case 1: \ - __put_user_asm("sb", (x), __gu_ptr, __pu_err); \ + __put_user_asm("sb", (x), __gu_ptr, label); \ break; \ case 2: \ - __put_user_asm("sh", (x), __gu_ptr, __pu_err); \ + __put_user_asm("sh", (x), __gu_ptr, label); \ break; \ case 4: \ - __put_user_asm("sw", (x), __gu_ptr, __pu_err); \ + __put_user_asm("sw", (x), __gu_ptr, label); \ break; \ case 8: \ - __put_user_8((x), __gu_ptr, __pu_err); \ + __put_user_8((x), __gu_ptr, label); \ break; \ default: \ BUILD_BUG(); \ } \ } while (0) +#define __put_user_error(x, ptr, err) \ +do { \ + __label__ err_label; \ + __put_user_nocheck(x, ptr, err_label); \ + break; \ +err_label: \ + (err) = -EFAULT; \ +} while (0) + /** * __put_user: - Write a simple value into user space, with less checking. * @x: Value to copy to user space. @@ -286,7 +383,7 @@ do { \ __chk_user_ptr(__gu_ptr); \ \ __enable_user_access(); \ - __put_user_nocheck(__val, __gu_ptr, __pu_err); \ + __put_user_error(__val, __gu_ptr, __pu_err); \ __disable_user_access(); \ \ __pu_err; \ @@ -350,24 +447,46 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __clear_user(untagged_addr(to), n) : n; } -#define __get_kernel_nofault(dst, src, type, err_label) \ -do { \ - long __kr_err = 0; \ - \ - __get_user_nocheck(*((type *)(dst)), (type *)(src), __kr_err); \ - if (unlikely(__kr_err)) \ - goto err_label; \ -} while (0) +#define arch_get_kernel_nofault(dst, src, type, err_label) \ + __get_user_nocheck(*((type *)(dst)), (__force __user type *)(src), err_label) -#define __put_kernel_nofault(dst, src, type, err_label) \ -do { \ - long __kr_err = 0; \ - \ - __put_user_nocheck(*((type *)(src)), (type *)(dst), __kr_err); \ - if (unlikely(__kr_err)) \ - goto err_label; \ +#define arch_put_kernel_nofault(dst, src, type, err_label) \ + __put_user_nocheck(*((type *)(src)), (__force __user type *)(dst), err_label) + +static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) +{ + if (unlikely(!access_ok(ptr, len))) + return 0; + __enable_user_access(); + return 1; +} +#define user_access_begin user_access_begin +#define user_access_end __disable_user_access + +static inline unsigned long user_access_save(void) { return 0UL; } +static inline void user_access_restore(unsigned long enabled) { } + +/* + * We want the unsafe accessors to always be inlined and use + * the error labels - thus the macro games. + */ +#define arch_unsafe_put_user(x, ptr, label) \ + __put_user_nocheck(x, (ptr), label) + +#define arch_unsafe_get_user(x, ptr, label) do { \ + __inttype(*(ptr)) __gu_val; \ + __get_user_nocheck(__gu_val, (ptr), label); \ + (x) = (__force __typeof__(*(ptr)))__gu_val; \ } while (0) +#define unsafe_copy_to_user(_dst, _src, _len, label) \ + if (__asm_copy_to_user_sum_enabled(_dst, _src, _len)) \ + goto label; + +#define unsafe_copy_from_user(_dst, _src, _len, label) \ + if (__asm_copy_from_user_sum_enabled(_dst, _src, _len)) \ + goto label; + #else /* CONFIG_MMU */ #include <asm-generic/uaccess.h> #endif /* CONFIG_MMU */ diff --git a/arch/riscv/include/asm/usercfi.h b/arch/riscv/include/asm/usercfi.h new file mode 100644 index 000000000000..f56966edbf5c --- /dev/null +++ b/arch/riscv/include/asm/usercfi.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Copyright (C) 2024 Rivos, Inc. + * Deepak Gupta <debug@rivosinc.com> + */ +#ifndef _ASM_RISCV_USERCFI_H +#define _ASM_RISCV_USERCFI_H + +#define CMDLINE_DISABLE_RISCV_USERCFI_FCFI 1 +#define CMDLINE_DISABLE_RISCV_USERCFI_BCFI 2 +#define CMDLINE_DISABLE_RISCV_USERCFI 3 + +#ifndef __ASSEMBLER__ +#include <linux/types.h> +#include <linux/prctl.h> +#include <linux/errno.h> + +struct task_struct; +struct kernel_clone_args; + +extern unsigned long riscv_nousercfi; + +#ifdef CONFIG_RISCV_USER_CFI +struct cfi_state { + unsigned long ubcfi_en : 1; /* Enable for backward cfi. */ + unsigned long ubcfi_locked : 1; + unsigned long ufcfi_en : 1; /* Enable for forward cfi. Note that ELP goes in sstatus */ + unsigned long ufcfi_locked : 1; + unsigned long user_shdw_stk; /* Current user shadow stack pointer */ + unsigned long shdw_stk_base; /* Base address of shadow stack */ + unsigned long shdw_stk_size; /* size of shadow stack */ +}; + +unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, + const struct kernel_clone_args *args); +void shstk_release(struct task_struct *tsk); +void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size); +unsigned long get_shstk_base(struct task_struct *task, unsigned long *size); +void set_active_shstk(struct task_struct *task, unsigned long shstk_addr); +bool is_shstk_enabled(struct task_struct *task); +bool is_shstk_locked(struct task_struct *task); +bool is_shstk_allocated(struct task_struct *task); +void set_shstk_lock(struct task_struct *task, bool lock); +void set_shstk_status(struct task_struct *task, bool enable); +unsigned long get_active_shstk(struct task_struct *task); +int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr); +int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr); +bool is_indir_lp_enabled(struct task_struct *task); +bool is_indir_lp_locked(struct task_struct *task); +void set_indir_lp_status(struct task_struct *task, bool enable); +void set_indir_lp_lock(struct task_struct *task, bool lock); + +#define PR_SHADOW_STACK_SUPPORTED_STATUS_MASK (PR_SHADOW_STACK_ENABLE) + +#else + +#define shstk_alloc_thread_stack(tsk, args) 0 + +#define shstk_release(tsk) + +#define get_shstk_base(task, size) 0UL + +#define set_shstk_base(task, shstk_addr, size) do {} while (0) + +#define set_active_shstk(task, shstk_addr) do {} while (0) + +#define is_shstk_enabled(task) false + +#define is_shstk_locked(task) false + +#define is_shstk_allocated(task) false + +#define set_shstk_lock(task, lock) do {} while (0) + +#define set_shstk_status(task, enable) do {} while (0) + +#define is_indir_lp_enabled(task) false + +#define is_indir_lp_locked(task) false + +#define set_indir_lp_status(task, enable) do {} while (0) + +#define set_indir_lp_lock(task, lock) do {} while (0) + +#define restore_user_shstk(tsk, shstk_ptr) -EINVAL + +#define save_user_shstk(tsk, saved_shstk_ptr) -EINVAL + +#define get_active_shstk(task) 0UL + +#endif /* CONFIG_RISCV_USER_CFI */ + +bool is_user_shstk_enabled(void); +bool is_user_lpad_enabled(void); + +#endif /* __ASSEMBLER__ */ + +#endif /* _ASM_RISCV_USERCFI_H */ diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index f891478829a5..35bf830a5576 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -14,13 +14,23 @@ */ #ifdef CONFIG_MMU -#define __VVAR_PAGES 2 +#define __VDSO_PAGES 4 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <generated/vdso-offsets.h> +#ifdef CONFIG_RISCV_USER_CFI +#include <generated/vdso-cfi-offsets.h> +#endif +#ifdef CONFIG_RISCV_USER_CFI #define VDSO_SYMBOL(base, name) \ - (void __user *)((unsigned long)(base) + __vdso_##name##_offset) + (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP) ? \ + (void __user *)((unsigned long)(base) + __vdso_##name##_cfi_offset) : \ + (void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#else +#define VDSO_SYMBOL(base, name) \ + ((void __user *)((unsigned long)(base) + __vdso_##name##_offset)) +#endif #ifdef CONFIG_COMPAT #include <generated/compat_vdso-offsets.h> @@ -33,8 +43,9 @@ extern char compat_vdso_start[], compat_vdso_end[]; #endif /* CONFIG_COMPAT */ extern char vdso_start[], vdso_end[]; +extern char vdso_cfi_start[], vdso_cfi_end[]; -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* CONFIG_MMU */ diff --git a/arch/riscv/include/asm/vdso/time_data.h b/arch/riscv/include/asm/vdso/arch_data.h index dfa65228999b..88b37af55175 100644 --- a/arch/riscv/include/asm/vdso/time_data.h +++ b/arch/riscv/include/asm/vdso/arch_data.h @@ -1,17 +1,23 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __RISCV_ASM_VDSO_TIME_DATA_H -#define __RISCV_ASM_VDSO_TIME_DATA_H +#ifndef __RISCV_ASM_VDSO_ARCH_DATA_H +#define __RISCV_ASM_VDSO_ARCH_DATA_H #include <linux/types.h> #include <vdso/datapage.h> #include <asm/hwprobe.h> -struct arch_vdso_time_data { +struct vdso_arch_data { /* Stash static answers to the hwprobe queries when all CPUs are selected. */ __u64 all_cpu_hwprobe_values[RISCV_HWPROBE_MAX_KEY + 1]; /* Boolean indicating all CPUs have the same static hwprobe values. */ __u8 homogeneous_cpus; + + /* + * A gate to check and see if the hwprobe data is actually ready, as + * probing is deferred to avoid boot slowdowns. + */ + __u8 ready; }; -#endif /* __RISCV_ASM_VDSO_TIME_DATA_H */ +#endif /* __RISCV_ASM_VDSO_ARCH_DATA_H */ diff --git a/arch/riscv/include/asm/vdso/getrandom.h b/arch/riscv/include/asm/vdso/getrandom.h new file mode 100644 index 000000000000..ab4aef955099 --- /dev/null +++ b/arch/riscv/include/asm/vdso/getrandom.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved. + */ +#ifndef __ASM_VDSO_GETRANDOM_H +#define __ASM_VDSO_GETRANDOM_H + +#ifndef __ASSEMBLER__ + +#include <asm/unistd.h> + +static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, unsigned int _flags) +{ + register long ret asm("a0"); + register long nr asm("a7") = __NR_getrandom; + register void *buffer asm("a0") = _buffer; + register size_t len asm("a1") = _len; + register unsigned int flags asm("a2") = _flags; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r" (nr), "r" (buffer), "r" (len), "r" (flags) + : "memory"); + + return ret; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* __ASM_VDSO_GETRANDOM_H */ diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h index ba3283cf7acc..9ec08fa04d35 100644 --- a/arch/riscv/include/asm/vdso/gettimeofday.h +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -2,7 +2,7 @@ #ifndef __ASM_VDSO_GETTIMEOFDAY_H #define __ASM_VDSO_GETTIMEOFDAY_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/barrier.h> #include <asm/unistd.h> @@ -69,7 +69,7 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) #endif /* CONFIG_GENERIC_TIME_VSYSCALL */ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { /* * The purpose of csr_read(CSR_TIME) is to trap the system into @@ -79,18 +79,6 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, return csr_read(CSR_TIME); } -static __always_inline const struct vdso_data *__arch_get_vdso_data(void) -{ - return _vdso_data; -} - -#ifdef CONFIG_TIME_NS -static __always_inline -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) -{ - return _timens_data; -} -#endif -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h index 8f383f05a290..c42f95dc8811 100644 --- a/arch/riscv/include/asm/vdso/processor.h +++ b/arch/riscv/include/asm/vdso/processor.h @@ -2,9 +2,10 @@ #ifndef __ASM_VDSO_PROCESSOR_H #define __ASM_VDSO_PROCESSOR_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/barrier.h> +#include <asm/errata_list.h> #include <asm/insn-def.h> static inline void cpu_relax(void) @@ -19,10 +20,10 @@ static inline void cpu_relax(void) * Reduce instruction retirement. * This assumes the PC changes. */ - __asm__ __volatile__ (RISCV_PAUSE); + ALT_RISCV_PAUSE(); barrier(); } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h index e8a9c4b53c0c..558eb9dfda52 100644 --- a/arch/riscv/include/asm/vdso/vsyscall.h +++ b/arch/riscv/include/asm/vdso/vsyscall.h @@ -2,22 +2,13 @@ #ifndef __ASM_VDSO_VSYSCALL_H #define __ASM_VDSO_VSYSCALL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <vdso/datapage.h> -extern struct vdso_data *vdso_data; - -static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void) -{ - return vdso_data; -} - -#define __arch_get_k_vdso_data __riscv_get_k_vdso_data - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index e8a83f55be2b..00cb9c0982b1 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -51,6 +51,7 @@ void put_cpu_vector_context(void); void riscv_v_thread_free(struct task_struct *tsk); void __init riscv_v_setup_ctx_cache(void); void riscv_v_thread_alloc(struct task_struct *tsk); +void __init update_regset_vector_info(unsigned long size); static inline u32 riscv_v_flags(void) { @@ -120,6 +121,11 @@ static __always_inline void riscv_v_disable(void) csr_clear(CSR_SSTATUS, SR_VS); } +static __always_inline bool riscv_v_is_on(void) +{ + return !!(csr_read(CSR_SSTATUS) & SR_VS); +} + static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) { asm volatile ( @@ -200,11 +206,11 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, THEAD_VSETVLI_T4X0E8M8D1 THEAD_VSB_V_V0T0 "add t0, t0, t4\n\t" - THEAD_VSB_V_V0T0 + THEAD_VSB_V_V8T0 "add t0, t0, t4\n\t" - THEAD_VSB_V_V0T0 + THEAD_VSB_V_V16T0 "add t0, t0, t4\n\t" - THEAD_VSB_V_V0T0 + THEAD_VSB_V_V24T0 : : "r" (datap) : "memory", "t0", "t4"); } else { asm volatile ( @@ -236,11 +242,11 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_ THEAD_VSETVLI_T4X0E8M8D1 THEAD_VLB_V_V0T0 "add t0, t0, t4\n\t" - THEAD_VLB_V_V0T0 + THEAD_VLB_V_V8T0 "add t0, t0, t4\n\t" - THEAD_VLB_V_V0T0 + THEAD_VLB_V_V16T0 "add t0, t0, t4\n\t" - THEAD_VLB_V_V0T0 + THEAD_VLB_V_V24T0 : : "r" (datap) : "memory", "t0", "t4"); } else { asm volatile ( @@ -366,6 +372,11 @@ static inline void __switch_to_vector(struct task_struct *prev, struct pt_regs *regs; if (riscv_preempt_v_started(prev)) { + if (riscv_v_is_on()) { + WARN_ON(prev->thread.riscv_v_flags & RISCV_V_CTX_DEPTH_MASK); + riscv_v_disable(); + prev->thread.riscv_v_flags |= RISCV_PREEMPT_V_IN_SCHEDULE; + } if (riscv_preempt_v_dirty(prev)) { __riscv_v_vstate_save(&prev->thread.kernel_vstate, prev->thread.kernel_vstate.datap); @@ -376,10 +387,16 @@ static inline void __switch_to_vector(struct task_struct *prev, riscv_v_vstate_save(&prev->thread.vstate, regs); } - if (riscv_preempt_v_started(next)) - riscv_preempt_v_set_restore(next); - else + if (riscv_preempt_v_started(next)) { + if (next->thread.riscv_v_flags & RISCV_PREEMPT_V_IN_SCHEDULE) { + next->thread.riscv_v_flags &= ~RISCV_PREEMPT_V_IN_SCHEDULE; + riscv_v_enable(); + } else { + riscv_preempt_v_set_restore(next); + } + } else { riscv_v_vstate_set_restore(next, task_pt_regs(next)); + } } void riscv_v_vstate_ctrl_init(struct task_struct *tsk); @@ -407,6 +424,9 @@ static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } #define riscv_v_thread_free(tsk) do {} while (0) #define riscv_v_setup_ctx_cache() do {} while (0) #define riscv_v_thread_alloc(tsk) do {} while (0) +#define get_cpu_vector_context() do {} while (0) +#define put_cpu_vector_context() do {} while (0) +#define riscv_v_vstate_set_restore(task, regs) do {} while (0) #endif /* CONFIG_RISCV_ISA_V */ diff --git a/arch/riscv/include/asm/vendor_extensions/mips.h b/arch/riscv/include/asm/vendor_extensions/mips.h new file mode 100644 index 000000000000..ffeb12dc17a3 --- /dev/null +++ b/arch/riscv/include/asm/vendor_extensions/mips.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 MIPS. + */ + +#ifndef _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_H +#define _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_H + +#include <linux/types.h> + +#define RISCV_ISA_VENDOR_EXT_XMIPSEXECTL 0 + +#ifndef __ASSEMBLER__ +struct riscv_isa_vendor_ext_data_list; +extern struct riscv_isa_vendor_ext_data_list riscv_isa_vendor_ext_list_mips; +#endif + +/* Extension specific instructions */ + +/* + * All of the xmipsexectl extension instructions are + * ‘hint’ encodings of the SLLI instruction, + * with rd = 0, rs1 = 0 and imm = 1 for IHB, imm = 3 for EHB, + * and imm = 5 for PAUSE. + * MIPS.PAUSE is an alternative opcode which is implemented to have the + * same behavior as PAUSE on some MIPS RISCV cores. + * MIPS.EHB clears all execution hazards before allowing + * any subsequent instructions to execute. + * MIPS.IHB clears all instruction hazards before + * allowing any subsequent instructions to fetch. + */ + +#define MIPS_PAUSE ASM_INSN_I("0x00501013\n\t") +#define MIPS_EHB ASM_INSN_I("0x00301013\n\t") +#define MIPS_IHB ASM_INSN_I("0x00101013\n\t") + +#endif // _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_H diff --git a/arch/riscv/include/asm/vendor_extensions/mips_hwprobe.h b/arch/riscv/include/asm/vendor_extensions/mips_hwprobe.h new file mode 100644 index 000000000000..e63f664b6b17 --- /dev/null +++ b/arch/riscv/include/asm/vendor_extensions/mips_hwprobe.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 MIPS. + */ + +#ifndef _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_HWPROBE_H_ +#define _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_HWPROBE_H_ + +#include <linux/cpumask.h> +#include <uapi/asm/hwprobe.h> + +#ifdef CONFIG_RISCV_ISA_VENDOR_EXT_MIPS +void hwprobe_isa_vendor_ext_mips_0(struct riscv_hwprobe *pair, const struct cpumask *cpus); +#else +static inline void hwprobe_isa_vendor_ext_mips_0(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + pair->value = 0; +} +#endif + +#endif // _ASM_RISCV_VENDOR_EXTENSIONS_MIPS_HWPROBE_H_ diff --git a/arch/riscv/include/asm/vendor_extensions/sifive.h b/arch/riscv/include/asm/vendor_extensions/sifive.h new file mode 100644 index 000000000000..ac00e500361c --- /dev/null +++ b/arch/riscv/include/asm/vendor_extensions/sifive.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_VENDOR_EXTENSIONS_SIFIVE_H +#define _ASM_RISCV_VENDOR_EXTENSIONS_SIFIVE_H + +#include <asm/vendor_extensions.h> + +#include <linux/types.h> + +#define RISCV_ISA_VENDOR_EXT_XSFVQMACCDOD 0 +#define RISCV_ISA_VENDOR_EXT_XSFVQMACCQOQ 1 +#define RISCV_ISA_VENDOR_EXT_XSFVFNRCLIPXFQF 2 +#define RISCV_ISA_VENDOR_EXT_XSFVFWMACCQQQ 3 + +extern struct riscv_isa_vendor_ext_data_list riscv_isa_vendor_ext_list_sifive; + +#endif diff --git a/arch/riscv/include/asm/vendor_extensions/sifive_hwprobe.h b/arch/riscv/include/asm/vendor_extensions/sifive_hwprobe.h new file mode 100644 index 000000000000..90a61abd033c --- /dev/null +++ b/arch/riscv/include/asm/vendor_extensions/sifive_hwprobe.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_VENDOR_EXTENSIONS_SIFIVE_HWPROBE_H +#define _ASM_RISCV_VENDOR_EXTENSIONS_SIFIVE_HWPROBE_H + +#include <linux/cpumask.h> + +#include <uapi/asm/hwprobe.h> + +#ifdef CONFIG_RISCV_ISA_VENDOR_EXT_SIFIVE +void hwprobe_isa_vendor_ext_sifive_0(struct riscv_hwprobe *pair, const struct cpumask *cpus); +#else +static inline void hwprobe_isa_vendor_ext_sifive_0(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + pair->value = 0; +} +#endif + +#endif diff --git a/arch/riscv/include/asm/vendorid_list.h b/arch/riscv/include/asm/vendorid_list.h index a5150cdf34d8..7f5030ee1fcf 100644 --- a/arch/riscv/include/asm/vendorid_list.h +++ b/arch/riscv/include/asm/vendorid_list.h @@ -7,6 +7,7 @@ #define ANDES_VENDOR_ID 0x31e #define MICROCHIP_VENDOR_ID 0x029 +#define MIPS_VENDOR_ID 0x127 #define SIFIVE_VENDOR_ID 0x489 #define THEAD_VENDOR_ID 0x5b7 diff --git a/arch/riscv/include/asm/xip_fixup.h b/arch/riscv/include/asm/xip_fixup.h deleted file mode 100644 index f3d56299bc22..000000000000 --- a/arch/riscv/include/asm/xip_fixup.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * XIP fixup macros, only useful in assembly. - */ -#ifndef _ASM_RISCV_XIP_FIXUP_H -#define _ASM_RISCV_XIP_FIXUP_H - -#include <linux/pgtable.h> - -#ifdef CONFIG_XIP_KERNEL -.macro XIP_FIXUP_OFFSET reg - /* Fix-up address in Flash into address in RAM early during boot before - * MMU is up. Because generated code "thinks" data is in Flash, but it - * is actually in RAM (actually data is also in Flash, but Flash is - * read-only, thus we need to use the data residing in RAM). - * - * The start of data in Flash is _sdata and the start of data in RAM is - * CONFIG_PHYS_RAM_BASE. So this fix-up essentially does this: - * reg += CONFIG_PHYS_RAM_BASE - _start - */ - li t0, CONFIG_PHYS_RAM_BASE - add \reg, \reg, t0 - la t0, _sdata - sub \reg, \reg, t0 -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg - /* In linker script, at the transition from read-only section to - * writable section, the VMA is increased while LMA remains the same. - * (See in linker script how _sdata, __data_loc and LOAD_OFFSET is - * changed) - * - * Consequently, early during boot before MMU is up, the generated code - * reads the "writable" section at wrong addresses, because VMA is used - * by compiler to generate code, but the data is located in Flash using - * LMA. - */ - la t0, _sdata - sub \reg, \reg, t0 - la t0, __data_loc - add \reg, \reg, t0 -.endm -#else -.macro XIP_FIXUP_OFFSET reg -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg -.endm -#endif /* CONFIG_XIP_KERNEL */ - -#endif diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h deleted file mode 100644 index 96011861e46b..000000000000 --- a/arch/riscv/include/asm/xor.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2021 SiFive - */ - -#include <linux/hardirq.h> -#include <asm-generic/xor.h> -#ifdef CONFIG_RISCV_ISA_V -#include <asm/vector.h> -#include <asm/switch_to.h> -#include <asm/asm-prototypes.h> - -static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2) -{ - kernel_vector_begin(); - xor_regs_2_(bytes, p1, p2); - kernel_vector_end(); -} - -static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3) -{ - kernel_vector_begin(); - xor_regs_3_(bytes, p1, p2, p3); - kernel_vector_end(); -} - -static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4) -{ - kernel_vector_begin(); - xor_regs_4_(bytes, p1, p2, p3, p4); - kernel_vector_end(); -} - -static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1, - const unsigned long *__restrict p2, - const unsigned long *__restrict p3, - const unsigned long *__restrict p4, - const unsigned long *__restrict p5) -{ - kernel_vector_begin(); - xor_regs_5_(bytes, p1, p2, p3, p4, p5); - kernel_vector_end(); -} - -static struct xor_block_template xor_block_rvv = { - .name = "rvv", - .do_2 = xor_vector_2, - .do_3 = xor_vector_3, - .do_4 = xor_vector_4, - .do_5 = xor_vector_5 -}; - -#undef XOR_TRY_TEMPLATES -#define XOR_TRY_TEMPLATES \ - do { \ - xor_speed(&xor_block_8regs); \ - xor_speed(&xor_block_32regs); \ - if (has_vector()) { \ - xor_speed(&xor_block_rvv);\ - } \ - } while (0) -#endif diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h index c3c1cc951cb9..9139edba0aec 100644 --- a/arch/riscv/include/uapi/asm/hwprobe.h +++ b/arch/riscv/include/uapi/asm/hwprobe.h @@ -73,6 +73,21 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47) #define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48) #define RISCV_HWPROBE_EXT_SUPM (1ULL << 49) +#define RISCV_HWPROBE_EXT_ZICNTR (1ULL << 50) +#define RISCV_HWPROBE_EXT_ZIHPM (1ULL << 51) +#define RISCV_HWPROBE_EXT_ZFBFMIN (1ULL << 52) +#define RISCV_HWPROBE_EXT_ZVFBFMIN (1ULL << 53) +#define RISCV_HWPROBE_EXT_ZVFBFWMA (1ULL << 54) +#define RISCV_HWPROBE_EXT_ZICBOM (1ULL << 55) +#define RISCV_HWPROBE_EXT_ZAAMO (1ULL << 56) +#define RISCV_HWPROBE_EXT_ZALRSC (1ULL << 57) +#define RISCV_HWPROBE_EXT_ZABHA (1ULL << 58) +#define RISCV_HWPROBE_EXT_ZALASR (1ULL << 59) +#define RISCV_HWPROBE_EXT_ZICBOP (1ULL << 60) +#define RISCV_HWPROBE_EXT_ZILSD (1ULL << 61) +#define RISCV_HWPROBE_EXT_ZCLSD (1ULL << 62) +#define RISCV_HWPROBE_EXT_ZICFILP (1ULL << 63) + #define RISCV_HWPROBE_KEY_CPUPERF_0 5 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0) #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0) @@ -95,6 +110,13 @@ struct riscv_hwprobe { #define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3 #define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4 #define RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0 11 +#define RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE 12 +#define RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0 13 +#define RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0 14 +#define RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE 15 +#define RISCV_HWPROBE_KEY_IMA_EXT_1 16 +#define RISCV_HWPROBE_EXT_ZICFISS (1ULL << 0) + /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ /* Flags */ diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index f06bc5efcd79..504e73305343 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -9,7 +9,7 @@ #ifndef __LINUX_KVM_RISCV_H #define __LINUX_KVM_RISCV_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <asm/bitsperlong.h> @@ -18,10 +18,13 @@ #define __KVM_HAVE_IRQ_LINE #define KVM_COALESCED_MMIO_PAGE_OFFSET 1 +#define KVM_DIRTY_LOG_PAGE_OFFSET 64 #define KVM_INTERRUPT_SET -1U #define KVM_INTERRUPT_UNSET -2U +#define KVM_EXIT_FAIL_ENTRY_NO_VSFILE (1ULL << 0) + /* for KVM_GET_REGS and KVM_SET_REGS */ struct kvm_regs { }; @@ -55,6 +58,7 @@ struct kvm_riscv_config { unsigned long mimpid; unsigned long zicboz_block_size; unsigned long satp_mode; + unsigned long zicbop_block_size; }; /* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ @@ -106,6 +110,10 @@ struct kvm_riscv_timer { __u64 state; }; +/* Possible states for kvm_riscv_timer */ +#define KVM_RISCV_TIMER_STATE_OFF 0 +#define KVM_RISCV_TIMER_STATE_ON 1 + /* * ISA extension IDs specific to KVM. This is not the same as the host ISA * extension IDs as that is internal to the host and should not be exposed @@ -182,6 +190,15 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_SVVPTC, KVM_RISCV_ISA_EXT_ZABHA, KVM_RISCV_ISA_EXT_ZICCRSE, + KVM_RISCV_ISA_EXT_ZAAMO, + KVM_RISCV_ISA_EXT_ZALRSC, + KVM_RISCV_ISA_EXT_ZICBOP, + KVM_RISCV_ISA_EXT_ZFBFMIN, + KVM_RISCV_ISA_EXT_ZVFBFMIN, + KVM_RISCV_ISA_EXT_ZVFBFWMA, + KVM_RISCV_ISA_EXT_ZCLSD, + KVM_RISCV_ISA_EXT_ZILSD, + KVM_RISCV_ISA_EXT_ZALASR, KVM_RISCV_ISA_EXT_MAX, }; @@ -202,6 +219,8 @@ enum KVM_RISCV_SBI_EXT_ID { KVM_RISCV_SBI_EXT_DBCN, KVM_RISCV_SBI_EXT_STA, KVM_RISCV_SBI_EXT_SUSP, + KVM_RISCV_SBI_EXT_FWFT, + KVM_RISCV_SBI_EXT_MPXY, KVM_RISCV_SBI_EXT_MAX, }; @@ -211,9 +230,17 @@ struct kvm_riscv_sbi_sta { unsigned long shmem_hi; }; -/* Possible states for kvm_riscv_timer */ -#define KVM_RISCV_TIMER_STATE_OFF 0 -#define KVM_RISCV_TIMER_STATE_ON 1 +struct kvm_riscv_sbi_fwft_feature { + unsigned long enable; + unsigned long flags; + unsigned long value; +}; + +/* SBI FWFT extension registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ +struct kvm_riscv_sbi_fwft { + struct kvm_riscv_sbi_fwft_feature misaligned_deleg; + struct kvm_riscv_sbi_fwft_feature pointer_masking; +}; /* If you need to interpret the index values, here is the key: */ #define KVM_REG_RISCV_TYPE_MASK 0x00000000FF000000 @@ -294,6 +321,9 @@ struct kvm_riscv_sbi_sta { #define KVM_REG_RISCV_SBI_STA (0x0 << KVM_REG_RISCV_SUBTYPE_SHIFT) #define KVM_REG_RISCV_SBI_STA_REG(name) \ (offsetof(struct kvm_riscv_sbi_sta, name) / sizeof(unsigned long)) +#define KVM_REG_RISCV_SBI_FWFT (0x1 << KVM_REG_RISCV_SUBTYPE_SHIFT) +#define KVM_REG_RISCV_SBI_FWFT_REG(name) \ + (offsetof(struct kvm_riscv_sbi_fwft, name) / sizeof(unsigned long)) /* Device Control API: RISC-V AIA */ #define KVM_DEV_RISCV_APLIC_ALIGN 0x1000 diff --git a/arch/riscv/include/uapi/asm/ptrace.h b/arch/riscv/include/uapi/asm/ptrace.h index a38268b19c3d..3de2b7124aff 100644 --- a/arch/riscv/include/uapi/asm/ptrace.h +++ b/arch/riscv/include/uapi/asm/ptrace.h @@ -6,9 +6,10 @@ #ifndef _UAPI_ASM_RISCV_PTRACE_H #define _UAPI_ASM_RISCV_PTRACE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> +#include <linux/const.h> #define PTRACE_GETFDPIC 33 @@ -127,6 +128,42 @@ struct __riscv_v_regset_state { */ #define RISCV_MAX_VLENB (8192) -#endif /* __ASSEMBLY__ */ +struct __sc_riscv_cfi_state { + unsigned long ss_ptr; /* shadow stack pointer */ +}; + +#define PTRACE_CFI_BRANCH_LANDING_PAD_EN_BIT 0 +#define PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_BIT 1 +#define PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_BIT 2 +#define PTRACE_CFI_SHADOW_STACK_EN_BIT 3 +#define PTRACE_CFI_SHADOW_STACK_LOCK_BIT 4 +#define PTRACE_CFI_SHADOW_STACK_PTR_BIT 5 + +#define PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE _BITUL(PTRACE_CFI_BRANCH_LANDING_PAD_EN_BIT) +#define PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE \ + _BITUL(PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_BIT) +#define PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE \ + _BITUL(PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_BIT) +#define PTRACE_CFI_SHADOW_STACK_EN_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_EN_BIT) +#define PTRACE_CFI_SHADOW_STACK_LOCK_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_LOCK_BIT) +#define PTRACE_CFI_SHADOW_STACK_PTR_STATE _BITUL(PTRACE_CFI_SHADOW_STACK_PTR_BIT) + +#define PTRACE_CFI_STATE_INVALID_MASK ~(PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE | \ + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE | \ + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE | \ + PTRACE_CFI_SHADOW_STACK_EN_STATE | \ + PTRACE_CFI_SHADOW_STACK_LOCK_STATE | \ + PTRACE_CFI_SHADOW_STACK_PTR_STATE) + +struct __cfi_status { + __u64 cfi_state; +}; + +struct user_cfi_state { + struct __cfi_status cfi_status; + __u64 shstk_ptr; +}; + +#endif /* __ASSEMBLER__ */ #endif /* _UAPI_ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/uapi/asm/setup.h b/arch/riscv/include/uapi/asm/setup.h index 66b13a522880..eb4f0209c696 100644 --- a/arch/riscv/include/uapi/asm/setup.h +++ b/arch/riscv/include/uapi/asm/setup.h @@ -3,6 +3,6 @@ #ifndef _UAPI_ASM_RISCV_SETUP_H #define _UAPI_ASM_RISCV_SETUP_H -#define COMMAND_LINE_SIZE 1024 +#define COMMAND_LINE_SIZE 2048 #endif /* _UAPI_ASM_RISCV_SETUP_H */ diff --git a/arch/riscv/include/uapi/asm/sigcontext.h b/arch/riscv/include/uapi/asm/sigcontext.h index cd4f175dc837..d22d0815d605 100644 --- a/arch/riscv/include/uapi/asm/sigcontext.h +++ b/arch/riscv/include/uapi/asm/sigcontext.h @@ -10,12 +10,13 @@ /* The Magic number for signal context frame header. */ #define RISCV_V_MAGIC 0x53465457 +#define RISCV_ZICFISS_MAGIC 0x9487 #define END_MAGIC 0x0 /* The size of END signal context header. */ #define END_HDR_SIZE 0x0 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct __sc_riscv_v_state { struct __riscv_v_ext_state v_state; @@ -35,6 +36,6 @@ struct sigcontext { }; }; -#endif /*!__ASSEMBLY__*/ +#endif /*!__ASSEMBLER__*/ #endif /* _UAPI_ASM_RISCV_SIGCONTEXT_H */ diff --git a/arch/riscv/include/uapi/asm/vendor/mips.h b/arch/riscv/include/uapi/asm/vendor/mips.h new file mode 100644 index 000000000000..e65ab268b265 --- /dev/null +++ b/arch/riscv/include/uapi/asm/vendor/mips.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL BIT(0) diff --git a/arch/riscv/include/uapi/asm/vendor/sifive.h b/arch/riscv/include/uapi/asm/vendor/sifive.h new file mode 100644 index 000000000000..9f3278a4b298 --- /dev/null +++ b/arch/riscv/include/uapi/asm/vendor/sifive.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +#define RISCV_HWPROBE_VENDOR_EXT_XSFVQMACCDOD (1 << 0) +#define RISCV_HWPROBE_VENDOR_EXT_XSFVQMACCQOQ (1 << 1) +#define RISCV_HWPROBE_VENDOR_EXT_XSFVFNRCLIPXFQF (1 << 2) +#define RISCV_HWPROBE_VENDOR_EXT_XSFVFWMACCQQQ (1 << 3) diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 8d186bfced45..cabb99cadfb6 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -3,14 +3,8 @@ # Makefile for the RISC-V Linux kernel # -ifdef CONFIG_FTRACE -CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) -endif -CFLAGS_syscall_table.o += $(call cc-option,-Wno-override-init,) -CFLAGS_compat_syscall_table.o += $(call cc-option,-Wno-override-init,) +CFLAGS_syscall_table.o += $(call cc-disable-warning, override-init) +CFLAGS_compat_syscall_table.o += $(call cc-disable-warning, override-init) ifdef CONFIG_KEXEC_CORE AFLAGS_kexec_relocate.o := -mcmodel=medany $(call cc-option,-mno-relax) @@ -24,7 +18,6 @@ CFLAGS_sbi_ecall.o := -mcmodel=medany ifdef CONFIG_FTRACE CFLAGS_REMOVE_alternative.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_cpufeature.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_sbi_ecall.o = $(CC_FLAGS_FTRACE) endif ifdef CONFIG_RELOCATABLE CFLAGS_alternative.o += -fno-pie @@ -43,7 +36,15 @@ CFLAGS_sbi_ecall.o += -D__NO_FORTIFY endif endif -extra-y += vmlinux.lds +ifdef CONFIG_FTRACE +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi_ecall.o = $(CC_FLAGS_FTRACE) +endif + +always-$(KBUILD_BUILTIN) += vmlinux.lds obj-y += head.o obj-y += soc.o @@ -72,6 +73,7 @@ obj-y += vendor_extensions/ obj-y += probes/ obj-y += tests/ obj-$(CONFIG_MMU) += vdso.o vdso/ +obj-$(CONFIG_RISCV_USER_CFI) += vdso_cfi/ obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o @@ -107,13 +109,13 @@ obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o obj-$(CONFIG_PARAVIRT) += paravirt.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KEXEC_CORE) += kexec_relocate.o crash_save_regs.o machine_kexec.o -obj-$(CONFIG_KEXEC_FILE) += elf_kexec.o machine_kexec_file.o +obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o kexec_image.o machine_kexec_file.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o -obj-$(CONFIG_CFI_CLANG) += cfi.o +obj-$(CONFIG_CFI) += cfi.o obj-$(CONFIG_EFI) += efi.o obj-$(CONFIG_COMPAT) += compat_syscall_table.o @@ -125,3 +127,4 @@ obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o +obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c index 2fd29695a788..068e0b404b6f 100644 --- a/arch/riscv/kernel/acpi.c +++ b/arch/riscv/kernel/acpi.c @@ -14,6 +14,7 @@ */ #include <linux/acpi.h> +#include <linux/efi-bgrt.h> #include <linux/efi.h> #include <linux/io.h> #include <linux/memblock.h> @@ -68,7 +69,7 @@ static int __init acpi_fadt_sanity_check(void) /* * FADT is required on riscv; retrieve it to check its presence - * and carry out revision and ACPI HW reduced compliancy tests + * and carry out revision and ACPI HW reduced compliance tests */ status = acpi_get_table(ACPI_SIG_FADT, 0, &table); if (ACPI_FAILURE(status)) { @@ -84,12 +85,12 @@ static int __init acpi_fadt_sanity_check(void) * The revision in the table header is the FADT's Major revision. The * FADT also has a minor revision, which is stored in the FADT itself. * - * TODO: Currently, we check for 6.5 as the minimum version to check - * for HW_REDUCED flag. However, once RISC-V updates are released in - * the ACPI spec, we need to update this check for exact minor revision + * ACPI 6.6 is required for RISC-V as it introduces RISC-V specific + * tables such as RHCT (RISC-V Hart Capabilities Table) and RIMT + * (RISC-V I/O Mapping Table). */ - if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 5)) - pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.5+\n", + if (table->revision < 6 || (table->revision == 6 && fadt->minor_revision < 6)) + pr_err(FW_BUG "Unsupported FADT revision %d.%d, should be 6.6+\n", table->revision, fadt->minor_revision); if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) { @@ -160,6 +161,8 @@ done: early_init_dt_scan_chosen_stdout(); } else { acpi_parse_spcr(earlycon_acpi_spcr_enable, true); + if (IS_ENABLED(CONFIG_ACPI_BGRT)) + acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt); } } @@ -305,7 +308,7 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) } } - return ioremap_prot(phys, size, pgprot_val(prot)); + return ioremap_prot(phys, size, prot); } #ifdef CONFIG_PCI @@ -334,3 +337,19 @@ int raw_pci_write(unsigned int domain, unsigned int bus, } #endif /* CONFIG_PCI */ + +int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) +{ + struct acpi_madt_rintc *rintc; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + rintc = acpi_cpu_get_madt_rintc(cpu); + if (!rintc) + return -ENODEV; + + *uid = rintc->uid; + return 0; +} +EXPORT_SYMBOL_GPL(acpi_get_cpu_uid); diff --git a/arch/riscv/kernel/acpi_numa.c b/arch/riscv/kernel/acpi_numa.c index 130769e3a99c..6a2d4289f806 100644 --- a/arch/riscv/kernel/acpi_numa.c +++ b/arch/riscv/kernel/acpi_numa.c @@ -37,11 +37,14 @@ static int __init acpi_numa_get_nid(unsigned int cpu) static inline int get_cpu_for_acpi_id(u32 uid) { - int cpu; + u32 cpu_uid; + int ret; - for (cpu = 0; cpu < nr_cpu_ids; cpu++) - if (uid == get_acpi_id_for_cpu(cpu)) + for (int cpu = 0; cpu < nr_cpu_ids; cpu++) { + ret = acpi_get_cpu_uid(cpu, &cpu_uid); + if (ret == 0 && uid == cpu_uid) return cpu; + } return -EINVAL; } diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 7eb3cb1215c6..7642704c7f18 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -47,6 +47,11 @@ static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info cpu_mfr_info->patch_func = andes_errata_patch_func; break; #endif +#ifdef CONFIG_ERRATA_MIPS + case MIPS_VENDOR_ID: + cpu_mfr_info->patch_func = mips_errata_patch_func; + break; +#endif #ifdef CONFIG_ERRATA_SIFIVE case SIFIVE_VENDOR_ID: cpu_mfr_info->patch_func = sifive_errata_patch_func; diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index e89455a6a0e5..af827448a609 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -3,6 +3,7 @@ * Copyright (C) 2012 Regents of the University of California * Copyright (C) 2017 SiFive */ +#define COMPILE_OFFSETS #include <linux/kbuild.h> #include <linux/mm.h> @@ -34,9 +35,9 @@ void asm_offsets(void) OFFSET(TASK_THREAD_S9, task_struct, thread.s[9]); OFFSET(TASK_THREAD_S10, task_struct, thread.s[10]); OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]); + OFFSET(TASK_THREAD_SUM, task_struct, thread.sum); OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); - OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags); OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); @@ -50,6 +51,10 @@ void asm_offsets(void) #endif OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu); +#ifdef CONFIG_RISCV_USER_CFI + OFFSET(TASK_TI_CFI_STATE, task_struct, thread_info.user_cfi_state); + OFFSET(TASK_TI_USER_SSP, task_struct, thread_info.user_cfi_state.user_shdw_stk); +#endif OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]); OFFSET(TASK_THREAD_F2, task_struct, thread.fstate.f[2]); @@ -347,6 +352,10 @@ void asm_offsets(void) offsetof(struct task_struct, thread.s[11]) - offsetof(struct task_struct, thread.ra) ); + DEFINE(TASK_THREAD_SUM_RA, + offsetof(struct task_struct, thread.sum) + - offsetof(struct task_struct, thread.ra) + ); DEFINE(TASK_THREAD_F0_F0, offsetof(struct task_struct, thread.fstate.f[0]) @@ -494,6 +503,12 @@ void asm_offsets(void) DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe), STACK_ALIGN)); OFFSET(STACKFRAME_FP, stackframe, fp); OFFSET(STACKFRAME_RA, stackframe, ra); +#ifdef CONFIG_FUNCTION_TRACER + DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func)); +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call)); +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +#endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS DEFINE(FREGS_SIZE_ON_STACK, ALIGN(sizeof(struct __arch_ftrace_regs), STACK_ALIGN)); @@ -502,6 +517,13 @@ void asm_offsets(void) DEFINE(FREGS_SP, offsetof(struct __arch_ftrace_regs, sp)); DEFINE(FREGS_S0, offsetof(struct __arch_ftrace_regs, s0)); DEFINE(FREGS_T1, offsetof(struct __arch_ftrace_regs, t1)); +#ifdef CONFIG_CC_IS_CLANG + DEFINE(FREGS_T2, offsetof(struct __arch_ftrace_regs, t2)); + DEFINE(FREGS_T3, offsetof(struct __arch_ftrace_regs, t3)); + DEFINE(FREGS_T4, offsetof(struct __arch_ftrace_regs, t4)); + DEFINE(FREGS_T5, offsetof(struct __arch_ftrace_regs, t5)); + DEFINE(FREGS_T6, offsetof(struct __arch_ftrace_regs, t6)); +#endif DEFINE(FREGS_A0, offsetof(struct __arch_ftrace_regs, a0)); DEFINE(FREGS_A1, offsetof(struct __arch_ftrace_regs, a1)); DEFINE(FREGS_A2, offsetof(struct __arch_ftrace_regs, a2)); @@ -511,4 +533,10 @@ void asm_offsets(void) DEFINE(FREGS_A6, offsetof(struct __arch_ftrace_regs, a6)); DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif +#ifdef CONFIG_RISCV_SBI + DEFINE(SBI_EXT_FWFT, SBI_EXT_FWFT); + DEFINE(SBI_EXT_FWFT_SET, SBI_EXT_FWFT_SET); + DEFINE(SBI_FWFT_SHADOW_STACK, SBI_FWFT_SHADOW_STACK); + DEFINE(SBI_FWFT_SET_FLAG_LOCK, SBI_FWFT_SET_FLAG_LOCK); +#endif } diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c index 2d40736fc37c..26b085dbdd07 100644 --- a/arch/riscv/kernel/cacheinfo.c +++ b/arch/riscv/kernel/cacheinfo.c @@ -108,11 +108,11 @@ int populate_cache_leaves(unsigned int cpu) if (!np) return -ENOENT; - if (of_property_read_bool(np, "cache-size")) + if (of_property_present(np, "cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_UNIFIED, level); - if (of_property_read_bool(np, "i-cache-size")) + if (of_property_present(np, "i-cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); - if (of_property_read_bool(np, "d-cache-size")) + if (of_property_present(np, "d-cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); prev = np; @@ -125,11 +125,11 @@ int populate_cache_leaves(unsigned int cpu) break; if (level <= levels) break; - if (of_property_read_bool(np, "cache-size")) + if (of_property_present(np, "cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_UNIFIED, level); - if (of_property_read_bool(np, "i-cache-size")) + if (of_property_present(np, "i-cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); - if (of_property_read_bool(np, "d-cache-size")) + if (of_property_present(np, "d-cache-size")) ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); levels = level; } diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 64bdd3e1ab8c..6ec9dbd7292e 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -75,56 +75,3 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) return report_cfi_failure(regs, regs->epc, &target, type); } - -#ifdef CONFIG_CFI_CLANG -struct bpf_insn; - -/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */ -extern unsigned int __bpf_prog_runX(const void *ctx, - const struct bpf_insn *insn); - -/* - * Force a reference to the external symbol so the compiler generates - * __kcfi_typid. - */ -__ADDRESSABLE(__bpf_prog_runX); - -/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ -asm ( -" .pushsection .data..ro_after_init,\"aw\",@progbits \n" -" .type cfi_bpf_hash,@object \n" -" .globl cfi_bpf_hash \n" -" .p2align 2, 0x0 \n" -"cfi_bpf_hash: \n" -" .word __kcfi_typeid___bpf_prog_runX \n" -" .size cfi_bpf_hash, 4 \n" -" .popsection \n" -); - -/* Must match bpf_callback_t */ -extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64); - -__ADDRESSABLE(__bpf_callback_fn); - -/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ -asm ( -" .pushsection .data..ro_after_init,\"aw\",@progbits \n" -" .type cfi_bpf_subprog_hash,@object \n" -" .globl cfi_bpf_subprog_hash \n" -" .p2align 2, 0x0 \n" -"cfi_bpf_subprog_hash: \n" -" .word __kcfi_typeid___bpf_callback_fn \n" -" .size cfi_bpf_subprog_hash, 4 \n" -" .popsection \n" -); - -u32 cfi_get_func_hash(void *func) -{ - u32 hash; - - if (get_kernel_nofault(hash, func - cfi_get_offset())) - return 0; - - return hash; -} -#endif diff --git a/arch/riscv/kernel/compat_signal.c b/arch/riscv/kernel/compat_signal.c index 6ec4e34255a9..cf3eb33a11e4 100644 --- a/arch/riscv/kernel/compat_signal.c +++ b/arch/riscv/kernel/compat_signal.c @@ -107,6 +107,8 @@ static long compat_restore_sigcontext(struct pt_regs *regs, /* sc_regs is structured the same as the start of pt_regs */ err = __copy_from_user(&cregs, &sc->sc_regs, sizeof(sc->sc_regs)); + if (unlikely(err)) + return err; cregs_to_regs(&cregs, regs); diff --git a/arch/riscv/kernel/copy-unaligned.S b/arch/riscv/kernel/copy-unaligned.S index 2b3d9398c113..90f3549621f7 100644 --- a/arch/riscv/kernel/copy-unaligned.S +++ b/arch/riscv/kernel/copy-unaligned.S @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2023 Rivos Inc. */ +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -9,7 +10,7 @@ /* void __riscv_copy_words_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using word loads and stores. */ /* Note: The size is truncated to a multiple of 8 * SZREG */ -SYM_FUNC_START(__riscv_copy_words_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_words_unaligned) andi a4, a2, ~((8*SZREG)-1) beqz a4, 2f add a3, a1, a4 @@ -41,7 +42,7 @@ SYM_FUNC_END(__riscv_copy_words_unaligned) /* void __riscv_copy_bytes_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using only byte accesses. */ /* Note: The size is truncated to a multiple of 8 */ -SYM_FUNC_START(__riscv_copy_bytes_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_bytes_unaligned) andi a4, a2, ~(8-1) beqz a4, 2f add a3, a1, a4 diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index a1e38ecfc8be..a0ee426f6d93 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -43,7 +43,6 @@ int __cpu_disable(void) return 0; } -#ifdef CONFIG_HOTPLUG_CPU /* * Called on the thread which is asking for a CPU to be shutdown, if the * CPU reported dead to the hotplug core. @@ -54,6 +53,7 @@ void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) pr_notice("CPU%u: off\n", cpu); + clear_tasks_mm_cpumask(cpu); /* Verify from the firmware if the cpu is really stopped*/ if (cpu_ops->cpu_is_stopped) ret = cpu_ops->cpu_is_stopped(cpu); @@ -74,4 +74,3 @@ void __noreturn arch_cpu_idle_dead(void) /* It should never reach here */ BUG(); } -#endif diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index f6b13e9f5e6c..3dbc8cc557dd 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -62,10 +62,8 @@ int __init riscv_early_of_processor_hartid(struct device_node *node, unsigned lo return -ENODEV; } - if (!of_device_is_available(node)) { - pr_info("CPU with hartid=%lu is not available\n", *hart); + if (!of_device_is_available(node)) return -ENODEV; - } if (of_property_read_string(node, "riscv,isa-base", &isa)) goto old_interface; diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index e6fbaaf54956..00aff669f5f2 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -18,10 +18,10 @@ const struct cpu_operations cpu_ops_sbi; /* * Ordered booting via HSM brings one cpu at a time. However, cpu hotplug can - * be invoked from multiple threads in parallel. Define a per cpu data + * be invoked from multiple threads in parallel. Define an array of boot data * to handle that. */ -static DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); +static struct sbi_hart_boot_data boot_data[NR_CPUS]; static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, unsigned long priv) @@ -67,7 +67,7 @@ static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle) unsigned long boot_addr = __pa_symbol(secondary_start_sbi); unsigned long hartid = cpuid_to_hartid_map(cpuid); unsigned long hsm_data; - struct sbi_hart_boot_data *bdata = &per_cpu(boot_data, cpuid); + struct sbi_hart_boot_data *bdata = &boot_data[cpuid]; /* Make sure tidle is updated */ smp_mb(); @@ -85,7 +85,7 @@ static void sbi_cpu_stop(void) int ret; ret = sbi_hsm_hart_stop(); - pr_crit("Unable to stop the cpu %u (%d)\n", smp_processor_id(), ret); + pr_crit("Unable to stop the cpu %d (%d)\n", smp_processor_id(), ret); } static int sbi_cpu_is_stopped(unsigned int cpuid) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index c6ba750536c3..f46aa5602d74 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -28,10 +28,13 @@ #include <asm/vector.h> #include <asm/vendor_extensions.h> #include <asm/vendor_extensions/thead.h> +#include <asm/usercfi.h> #define NUM_ALPHA_EXTS ('z' - 'a' + 1) static bool any_cpu_has_zicboz; +static bool any_cpu_has_zicbop; +static bool any_cpu_has_zicbom; unsigned long elf_hwcap __read_mostly; @@ -53,9 +56,7 @@ u32 thead_vlenb_of; */ unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) { - if (!isa_bitmap) - return riscv_isa[0]; - return isa_bitmap[0]; + return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0]; } EXPORT_SYMBOL_GPL(riscv_isa_extension_base); @@ -76,10 +77,19 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i if (bit >= RISCV_ISA_EXT_MAX) return false; - return test_bit(bit, bmap) ? true : false; + return test_bit(bit, bmap); } EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); +static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f)) + return 0; + + return -EPROBE_DEFER; +} + static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -91,6 +101,8 @@ static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data, pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); return -EINVAL; } + + any_cpu_has_zicbom = true; return 0; } @@ -109,6 +121,97 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!riscv_cbop_block_size) { + pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n"); + return -EINVAL; + } + if (!is_power_of_2(riscv_cbop_block_size)) { + pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n"); + return -EINVAL; + } + any_cpu_has_zicbop = true; + return 0; +} + +static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + /* + * Due to extension ordering, d is checked before f, so no deferral + * is required. + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) { + pr_warn_once("This kernel does not support systems with F but not D\n"); + return -EINVAL; + } + + return 0; +} + +static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + return 0; +} + +static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + return 0; +} + +static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_FPU)) + return -EINVAL; + + /* + * The kernel doesn't support systems that don't implement both of + * F and D, so if any of the vector extensions that do floating point + * are to be usable, both floating point extensions need to be usable. + * + * Since this function validates vector only, and v/Zve* are probed + * after f/d, there's no need for a deferral here. + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) + return -EINVAL; + + return 0; +} + +static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + /* + * It isn't the kernel's job to check that the binding is correct, so + * it should be enough to check that any of the vector extensions are + * enabled, which in-turn means that vector is usable in this kernel + */ + if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X)) + return -EPROBE_DEFER; + + return 0; +} + static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -140,6 +243,50 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data, return -EPROBE_DEFER; } +static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return -EINVAL; + + return 0; +} + +static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return -EINVAL; + + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) + return -EINVAL; + + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F)) + return 0; + + return -EPROBE_DEFER; +} + +static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) && + __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN)) + return 0; + + return -EPROBE_DEFER; +} + static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap) { @@ -150,23 +297,47 @@ static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data, return 0; } +static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || + (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)) + return -EINVAL; + + return 0; +} + +static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data, + const unsigned long *isa_bitmap) +{ + if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) || + (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)) + return -EINVAL; + + return 0; +} + +static const unsigned int riscv_a_exts[] = { + RISCV_ISA_EXT_ZAAMO, + RISCV_ISA_EXT_ZALRSC, +}; + +#define RISCV_ISA_EXT_ZKN \ + RISCV_ISA_EXT_ZBKB, \ + RISCV_ISA_EXT_ZBKC, \ + RISCV_ISA_EXT_ZBKX, \ + RISCV_ISA_EXT_ZKND, \ + RISCV_ISA_EXT_ZKNE, \ + RISCV_ISA_EXT_ZKNH + static const unsigned int riscv_zk_bundled_exts[] = { - RISCV_ISA_EXT_ZBKB, - RISCV_ISA_EXT_ZBKC, - RISCV_ISA_EXT_ZBKX, - RISCV_ISA_EXT_ZKND, - RISCV_ISA_EXT_ZKNE, + RISCV_ISA_EXT_ZKN, RISCV_ISA_EXT_ZKR, - RISCV_ISA_EXT_ZKT, + RISCV_ISA_EXT_ZKT }; static const unsigned int riscv_zkn_bundled_exts[] = { - RISCV_ISA_EXT_ZBKB, - RISCV_ISA_EXT_ZBKC, - RISCV_ISA_EXT_ZBKX, - RISCV_ISA_EXT_ZKND, - RISCV_ISA_EXT_ZKNE, - RISCV_ISA_EXT_ZKNH, + RISCV_ISA_EXT_ZKN }; static const unsigned int riscv_zks_bundled_exts[] = { @@ -321,18 +492,21 @@ static const unsigned int riscv_c_exts[] = { const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), - __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), - __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), - __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), + __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts), + __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate), __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts), - __RISCV_ISA_EXT_SUPERSET(v, RISCV_ISA_EXT_v, riscv_v_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, - riscv_ext_zicbom_validate), - __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, - riscv_ext_zicboz_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate), __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts, + riscv_cfilp_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts, + riscv_cfiss_validate), __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), @@ -341,17 +515,23 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP), + __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR), + __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), - __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), - __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), - __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), + __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends), + __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends), + __RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends), + __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends), __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA), __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends), __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate), __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends), + __RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate), __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), @@ -370,29 +550,31 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), - __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), - __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), - __RISCV_ISA_EXT_SUPERSET(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts), - __RISCV_ISA_EXT_DATA(zve32x, RISCV_ISA_EXT_ZVE32X), - __RISCV_ISA_EXT_SUPERSET(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts), - __RISCV_ISA_EXT_SUPERSET(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts), - __RISCV_ISA_EXT_SUPERSET(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate), + __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate), __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), - __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), - __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), - __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), - __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), - __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), - __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), - __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), - __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), - __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), - __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), - __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), - __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), - __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), - __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate), + __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate), __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM), __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts), @@ -406,6 +588,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), + __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), }; @@ -479,7 +662,7 @@ static void __init riscv_resolve_isa(unsigned long *source_isa, if (bit < RISCV_ISA_EXT_BASE) *this_hwcap |= isa2hwcap[bit]; } - } while (loop && memcmp(prev_resolved_isa, resolved_isa, sizeof(prev_resolved_isa))); + } while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX)); } static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap) @@ -713,10 +896,8 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) * CPU cores with the ratified spec will contain non-zero * marchid. */ - if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) { - this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; + if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) clear_bit(RISCV_ISA_EXT_v, source_isa); - } riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); @@ -799,9 +980,9 @@ static int has_thead_homogeneous_vlenb(void) { int cpu; u32 prev_vlenb = 0; - u32 vlenb; + u32 vlenb = 0; - /* Ignore thead,vlenb property if xtheavector is not enabled in the kernel */ + /* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */ if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR)) return 0; @@ -921,16 +1102,16 @@ early_param("riscv_isa_fallback", riscv_isa_fallback_setup); void __init riscv_fill_hwcap(void) { char print_str[NUM_ALPHA_EXTS + 1]; - unsigned long isa2hwcap[26] = {0}; + unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0}; int i, j; - isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; - isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; - isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; - isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; - isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; - isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; - isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; + isa2hwcap[RISCV_ISA_EXT_i] = COMPAT_HWCAP_ISA_I; + isa2hwcap[RISCV_ISA_EXT_m] = COMPAT_HWCAP_ISA_M; + isa2hwcap[RISCV_ISA_EXT_a] = COMPAT_HWCAP_ISA_A; + isa2hwcap[RISCV_ISA_EXT_f] = COMPAT_HWCAP_ISA_F; + isa2hwcap[RISCV_ISA_EXT_d] = COMPAT_HWCAP_ISA_D; + isa2hwcap[RISCV_ISA_EXT_c] = COMPAT_HWCAP_ISA_C; + isa2hwcap[RISCV_ISA_EXT_v] = COMPAT_HWCAP_ISA_V; if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); @@ -960,16 +1141,6 @@ void __init riscv_fill_hwcap(void) riscv_v_setup_vsize(); } - if (elf_hwcap & COMPAT_HWCAP_ISA_V) { - /* - * ISA string in device tree might have 'v' flag, but - * CONFIG_RISCV_ISA_V is disabled in kernel. - * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. - */ - if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) - elf_hwcap &= ~COMPAT_HWCAP_ISA_V; - } - memset(print_str, 0, sizeof(print_str)); for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) if (riscv_isa[0] & BIT_MASK(i)) @@ -1001,6 +1172,15 @@ void __init riscv_user_isa_enable(void) current->thread.envcfg |= ENVCFG_CBZE; else if (any_cpu_has_zicboz) pr_warn("Zicboz disabled as it is unavailable on some harts\n"); + + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM)) + current->thread.envcfg |= ENVCFG_CBCFE; + else if (any_cpu_has_zicbom) + pr_warn("Zicbom disabled as it is unavailable on some harts\n"); + + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) && + any_cpu_has_zicbop) + pr_warn("Zicbop disabled as it is unavailable on some harts\n"); } #ifdef CONFIG_RISCV_ALTERNATIVE diff --git a/arch/riscv/kernel/efi-header.S b/arch/riscv/kernel/efi-header.S index c5f17c2710b5..2efc3aaf4a8c 100644 --- a/arch/riscv/kernel/efi-header.S +++ b/arch/riscv/kernel/efi-header.S @@ -9,7 +9,7 @@ #include <asm/set_memory.h> .macro __EFI_PE_HEADER - .long PE_MAGIC + .long IMAGE_NT_SIGNATURE coff_header: #ifdef CONFIG_64BIT .short IMAGE_FILE_MACHINE_RISCV64 // Machine @@ -27,9 +27,9 @@ coff_header: optional_header: #ifdef CONFIG_64BIT - .short PE_OPT_MAGIC_PE32PLUS // PE32+ format + .short IMAGE_NT_OPTIONAL_HDR64_MAGIC // PE32+ format #else - .short PE_OPT_MAGIC_PE32 // PE32 format + .short IMAGE_NT_OPTIONAL_HDR32_MAGIC // PE32 format #endif .byte 0x02 // MajorLinkerVersion .byte 0x14 // MinorLinkerVersion @@ -64,7 +64,7 @@ extra_header_fields: .long efi_header_end - _start // SizeOfHeaders .long 0 // CheckSum .short IMAGE_SUBSYSTEM_EFI_APPLICATION // Subsystem - .short IMAGE_DLL_CHARACTERISTICS_NX_COMPAT // DllCharacteristics + .short IMAGE_DLLCHARACTERISTICS_NX_COMPAT // DllCharacteristics .quad 0 // SizeOfStackReserve .quad 0 // SizeOfStackCommit .quad 0 // SizeOfHeapReserve diff --git a/arch/riscv/kernel/elf_kexec.c b/arch/riscv/kernel/elf_kexec.c deleted file mode 100644 index 3c37661801f9..000000000000 --- a/arch/riscv/kernel/elf_kexec.c +++ /dev/null @@ -1,482 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Load ELF vmlinux file for the kexec_file_load syscall. - * - * Copyright (C) 2021 Huawei Technologies Co, Ltd. - * - * Author: Liao Chang (liaochang1@huawei.com) - * - * Based on kexec-tools' kexec-elf-riscv.c, heavily modified - * for kernel. - */ - -#define pr_fmt(fmt) "kexec_image: " fmt - -#include <linux/elf.h> -#include <linux/kexec.h> -#include <linux/slab.h> -#include <linux/of.h> -#include <linux/libfdt.h> -#include <linux/types.h> -#include <linux/memblock.h> -#include <linux/vmalloc.h> -#include <asm/setup.h> - -int arch_kimage_file_post_load_cleanup(struct kimage *image) -{ - kvfree(image->arch.fdt); - image->arch.fdt = NULL; - - vfree(image->elf_headers); - image->elf_headers = NULL; - image->elf_headers_sz = 0; - - return kexec_image_post_load_cleanup_default(image); -} - -static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, - struct kexec_elf_info *elf_info, unsigned long old_pbase, - unsigned long new_pbase) -{ - int i; - int ret = 0; - size_t size; - struct kexec_buf kbuf; - const struct elf_phdr *phdr; - - kbuf.image = image; - - for (i = 0; i < ehdr->e_phnum; i++) { - phdr = &elf_info->proghdrs[i]; - if (phdr->p_type != PT_LOAD) - continue; - - size = phdr->p_filesz; - if (size > phdr->p_memsz) - size = phdr->p_memsz; - - kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset; - kbuf.bufsz = size; - kbuf.buf_align = phdr->p_align; - kbuf.mem = phdr->p_paddr - old_pbase + new_pbase; - kbuf.memsz = phdr->p_memsz; - kbuf.top_down = false; - ret = kexec_add_buffer(&kbuf); - if (ret) - break; - } - - return ret; -} - -/* - * Go through the available phsyical memory regions and find one that hold - * an image of the specified size. - */ -static int elf_find_pbase(struct kimage *image, unsigned long kernel_len, - struct elfhdr *ehdr, struct kexec_elf_info *elf_info, - unsigned long *old_pbase, unsigned long *new_pbase) -{ - int i; - int ret; - struct kexec_buf kbuf; - const struct elf_phdr *phdr; - unsigned long lowest_paddr = ULONG_MAX; - unsigned long lowest_vaddr = ULONG_MAX; - - for (i = 0; i < ehdr->e_phnum; i++) { - phdr = &elf_info->proghdrs[i]; - if (phdr->p_type != PT_LOAD) - continue; - - if (lowest_paddr > phdr->p_paddr) - lowest_paddr = phdr->p_paddr; - - if (lowest_vaddr > phdr->p_vaddr) - lowest_vaddr = phdr->p_vaddr; - } - - kbuf.image = image; - kbuf.buf_min = lowest_paddr; - kbuf.buf_max = ULONG_MAX; - - /* - * Current riscv boot protocol requires 2MB alignment for - * RV64 and 4MB alignment for RV32 - * - */ - kbuf.buf_align = PMD_SIZE; - kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; - kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE); - kbuf.top_down = false; - ret = arch_kexec_locate_mem_hole(&kbuf); - if (!ret) { - *old_pbase = lowest_paddr; - *new_pbase = kbuf.mem; - image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem; - } - return ret; -} - -#ifdef CONFIG_CRASH_DUMP -static int get_nr_ram_ranges_callback(struct resource *res, void *arg) -{ - unsigned int *nr_ranges = arg; - - (*nr_ranges)++; - return 0; -} - -static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) -{ - struct crash_mem *cmem = arg; - - cmem->ranges[cmem->nr_ranges].start = res->start; - cmem->ranges[cmem->nr_ranges].end = res->end; - cmem->nr_ranges++; - - return 0; -} - -static int prepare_elf_headers(void **addr, unsigned long *sz) -{ - struct crash_mem *cmem; - unsigned int nr_ranges; - int ret; - - nr_ranges = 1; /* For exclusion of crashkernel region */ - walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); - - cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); - if (!cmem) - return -ENOMEM; - - cmem->max_nr_ranges = nr_ranges; - cmem->nr_ranges = 0; - ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); - if (ret) - goto out; - - /* Exclude crashkernel region */ - ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); - -out: - kfree(cmem); - return ret; -} - -static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, - unsigned long cmdline_len) -{ - int elfcorehdr_strlen; - char *cmdline_ptr; - - cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL); - if (!cmdline_ptr) - return NULL; - - elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ", - image->elf_load_addr); - - if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) { - pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n"); - kfree(cmdline_ptr); - return NULL; - } - - memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len); - /* Ensure it's nul terminated */ - cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0'; - return cmdline_ptr; -} -#endif - -static void *elf_kexec_load(struct kimage *image, char *kernel_buf, - unsigned long kernel_len, char *initrd, - unsigned long initrd_len, char *cmdline, - unsigned long cmdline_len) -{ - int ret; - void *fdt; - unsigned long old_kernel_pbase = ULONG_MAX; - unsigned long new_kernel_pbase = 0UL; - unsigned long initrd_pbase = 0UL; - unsigned long kernel_start; - struct elfhdr ehdr; - struct kexec_buf kbuf; - struct kexec_elf_info elf_info; - char *modified_cmdline = NULL; - - ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info); - if (ret) - return ERR_PTR(ret); - - ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info, - &old_kernel_pbase, &new_kernel_pbase); - if (ret) - goto out; - kernel_start = image->start; - - /* Add the kernel binary to the image */ - ret = riscv_kexec_elf_load(image, &ehdr, &elf_info, - old_kernel_pbase, new_kernel_pbase); - if (ret) - goto out; - - kbuf.image = image; - kbuf.buf_min = new_kernel_pbase + kernel_len; - kbuf.buf_max = ULONG_MAX; - -#ifdef CONFIG_CRASH_DUMP - /* Add elfcorehdr */ - if (image->type == KEXEC_TYPE_CRASH) { - void *headers; - unsigned long headers_sz; - ret = prepare_elf_headers(&headers, &headers_sz); - if (ret) { - pr_err("Preparing elf core header failed\n"); - goto out; - } - - kbuf.buffer = headers; - kbuf.bufsz = headers_sz; - kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; - kbuf.memsz = headers_sz; - kbuf.buf_align = ELF_CORE_HEADER_ALIGN; - kbuf.top_down = true; - - ret = kexec_add_buffer(&kbuf); - if (ret) { - vfree(headers); - goto out; - } - image->elf_headers = headers; - image->elf_load_addr = kbuf.mem; - image->elf_headers_sz = headers_sz; - - kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n", - image->elf_load_addr, kbuf.bufsz, kbuf.memsz); - - /* Setup cmdline for kdump kernel case */ - modified_cmdline = setup_kdump_cmdline(image, cmdline, - cmdline_len); - if (!modified_cmdline) { - pr_err("Setting up cmdline for kdump kernel failed\n"); - ret = -EINVAL; - goto out; - } - cmdline = modified_cmdline; - } -#endif - -#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY - /* Add purgatory to the image */ - kbuf.top_down = true; - kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; - ret = kexec_load_purgatory(image, &kbuf); - if (ret) { - pr_err("Error loading purgatory ret=%d\n", ret); - goto out; - } - kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem); - - ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry", - &kernel_start, - sizeof(kernel_start), 0); - if (ret) - pr_err("Error update purgatory ret=%d\n", ret); -#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */ - - /* Add the initrd to the image */ - if (initrd != NULL) { - kbuf.buffer = initrd; - kbuf.bufsz = kbuf.memsz = initrd_len; - kbuf.buf_align = PAGE_SIZE; - kbuf.top_down = true; - kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; - ret = kexec_add_buffer(&kbuf); - if (ret) - goto out; - initrd_pbase = kbuf.mem; - kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase); - } - - /* Add the DTB to the image */ - fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase, - initrd_len, cmdline, 0); - if (!fdt) { - pr_err("Error setting up the new device tree.\n"); - ret = -EINVAL; - goto out; - } - - fdt_pack(fdt); - kbuf.buffer = fdt; - kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt); - kbuf.buf_align = PAGE_SIZE; - kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; - kbuf.top_down = true; - ret = kexec_add_buffer(&kbuf); - if (ret) { - pr_err("Error add DTB kbuf ret=%d\n", ret); - goto out_free_fdt; - } - /* Cache the fdt buffer address for memory cleanup */ - image->arch.fdt = fdt; - kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem); - goto out; - -out_free_fdt: - kvfree(fdt); -out: - kfree(modified_cmdline); - kexec_free_elf_info(&elf_info); - return ret ? ERR_PTR(ret) : NULL; -} - -#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) -#define RISCV_IMM_BITS 12 -#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS) -#define RISCV_CONST_HIGH_PART(x) \ - (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1)) -#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x)) - -#define ENCODE_ITYPE_IMM(x) \ - (RV_X(x, 0, 12) << 20) -#define ENCODE_BTYPE_IMM(x) \ - ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \ - (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31)) -#define ENCODE_UTYPE_IMM(x) \ - (RV_X(x, 12, 20) << 12) -#define ENCODE_JTYPE_IMM(x) \ - ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \ - (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31)) -#define ENCODE_CBTYPE_IMM(x) \ - ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \ - (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12)) -#define ENCODE_CJTYPE_IMM(x) \ - ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \ - (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \ - (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12)) -#define ENCODE_UJTYPE_IMM(x) \ - (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \ - (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32)) -#define ENCODE_UITYPE_IMM(x) \ - (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32)) - -#define CLEAN_IMM(type, x) \ - ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x)) - -int arch_kexec_apply_relocations_add(struct purgatory_info *pi, - Elf_Shdr *section, - const Elf_Shdr *relsec, - const Elf_Shdr *symtab) -{ - const char *strtab, *name, *shstrtab; - const Elf_Shdr *sechdrs; - Elf64_Rela *relas; - int i, r_type; - - /* String & section header string table */ - sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff; - strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset; - shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset; - - relas = (void *)pi->ehdr + relsec->sh_offset; - - for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) { - const Elf_Sym *sym; /* symbol to relocate */ - unsigned long addr; /* final location after relocation */ - unsigned long val; /* relocated symbol value */ - unsigned long sec_base; /* relocated symbol value */ - void *loc; /* tmp location to modify */ - - sym = (void *)pi->ehdr + symtab->sh_offset; - sym += ELF64_R_SYM(relas[i].r_info); - - if (sym->st_name) - name = strtab + sym->st_name; - else - name = shstrtab + sechdrs[sym->st_shndx].sh_name; - - loc = pi->purgatory_buf; - loc += section->sh_offset; - loc += relas[i].r_offset; - - if (sym->st_shndx == SHN_ABS) - sec_base = 0; - else if (sym->st_shndx >= pi->ehdr->e_shnum) { - pr_err("Invalid section %d for symbol %s\n", - sym->st_shndx, name); - return -ENOEXEC; - } else - sec_base = pi->sechdrs[sym->st_shndx].sh_addr; - - val = sym->st_value; - val += sec_base; - val += relas[i].r_addend; - - addr = section->sh_addr + relas[i].r_offset; - - r_type = ELF64_R_TYPE(relas[i].r_info); - - switch (r_type) { - case R_RISCV_BRANCH: - *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) | - ENCODE_BTYPE_IMM(val - addr); - break; - case R_RISCV_JAL: - *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) | - ENCODE_JTYPE_IMM(val - addr); - break; - /* - * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I - * sym is expected to be next to R_RISCV_PCREL_HI20 - * in purgatory relsec. Handle it like R_RISCV_CALL - * sym, instead of searching the whole relsec. - */ - case R_RISCV_PCREL_HI20: - case R_RISCV_CALL_PLT: - case R_RISCV_CALL: - *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) | - ENCODE_UJTYPE_IMM(val - addr); - break; - case R_RISCV_RVC_BRANCH: - *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) | - ENCODE_CBTYPE_IMM(val - addr); - break; - case R_RISCV_RVC_JUMP: - *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | - ENCODE_CJTYPE_IMM(val - addr); - break; - case R_RISCV_ADD16: - *(u16 *)loc += val; - break; - case R_RISCV_SUB16: - *(u16 *)loc -= val; - break; - case R_RISCV_ADD32: - *(u32 *)loc += val; - break; - case R_RISCV_SUB32: - *(u32 *)loc -= val; - break; - /* It has been applied by R_RISCV_PCREL_HI20 sym */ - case R_RISCV_PCREL_LO12_I: - case R_RISCV_ALIGN: - case R_RISCV_RELAX: - break; - default: - pr_err("Unknown rela relocation: %d\n", r_type); - return -ENOEXEC; - } - } - return 0; -} - -const struct kexec_file_ops elf_kexec_ops = { - .probe = kexec_elf_probe, - .load = elf_kexec_load, -}; diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 33a5a9f2a0d4..d011fb51c59a 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -7,6 +7,7 @@ #include <linux/init.h> #include <linux/linkage.h> +#include <asm/alternative-macros.h> #include <asm/asm.h> #include <asm/csr.h> #include <asm/scs.h> @@ -46,7 +47,7 @@ * a0 = &new_vmalloc[BIT_WORD(cpu)] * a1 = BIT_MASK(cpu) */ - REG_L a2, TASK_TI_CPU(tp) + lw a2, TASK_TI_CPU(tp) /* * Compute the new_vmalloc element position: * (cpu / 64) * 8 = (cpu >> 6) << 3 @@ -91,6 +92,35 @@ REG_L a0, TASK_TI_A0(tp) .endm +/* + * If previous mode was U, capture shadow stack pointer and save it away + * Zero CSR_SSP at the same time for sanitization. + */ +.macro save_userssp tmp, status + ALTERNATIVE("nops(4)", + __stringify( \ + andi \tmp, \status, SR_SPP; \ + bnez \tmp, skip_ssp_save; \ + csrrw \tmp, CSR_SSP, x0; \ + REG_S \tmp, TASK_TI_USER_SSP(tp); \ + skip_ssp_save:), + 0, + RISCV_ISA_EXT_ZICFISS, + CONFIG_RISCV_USER_CFI) +.endm + +.macro restore_userssp tmp, status + ALTERNATIVE("nops(4)", + __stringify( \ + andi \tmp, \status, SR_SPP; \ + bnez \tmp, skip_ssp_restore; \ + REG_L \tmp, TASK_TI_USER_SSP(tp); \ + csrw CSR_SSP, \tmp; \ + skip_ssp_restore:), + 0, + RISCV_ISA_EXT_ZICFISS, + CONFIG_RISCV_USER_CFI) +.endm SYM_CODE_START(handle_exception) /* @@ -144,9 +174,14 @@ SYM_CODE_START(handle_exception) * or vector in kernel space. */ li t0, SR_SUM | SR_FS_VS +#ifdef CONFIG_64BIT + li t1, SR_ELP + or t0, t0, t1 +#endif REG_L s0, TASK_TI_USER_SP(tp) csrrc s1, CSR_STATUS, t0 + save_userssp s2, s1 csrr s2, CSR_EPC csrr s3, CSR_TVAL csrr s4, CSR_CAUSE @@ -220,7 +255,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception) #endif bnez s0, 1f -#ifdef CONFIG_GCC_PLUGIN_STACKLEAK +#ifdef CONFIG_KSTACK_ERASE call stackleak_erase_on_task_stack #endif @@ -242,6 +277,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception) call riscv_v_context_nesting_end #endif REG_L a0, PT_STATUS(sp) + restore_userssp s3, a0 /* * The current load reservation is effectively part of the processor's * state, in the sense that load reservations cannot be shared between @@ -319,17 +355,21 @@ SYM_CODE_END(handle_kernel_stack_overflow) ASM_NOKPROBE(handle_kernel_stack_overflow) #endif -SYM_CODE_START(ret_from_fork) +SYM_CODE_START(ret_from_fork_kernel_asm) + call schedule_tail + move a0, s1 /* fn_arg */ + move a1, s0 /* fn */ + move a2, sp /* pt_regs */ + call ret_from_fork_kernel + j ret_from_exception +SYM_CODE_END(ret_from_fork_kernel_asm) + +SYM_CODE_START(ret_from_fork_user_asm) call schedule_tail - beqz s0, 1f /* not from kernel thread */ - /* Call fn(arg) */ - move a0, s1 - jalr s0 -1: move a0, sp /* pt_regs */ - call syscall_exit_to_user_mode + call ret_from_fork_user j ret_from_exception -SYM_CODE_END(ret_from_fork) +SYM_CODE_END(ret_from_fork_user_asm) #ifdef CONFIG_IRQ_STACKS /* @@ -397,9 +437,18 @@ SYM_FUNC_START(__switch_to) REG_S s9, TASK_THREAD_S9_RA(a3) REG_S s10, TASK_THREAD_S10_RA(a3) REG_S s11, TASK_THREAD_S11_RA(a3) + + /* save the user space access flag */ + csrr s0, CSR_STATUS + REG_S s0, TASK_THREAD_SUM_RA(a3) + /* Save the kernel shadow call stack pointer */ scs_save_current /* Restore context from next->thread */ + REG_L s0, TASK_THREAD_SUM_RA(a4) + li s1, SR_SUM + and s0, s0, s1 + csrs CSR_STATUS, s0 REG_L ra, TASK_THREAD_RA_RA(a4) REG_L sp, TASK_THREAD_SP_RA(a4) REG_L s0, TASK_THREAD_S0_RA(a4) @@ -441,11 +490,15 @@ SYM_DATA_START_LOCAL(excp_vect_table) RISCV_PTR do_trap_ecall_s RISCV_PTR do_trap_unknown RISCV_PTR do_trap_ecall_m - /* instruciton page fault */ + /* instruction page fault */ ALT_PAGE_FAULT(RISCV_PTR do_page_fault) RISCV_PTR do_page_fault /* load page fault */ RISCV_PTR do_trap_unknown RISCV_PTR do_page_fault /* store page fault */ + RISCV_PTR do_trap_unknown /* cause=16 */ + RISCV_PTR do_trap_unknown /* cause=17 */ + RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */ + RISCV_PTR do_trap_hardware_error /* hardware error (19) */ SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end) #ifndef CONFIG_MMU diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 3524db5e4fa0..b430edfb83f4 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -8,98 +8,139 @@ #include <linux/ftrace.h> #include <linux/uaccess.h> #include <linux/memory.h> +#include <linux/irqflags.h> #include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> #ifdef CONFIG_DYNAMIC_FTRACE -void ftrace_arch_code_modify_prepare(void) __acquires(&text_mutex) +void ftrace_arch_code_modify_prepare(void) + __acquires(&text_mutex) { mutex_lock(&text_mutex); - - /* - * The code sequences we use for ftrace can't be patched while the - * kernel is running, so we need to use stop_machine() to modify them - * for now. This doesn't play nice with text_mutex, we use this flag - * to elide the check. - */ - riscv_patch_in_stop_machine = true; } -void ftrace_arch_code_modify_post_process(void) __releases(&text_mutex) +void ftrace_arch_code_modify_post_process(void) + __releases(&text_mutex) { - riscv_patch_in_stop_machine = false; mutex_unlock(&text_mutex); } -static int ftrace_check_current_call(unsigned long hook_pos, - unsigned int *expected) +unsigned long ftrace_call_adjust(unsigned long addr) { - unsigned int replaced[2]; - unsigned int nops[2] = {NOP4, NOP4}; + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) + return addr + 8 + MCOUNT_AUIPC_SIZE; - /* we expect nops at the hook position */ - if (!expected) - expected = nops; + return addr + MCOUNT_AUIPC_SIZE; +} - /* - * Read the text we want to modify; - * return must be -EFAULT on read error - */ - if (copy_from_kernel_nofault(replaced, (void *)hook_pos, - MCOUNT_INSN_SIZE)) - return -EFAULT; +unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip) +{ + return fentry_ip - MCOUNT_AUIPC_SIZE; +} - /* - * Make sure it is what we expect it to be; - * return must be -EINVAL on failed comparison - */ - if (memcmp(expected, replaced, sizeof(replaced))) { - pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n", - (void *)hook_pos, expected[0], expected[1], replaced[0], - replaced[1]); - return -EINVAL; +void arch_ftrace_update_code(int command) +{ + command |= FTRACE_MAY_SLEEP; + ftrace_modify_all_code(command); + flush_icache_all(); +} + +static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate) +{ + unsigned int call[2], offset; + unsigned int replaced[2]; + + offset = target - source; + call[1] = to_jalr_t0(offset); + + if (validate) { + call[0] = to_auipc_t0(offset); + /* + * Read the text we want to modify; + * return must be -EFAULT on read error + */ + if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) + return -EFAULT; + + if (replaced[0] != call[0]) { + pr_err("%p: expected (%08x) but got (%08x)\n", + (void *)source, call[0], replaced[0]); + return -EINVAL; + } } + /* Replace the jalr at once. Return -EPERM on write error. */ + if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1, MCOUNT_JALR_SIZE)) + return -EPERM; + return 0; } -static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, - bool enable, bool ra) +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS +static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec) { - unsigned int call[2]; - unsigned int nops[2] = {NOP4, NOP4}; + const struct ftrace_ops *ops = NULL; - if (ra) - make_call_ra(hook_pos, target, call); - else - make_call_t0(hook_pos, target, call); + if (rec->flags & FTRACE_FL_CALL_OPS_EN) { + ops = ftrace_find_unique_ops(rec); + WARN_ON_ONCE(!ops); + } - /* Replace the auipc-jalr pair at once. Return -EPERM on write error. */ - if (patch_insn_write((void *)hook_pos, enable ? call : nops, MCOUNT_INSN_SIZE)) - return -EPERM; + if (!ops) + ops = &ftrace_list_ops; - return 0; + return ops; +} + +static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops) +{ + unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8); + + return patch_text_nosync((void *)literal, &ops, sizeof(ops)); } +static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) +{ + return ftrace_rec_set_ops(rec, &ftrace_nop_ops); +} + +static int ftrace_rec_update_ops(struct dyn_ftrace *rec) +{ + return ftrace_rec_set_ops(rec, riscv64_rec_get_ops(rec)); +} +#else +static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; } +static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; } +#endif + int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - unsigned int call[2]; + unsigned long distance, orig_addr, pc = rec->ip - MCOUNT_AUIPC_SIZE; + int ret; - make_call_t0(rec->ip, addr, call); + ret = ftrace_rec_update_ops(rec); + if (ret) + return ret; - if (patch_insn_write((void *)rec->ip, call, MCOUNT_INSN_SIZE)) - return -EPERM; + orig_addr = (unsigned long)&ftrace_caller; + distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr; + if (distance > JALR_RANGE) + addr = FTRACE_ADDR; - return 0; + return __ftrace_modify_call(pc, addr, false); } -int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, - unsigned long addr) +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - unsigned int nops[2] = {NOP4, NOP4}; + u32 nop4 = RISCV_INSN_NOP4; + int ret; + + ret = ftrace_rec_set_nop_ops(rec); + if (ret) + return ret; - if (patch_insn_write((void *)rec->ip, nops, MCOUNT_INSN_SIZE)) + if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE)) return -EPERM; return 0; @@ -107,82 +148,78 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, /* * This is called early on, and isn't wrapped by - * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold + * ftrace_arch_code_modify_{prepare,post_process}() and therefore doesn't hold * text_mutex, which triggers a lockdep failure. SMP isn't running so we could * just directly poke the text, but it's simpler to just take the lock * ourselves. */ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { - int out; + unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE; + unsigned int nops[2], offset; + int ret; - mutex_lock(&text_mutex); - out = ftrace_make_nop(mod, rec, MCOUNT_ADDR); - mutex_unlock(&text_mutex); + guard(mutex)(&text_mutex); - return out; -} + ret = ftrace_rec_set_nop_ops(rec); + if (ret) + return ret; -int ftrace_update_ftrace_func(ftrace_func_t func) -{ - int ret = __ftrace_modify_call((unsigned long)&ftrace_call, - (unsigned long)func, true, true); + offset = (unsigned long) &ftrace_caller - pc; + nops[0] = to_auipc_t0(offset); + nops[1] = RISCV_INSN_NOP4; + + ret = patch_insn_write((void *)pc, nops, 2 * MCOUNT_INSN_SIZE); return ret; } -struct ftrace_modify_param { - int command; - atomic_t cpu_count; -}; - -static int __ftrace_modify_code(void *data) +ftrace_func_t ftrace_call_dest = ftrace_stub; +int ftrace_update_ftrace_func(ftrace_func_t func) { - struct ftrace_modify_param *param = data; - - if (atomic_inc_return(¶m->cpu_count) == num_online_cpus()) { - ftrace_modify_all_code(param->command); - /* - * Make sure the patching store is effective *before* we - * increment the counter which releases all waiting CPUs - * by using the release variant of atomic increment. The - * release pairs with the call to local_flush_icache_all() - * on the waiting CPU. - */ - atomic_inc_return_release(¶m->cpu_count); - } else { - while (atomic_read(¶m->cpu_count) <= num_online_cpus()) - cpu_relax(); - - local_flush_icache_all(); - } + /* + * When using CALL_OPS, the function to call is associated with the + * call site, and we don't have a global function pointer to update. + */ + if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS)) + return 0; + WRITE_ONCE(ftrace_call_dest, func); + /* + * The data fence ensure that the update to ftrace_call_dest happens + * before the write to function_trace_op later in the generic ftrace. + * If the sequence is not enforced, then an old ftrace_call_dest may + * race loading a new function_trace_op set in ftrace_modify_all_code + */ + smp_wmb(); + /* + * Updating ftrace dpes not take stop_machine path, so irqs should not + * be disabled. + */ + WARN_ON(irqs_disabled()); + smp_call_function(ftrace_sync_ipi, NULL, 1); return 0; } -void arch_ftrace_update_code(int command) +#else /* CONFIG_DYNAMIC_FTRACE */ +unsigned long ftrace_call_adjust(unsigned long addr) { - struct ftrace_modify_param param = { command, ATOMIC_INIT(0) }; - - stop_machine(__ftrace_modify_code, ¶m, cpu_online_mask); + return addr; } -#endif +#endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { - unsigned int call[2]; - unsigned long caller = rec->ip; + unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE; int ret; - make_call_t0(caller, old_addr, call); - ret = ftrace_check_current_call(caller, call); - + ret = ftrace_rec_update_ops(rec); if (ret) return ret; - return __ftrace_modify_call(caller, addr, true, false); + return __ftrace_modify_call(caller, FTRACE_ADDR, true); } #endif @@ -210,7 +247,6 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, } #ifdef CONFIG_DYNAMIC_FTRACE -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { @@ -231,19 +267,5 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, if (!function_graph_enter_regs(old, ip, frame_pointer, parent, fregs)) *parent = return_hooker; } -#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ -extern void ftrace_graph_call(void); -int ftrace_enable_ftrace_graph_caller(void) -{ - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, - (unsigned long)&prepare_ftrace_return, true, true); -} - -int ftrace_disable_ftrace_graph_caller(void) -{ - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, - (unsigned long)&prepare_ftrace_return, false, true); -} -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 356d5397b2a2..f6a8ca49e627 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -14,7 +14,7 @@ #include <asm/hwcap.h> #include <asm/image.h> #include <asm/scs.h> -#include <asm/xip_fixup.h> +#include <asm/usercfi.h> #include "efi-header.S" __HEAD @@ -74,13 +74,12 @@ pe_head_start: relocate_enable_mmu: /* Relocate return address */ la a1, kernel_map - XIP_FIXUP_OFFSET a1 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1 - /* Point stvec to virtual address of intruction after satp write */ + /* Point stvec to virtual address of instruction after satp write */ la a2, 1f add a2, a2, a1 csrw CSR_TVEC, a2 @@ -88,7 +87,6 @@ relocate_enable_mmu: /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT la a1, satp_mode - XIP_FIXUP_OFFSET a1 REG_L a1, 0(a1) or a2, a2, a1 @@ -99,7 +97,6 @@ relocate_enable_mmu: * to ensure the new translations are in use. */ la a0, trampoline_pg_dir - XIP_FIXUP_OFFSET a0 srl a0, a0, PAGE_SHIFT or a0, a0, a1 sfence.vma @@ -131,6 +128,12 @@ secondary_start_sbi: csrw CSR_IE, zero csrw CSR_IP, zero +#ifndef CONFIG_RISCV_M_MODE + /* Enable time CSR */ + li t0, 0x2 + csrw CSR_SCOUNTEREN, t0 +#endif + /* Load the global pointer */ load_global_pointer @@ -147,11 +150,9 @@ secondary_start_sbi: /* a0 contains the hartid & a1 contains boot data */ li a2, SBI_HART_BOOT_TASK_PTR_OFFSET - XIP_FIXUP_OFFSET a2 add a2, a2, a1 REG_L tp, (a2) li a3, SBI_HART_BOOT_STACK_PTR_OFFSET - XIP_FIXUP_OFFSET a3 add a3, a3, a1 REG_L sp, (a3) @@ -160,10 +161,22 @@ secondary_start_sbi: #ifdef CONFIG_MMU /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif call .Lsetup_trap_vector +#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI) + li a7, SBI_EXT_FWFT + li a6, SBI_EXT_FWFT_SET + li a0, SBI_FWFT_SHADOW_STACK + li a1, 1 /* enable supervisor to access shadow stack access */ + li a2, SBI_FWFT_SET_FLAG_LOCK + ecall + beqz a0, 1f + la a1, riscv_nousercfi + li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI + REG_S a0, (a1) +1: +#endif scs_load_current call smp_callin #endif /* CONFIG_SMP */ @@ -226,6 +239,10 @@ SYM_CODE_START(_start_kernel) * to hand it to us. */ csrr a0, CSR_MHARTID +#else + /* Enable time CSR */ + li t0, 0x2 + csrw CSR_SCOUNTEREN, t0 #endif /* CONFIG_RISCV_M_MODE */ /* Load the global pointer */ @@ -245,40 +262,13 @@ SYM_CODE_START(_start_kernel) .Lgood_cores: /* The lottery system is only required for spinwait booting method */ -#ifndef CONFIG_XIP_KERNEL /* Pick one hart to run the main boot sequence */ la a3, hart_lottery li a2, 1 amoadd.w a3, a2, (a3) bnez a3, .Lsecondary_start - -#else - /* hart_lottery in flash contains a magic number */ - la a3, hart_lottery - mv a2, a3 - XIP_FIXUP_OFFSET a2 - XIP_FIXUP_FLASH_OFFSET a3 - lw t1, (a3) - amoswap.w t0, t1, (a2) - /* first time here if hart_lottery in RAM is not set */ - beq t0, t1, .Lsecondary_start - -#endif /* CONFIG_XIP */ #endif /* CONFIG_RISCV_BOOT_SPINWAIT */ -#ifdef CONFIG_XIP_KERNEL - la sp, _end + THREAD_SIZE - XIP_FIXUP_OFFSET sp - mv s0, a0 - mv s1, a1 - call __copy_data - - /* Restore a0 & a1 copy */ - mv a0, s0 - mv a1, s1 -#endif - -#ifndef CONFIG_XIP_KERNEL /* Clear BSS for flat non-ELF images */ la a3, __bss_start la a4, __bss_stop @@ -288,20 +278,16 @@ SYM_CODE_START(_start_kernel) add a3, a3, RISCV_SZPTR blt a3, a4, .Lclear_bss .Lclear_bss_done: -#endif la a2, boot_cpu_hartid - XIP_FIXUP_OFFSET a2 REG_S a0, (a2) /* Initialize page tables and relocate to virtual addresses */ la tp, init_task la sp, init_thread_union + THREAD_SIZE - XIP_FIXUP_OFFSET sp addi sp, sp, -PT_SIZE_ON_STACK scs_load_init_stack #ifdef CONFIG_BUILTIN_DTB la a0, __dtb_start - XIP_FIXUP_OFFSET a0 #else mv a0, a1 #endif /* CONFIG_BUILTIN_DTB */ @@ -311,7 +297,6 @@ SYM_CODE_START(_start_kernel) call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu #endif /* CONFIG_MMU */ @@ -320,6 +305,19 @@ SYM_CODE_START(_start_kernel) la tp, init_task la sp, init_thread_union + THREAD_SIZE addi sp, sp, -PT_SIZE_ON_STACK +#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI) + li a7, SBI_EXT_FWFT + li a6, SBI_EXT_FWFT_SET + li a0, SBI_FWFT_SHADOW_STACK + li a1, 1 /* enable supervisor to access shadow stack access */ + li a2, SBI_FWFT_SET_FLAG_LOCK + ecall + beqz a0, 1f + la a1, riscv_nousercfi + li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI + REG_S a0, (a1) +1: +#endif scs_load_current #ifdef CONFIG_KASAN @@ -337,9 +335,7 @@ SYM_CODE_START(_start_kernel) slli a3, a0, LGREG la a1, __cpu_spinwait_stack_pointer - XIP_FIXUP_OFFSET a1 la a2, __cpu_spinwait_task_pointer - XIP_FIXUP_OFFSET a2 add a1, a3, a1 add a2, a3, a2 diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h index a556fdaafed9..05a04bef442b 100644 --- a/arch/riscv/kernel/head.h +++ b/arch/riscv/kernel/head.h @@ -11,9 +11,6 @@ extern atomic_t hart_lottery; asmlinkage void __init setup_vm(uintptr_t dtb_pa); -#ifdef CONFIG_XIP_KERNEL -asmlinkage void __init __copy_data(void); -#endif #ifdef CONFIG_RISCV_BOOT_SPINWAIT extern void *__cpu_spinwait_stack_pointer[]; diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c index 671b686c0158..982843828adb 100644 --- a/arch/riscv/kernel/hibernate.c +++ b/arch/riscv/kernel/hibernate.c @@ -415,7 +415,7 @@ int hibernate_resume_nonboot_cpu_disable(void) static int __init riscv_hibernate_init(void) { - hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL); + hibernate_cpu_context = kzalloc_obj(*hibernate_cpu_context); if (WARN_ON(!hibernate_cpu_context)) return -ENOMEM; diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 3df30dd1c458..3bd9d06a8b8f 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -29,7 +29,7 @@ __efistub__end = _end; __efistub__edata = _edata; __efistub___init_text_end = __init_text_end; #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB) -__efistub_screen_info = screen_info; +__efistub_sysfb_primary_display = sysfb_primary_display; #endif #endif diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 9ceda02507ca..b6af20bc300f 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -32,6 +32,40 @@ struct fwnode_handle *riscv_get_intc_hwnode(void) } EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode); +/** + * riscv_get_hart_index() - get hart index for interrupt delivery + * @fwnode: interrupt controller node + * @logical_index: index within the "interrupts-extended" property + * @hart_index: filled with the hart index to use + * + * RISC-V uses term "hart index" for its interrupt controllers, for the + * purpose of the interrupt routing to destination harts. + * It may be arbitrary numbers assigned to each destination hart in context + * of the particular interrupt domain. + * + * These numbers encoded in the optional property "riscv,hart-indexes" + * that should contain hart index for each interrupt destination in the same + * order as in the "interrupts-extended" property. If this property + * not exist, it assumed equal to the logical index, i.e. index within the + * "interrupts-extended" property. + * + * Return: error code + */ +int riscv_get_hart_index(struct fwnode_handle *fwnode, u32 logical_index, + u32 *hart_index) +{ + static const char *prop_hart_index = "riscv,hart-indexes"; + struct device_node *np = to_of_node(fwnode); + + if (!np || !of_property_present(np, prop_hart_index)) { + *hart_index = logical_index; + return 0; + } + + return of_property_read_u32_index(np, prop_hart_index, + logical_index, hart_index); +} + #ifdef CONFIG_IRQ_STACKS #include <asm/irq_stack.h> diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c index 654ed159c830..b4c1a6a3fbd2 100644 --- a/arch/riscv/kernel/jump_label.c +++ b/arch/riscv/kernel/jump_label.c @@ -11,8 +11,8 @@ #include <asm/bug.h> #include <asm/cacheflush.h> #include <asm/text-patching.h> +#include <asm/insn-def.h> -#define RISCV_INSN_NOP 0x00000013U #define RISCV_INSN_JAL 0x0000006fU bool arch_jump_label_transform_queue(struct jump_entry *entry, @@ -33,7 +33,7 @@ bool arch_jump_label_transform_queue(struct jump_entry *entry, (((u32)offset & GENMASK(10, 1)) << (21 - 1)) | (((u32)offset & GENMASK(20, 20)) << (31 - 20)); } else { - insn = RISCV_INSN_NOP; + insn = RISCV_INSN_NOP4; } if (early_boot_irqs_disabled) { diff --git a/arch/riscv/kernel/kexec_elf.c b/arch/riscv/kernel/kexec_elf.c new file mode 100644 index 000000000000..531d348db84d --- /dev/null +++ b/arch/riscv/kernel/kexec_elf.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Load ELF vmlinux file for the kexec_file_load syscall. + * + * Copyright (C) 2021 Huawei Technologies Co, Ltd. + * + * Author: Liao Chang (liaochang1@huawei.com) + * + * Based on kexec-tools' kexec-elf-riscv.c, heavily modified + * for kernel. + */ + +#define pr_fmt(fmt) "kexec_image: " fmt + +#include <linux/elf.h> +#include <linux/kexec.h> +#include <linux/slab.h> +#include <linux/of.h> +#include <linux/libfdt.h> +#include <linux/types.h> +#include <linux/memblock.h> +#include <asm/setup.h> + +static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr, + struct kexec_elf_info *elf_info, unsigned long old_pbase, + unsigned long new_pbase) +{ + int i; + int ret = 0; + size_t size; + struct kexec_buf kbuf = {}; + const struct elf_phdr *phdr; + + kbuf.image = image; + + for (i = 0; i < ehdr->e_phnum; i++) { + phdr = &elf_info->proghdrs[i]; + if (phdr->p_type != PT_LOAD) + continue; + + size = phdr->p_filesz; + if (size > phdr->p_memsz) + size = phdr->p_memsz; + + kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset; + kbuf.bufsz = size; + kbuf.buf_align = phdr->p_align; + kbuf.mem = phdr->p_paddr - old_pbase + new_pbase; + kbuf.memsz = phdr->p_memsz; + kbuf.top_down = false; + ret = kexec_add_buffer(&kbuf); + if (ret) + break; + } + + return ret; +} + +/* + * Go through the available phsyical memory regions and find one that hold + * an image of the specified size. + */ +static int elf_find_pbase(struct kimage *image, unsigned long kernel_len, + struct elfhdr *ehdr, struct kexec_elf_info *elf_info, + unsigned long *old_pbase, unsigned long *new_pbase) +{ + int i; + int ret; + struct kexec_buf kbuf = {}; + const struct elf_phdr *phdr; + unsigned long lowest_paddr = ULONG_MAX; + unsigned long lowest_vaddr = ULONG_MAX; + + for (i = 0; i < ehdr->e_phnum; i++) { + phdr = &elf_info->proghdrs[i]; + if (phdr->p_type != PT_LOAD) + continue; + + if (lowest_paddr > phdr->p_paddr) + lowest_paddr = phdr->p_paddr; + + if (lowest_vaddr > phdr->p_vaddr) + lowest_vaddr = phdr->p_vaddr; + } + + kbuf.image = image; + kbuf.buf_min = lowest_paddr; + kbuf.buf_max = ULONG_MAX; + + /* + * Current riscv boot protocol requires 2MB alignment for + * RV64 and 4MB alignment for RV32 + * + */ + kbuf.buf_align = PMD_SIZE; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE); + kbuf.cma = NULL; + kbuf.top_down = false; + ret = arch_kexec_locate_mem_hole(&kbuf); + if (!ret) { + *old_pbase = lowest_paddr; + *new_pbase = kbuf.mem; + image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem; + } + return ret; +} + +static void *elf_kexec_load(struct kimage *image, char *kernel_buf, + unsigned long kernel_len, char *initrd, + unsigned long initrd_len, char *cmdline, + unsigned long cmdline_len) +{ + int ret; + unsigned long old_kernel_pbase = ULONG_MAX; + unsigned long new_kernel_pbase = 0UL; + struct elfhdr ehdr; + struct kexec_elf_info elf_info; + + ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info); + if (ret) + return ERR_PTR(ret); + + ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info, + &old_kernel_pbase, &new_kernel_pbase); + if (ret) + goto out; + + /* Add the kernel binary to the image */ + ret = riscv_kexec_elf_load(image, &ehdr, &elf_info, + old_kernel_pbase, new_kernel_pbase); + if (ret) + goto out; + + ret = load_extra_segments(image, image->start, kernel_len, + initrd, initrd_len, cmdline, cmdline_len); +out: + kexec_free_elf_info(&elf_info); + return ret ? ERR_PTR(ret) : NULL; +} + +const struct kexec_file_ops elf_kexec_ops = { + .probe = kexec_elf_probe, + .load = elf_kexec_load, +}; diff --git a/arch/riscv/kernel/kexec_image.c b/arch/riscv/kernel/kexec_image.c new file mode 100644 index 000000000000..51dc89259f16 --- /dev/null +++ b/arch/riscv/kernel/kexec_image.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * RISC-V Kexec image loader + * + */ + +#define pr_fmt(fmt) "kexec_file(Image): " fmt + +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/kexec.h> +#include <linux/pe.h> +#include <linux/string.h> +#include <asm/byteorder.h> +#include <asm/image.h> + +static int image_probe(const char *kernel_buf, unsigned long kernel_len) +{ + const struct riscv_image_header *h = (const struct riscv_image_header *)kernel_buf; + + if (!h || kernel_len < sizeof(*h)) + return -EINVAL; + + /* According to Documentation/arch/riscv/boot-image-header.rst, + * use "magic2" field to check when version >= 0.2. + */ + + if (h->version >= RISCV_HEADER_VERSION && + memcmp(&h->magic2, RISCV_IMAGE_MAGIC2, sizeof(h->magic2))) + return -EINVAL; + + return 0; +} + +static void *image_load(struct kimage *image, + char *kernel, unsigned long kernel_len, + char *initrd, unsigned long initrd_len, + char *cmdline, unsigned long cmdline_len) +{ + struct riscv_image_header *h; + u64 flags; + bool be_image, be_kernel; + struct kexec_buf kbuf = {}; + int ret; + + /* Check Image header */ + h = (struct riscv_image_header *)kernel; + if (!h->image_size) { + ret = -EINVAL; + goto out; + } + + /* Check endianness */ + flags = le64_to_cpu(h->flags); + be_image = riscv_image_flag_field(flags, RISCV_IMAGE_FLAG_BE); + be_kernel = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); + if (be_image != be_kernel) { + ret = -EINVAL; + goto out; + } + + /* Load the kernel image */ + kbuf.image = image; + kbuf.buf_min = 0; + kbuf.buf_max = ULONG_MAX; + kbuf.top_down = false; + + kbuf.buffer = kernel; + kbuf.bufsz = kernel_len; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + kbuf.memsz = le64_to_cpu(h->image_size); + kbuf.buf_align = le64_to_cpu(h->text_offset); + + ret = kexec_add_buffer(&kbuf); + if (ret) { + pr_err("Error add kernel image ret=%d\n", ret); + goto out; + } + + image->start = kbuf.mem; + + pr_info("Loaded kernel at 0x%lx bufsz=0x%lx memsz=0x%lx\n", + kbuf.mem, kbuf.bufsz, kbuf.memsz); + + ret = load_extra_segments(image, kbuf.mem, kbuf.memsz, + initrd, initrd_len, cmdline, cmdline_len); + +out: + return ret ? ERR_PTR(ret) : NULL; +} + +const struct kexec_file_ops image_kexec_ops = { + .probe = image_probe, + .load = image_load, +}; diff --git a/arch/riscv/kernel/kgdb.c b/arch/riscv/kernel/kgdb.c index 2e0266ae6bd7..0bf629204c76 100644 --- a/arch/riscv/kernel/kgdb.c +++ b/arch/riscv/kernel/kgdb.c @@ -175,7 +175,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { {DBG_REG_T1, GDB_SIZEOF_REG, offsetof(struct pt_regs, t1)}, {DBG_REG_T2, GDB_SIZEOF_REG, offsetof(struct pt_regs, t2)}, {DBG_REG_FP, GDB_SIZEOF_REG, offsetof(struct pt_regs, s0)}, - {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, + {DBG_REG_S1, GDB_SIZEOF_REG, offsetof(struct pt_regs, s1)}, {DBG_REG_A0, GDB_SIZEOF_REG, offsetof(struct pt_regs, a0)}, {DBG_REG_A1, GDB_SIZEOF_REG, offsetof(struct pt_regs, a1)}, {DBG_REG_A2, GDB_SIZEOF_REG, offsetof(struct pt_regs, a2)}, @@ -244,8 +244,9 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) gdb_regs[DBG_REG_S6_OFF] = task->thread.s[6]; gdb_regs[DBG_REG_S7_OFF] = task->thread.s[7]; gdb_regs[DBG_REG_S8_OFF] = task->thread.s[8]; - gdb_regs[DBG_REG_S9_OFF] = task->thread.s[10]; - gdb_regs[DBG_REG_S10_OFF] = task->thread.s[11]; + gdb_regs[DBG_REG_S9_OFF] = task->thread.s[9]; + gdb_regs[DBG_REG_S10_OFF] = task->thread.s[10]; + gdb_regs[DBG_REG_S11_OFF] = task->thread.s[11]; gdb_regs[DBG_REG_EPC_OFF] = task->thread.ra; } @@ -254,15 +255,21 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) regs->epc = pc; } +noinline void arch_kgdb_breakpoint(void) +{ + asm(".global kgdb_compiled_break\n" + "kgdb_compiled_break: ebreak\n"); +} + void kgdb_arch_handle_qxfer_pkt(char *remcom_in_buffer, char *remcom_out_buffer) { if (!strncmp(remcom_in_buffer, gdb_xfer_read_target, sizeof(gdb_xfer_read_target))) - strcpy(remcom_out_buffer, riscv_gdb_stub_target_desc); + strscpy(remcom_out_buffer, riscv_gdb_stub_target_desc, BUFMAX); else if (!strncmp(remcom_in_buffer, gdb_xfer_read_cpuxml, sizeof(gdb_xfer_read_cpuxml))) - strcpy(remcom_out_buffer, riscv_gdb_stub_cpuxml); + strscpy(remcom_out_buffer, riscv_gdb_stub_cpuxml, BUFMAX); } static inline void kgdb_arch_update_addr(struct pt_regs *regs, diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index b0bf8c1722c0..54e2d9552e93 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -7,8 +7,369 @@ * Author: Liao Chang (liaochang1@huawei.com) */ #include <linux/kexec.h> +#include <linux/elf.h> +#include <linux/slab.h> +#include <linux/of.h> +#include <linux/libfdt.h> +#include <linux/types.h> +#include <linux/memblock.h> +#include <linux/vmalloc.h> +#include <asm/setup.h> +#include <asm/insn.h> const struct kexec_file_ops * const kexec_file_loaders[] = { &elf_kexec_ops, + &image_kexec_ops, NULL }; + +int arch_kimage_file_post_load_cleanup(struct kimage *image) +{ + kvfree(image->arch.fdt); + image->arch.fdt = NULL; + + vfree(image->elf_headers); + image->elf_headers = NULL; + image->elf_headers_sz = 0; + + return kexec_image_post_load_cleanup_default(image); +} + +#ifdef CONFIG_CRASH_DUMP +static int get_nr_ram_ranges_callback(struct resource *res, void *arg) +{ + unsigned int *nr_ranges = arg; + + (*nr_ranges)++; + return 0; +} + +static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) +{ + struct crash_mem *cmem = arg; + + cmem->ranges[cmem->nr_ranges].start = res->start; + cmem->ranges[cmem->nr_ranges].end = res->end; + cmem->nr_ranges++; + + return 0; +} + +static int prepare_elf_headers(void **addr, unsigned long *sz) +{ + struct crash_mem *cmem; + unsigned int nr_ranges; + int ret; + + nr_ranges = 1; /* For exclusion of crashkernel region */ + walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); + + cmem = kmalloc_flex(*cmem, ranges, nr_ranges); + if (!cmem) + return -ENOMEM; + + cmem->max_nr_ranges = nr_ranges; + cmem->nr_ranges = 0; + ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); + if (ret) + goto out; + + /* Exclude crashkernel region */ + ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); + if (!ret) + ret = crash_prepare_elf64_headers(cmem, true, addr, sz); + +out: + kfree(cmem); + return ret; +} + +static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, + unsigned long cmdline_len) +{ + int elfcorehdr_strlen; + char *cmdline_ptr; + + cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL); + if (!cmdline_ptr) + return NULL; + + elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ", + image->elf_load_addr); + + if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) { + pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n"); + kfree(cmdline_ptr); + return NULL; + } + + memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len); + /* Ensure it's nul terminated */ + cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0'; + return cmdline_ptr; +} +#endif + +#define RISCV_IMM_BITS 12 +#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS) +#define RISCV_CONST_HIGH_PART(x) \ + (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1)) +#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x)) + +#define ENCODE_ITYPE_IMM(x) \ + (RV_X(x, 0, 12) << 20) +#define ENCODE_BTYPE_IMM(x) \ + ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \ + (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31)) +#define ENCODE_UTYPE_IMM(x) \ + (RV_X(x, 12, 20) << 12) +#define ENCODE_JTYPE_IMM(x) \ + ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \ + (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31)) +#define ENCODE_CBTYPE_IMM(x) \ + ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \ + (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12)) +#define ENCODE_CJTYPE_IMM(x) \ + ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \ + (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \ + (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12)) +#define ENCODE_UJTYPE_IMM(x) \ + (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \ + (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32)) +#define ENCODE_UITYPE_IMM(x) \ + (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32)) + +#define CLEAN_IMM(type, x) \ + ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x)) + +int arch_kexec_apply_relocations_add(struct purgatory_info *pi, + Elf_Shdr *section, + const Elf_Shdr *relsec, + const Elf_Shdr *symtab) +{ + const char *strtab, *name, *shstrtab; + const Elf_Shdr *sechdrs; + Elf64_Rela *relas; + int i, r_type; + + /* String & section header string table */ + sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff; + strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset; + shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset; + + relas = (void *)pi->ehdr + relsec->sh_offset; + + for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) { + const Elf_Sym *sym; /* symbol to relocate */ + unsigned long addr; /* final location after relocation */ + unsigned long val; /* relocated symbol value */ + unsigned long sec_base; /* relocated symbol value */ + void *loc; /* tmp location to modify */ + + sym = (void *)pi->ehdr + symtab->sh_offset; + sym += ELF64_R_SYM(relas[i].r_info); + + if (sym->st_name) + name = strtab + sym->st_name; + else + name = shstrtab + sechdrs[sym->st_shndx].sh_name; + + loc = pi->purgatory_buf; + loc += section->sh_offset; + loc += relas[i].r_offset; + + if (sym->st_shndx == SHN_ABS) + sec_base = 0; + else if (sym->st_shndx >= pi->ehdr->e_shnum) { + pr_err("Invalid section %d for symbol %s\n", + sym->st_shndx, name); + return -ENOEXEC; + } else + sec_base = pi->sechdrs[sym->st_shndx].sh_addr; + + val = sym->st_value; + val += sec_base; + val += relas[i].r_addend; + + addr = section->sh_addr + relas[i].r_offset; + + r_type = ELF64_R_TYPE(relas[i].r_info); + + switch (r_type) { + case R_RISCV_BRANCH: + *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) | + ENCODE_BTYPE_IMM(val - addr); + break; + case R_RISCV_JAL: + *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) | + ENCODE_JTYPE_IMM(val - addr); + break; + /* + * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I + * sym is expected to be next to R_RISCV_PCREL_HI20 + * in purgatory relsec. Handle it like R_RISCV_CALL + * sym, instead of searching the whole relsec. + */ + case R_RISCV_PCREL_HI20: + case R_RISCV_CALL_PLT: + case R_RISCV_CALL: + *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) | + ENCODE_UJTYPE_IMM(val - addr); + break; + case R_RISCV_RVC_BRANCH: + *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) | + ENCODE_CBTYPE_IMM(val - addr); + break; + case R_RISCV_RVC_JUMP: + *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) | + ENCODE_CJTYPE_IMM(val - addr); + break; + case R_RISCV_ADD16: + *(u16 *)loc += val; + break; + case R_RISCV_SUB16: + *(u16 *)loc -= val; + break; + case R_RISCV_ADD32: + *(u32 *)loc += val; + break; + case R_RISCV_SUB32: + *(u32 *)loc -= val; + break; + /* It has been applied by R_RISCV_PCREL_HI20 sym */ + case R_RISCV_PCREL_LO12_I: + case R_RISCV_ALIGN: + case R_RISCV_RELAX: + break; + case R_RISCV_64: + *(u64 *)loc = val; + break; + default: + pr_err("Unknown rela relocation: %d\n", r_type); + return -ENOEXEC; + } + } + return 0; +} + + +int load_extra_segments(struct kimage *image, unsigned long kernel_start, + unsigned long kernel_len, char *initrd, + unsigned long initrd_len, char *cmdline, + unsigned long cmdline_len) +{ + int ret; + void *fdt; + unsigned long initrd_pbase = 0UL; + struct kexec_buf kbuf = {}; + char *modified_cmdline = NULL; + + kbuf.image = image; + kbuf.buf_min = kernel_start + kernel_len; + kbuf.buf_max = ULONG_MAX; + +#ifdef CONFIG_CRASH_DUMP + /* Add elfcorehdr */ + if (image->type == KEXEC_TYPE_CRASH) { + void *headers; + unsigned long headers_sz; + ret = prepare_elf_headers(&headers, &headers_sz); + if (ret) { + pr_err("Preparing elf core header failed\n"); + goto out; + } + + kbuf.buffer = headers; + kbuf.bufsz = headers_sz; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + kbuf.memsz = headers_sz; + kbuf.buf_align = ELF_CORE_HEADER_ALIGN; + kbuf.top_down = true; + + ret = kexec_add_buffer(&kbuf); + if (ret) { + vfree(headers); + goto out; + } + image->elf_headers = headers; + image->elf_load_addr = kbuf.mem; + image->elf_headers_sz = headers_sz; + + kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n", + image->elf_load_addr, kbuf.bufsz, kbuf.memsz); + + /* Setup cmdline for kdump kernel case */ + modified_cmdline = setup_kdump_cmdline(image, cmdline, + cmdline_len); + if (!modified_cmdline) { + pr_err("Setting up cmdline for kdump kernel failed\n"); + ret = -EINVAL; + goto out; + } + cmdline = modified_cmdline; + } +#endif + +#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY + /* Add purgatory to the image */ + kbuf.top_down = true; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + ret = kexec_load_purgatory(image, &kbuf); + if (ret) { + pr_err("Error loading purgatory ret=%d\n", ret); + goto out; + } + kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem); + + ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry", + &kernel_start, + sizeof(kernel_start), 0); + if (ret) + pr_err("Error update purgatory ret=%d\n", ret); +#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */ + + /* Add the initrd to the image */ + if (initrd != NULL) { + kbuf.buffer = initrd; + kbuf.bufsz = kbuf.memsz = initrd_len; + kbuf.buf_align = PAGE_SIZE; + kbuf.top_down = true; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + ret = kexec_add_buffer(&kbuf); + if (ret) + goto out; + initrd_pbase = kbuf.mem; + kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase); + } + + /* Add the DTB to the image */ + fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase, + initrd_len, cmdline, 0); + if (!fdt) { + pr_err("Error setting up the new device tree.\n"); + ret = -EINVAL; + goto out; + } + + fdt_pack(fdt); + kbuf.buffer = fdt; + kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt); + kbuf.buf_align = PAGE_SIZE; + kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; + kbuf.top_down = true; + ret = kexec_add_buffer(&kbuf); + if (ret) { + pr_err("Error add DTB kbuf ret=%d\n", ret); + goto out_free_fdt; + } + /* Cache the fdt buffer address for memory cleanup */ + image->arch.fdt = fdt; + kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem); + goto out; + +out_free_fdt: + kvfree(fdt); +out: + kfree(modified_cmdline); + return ret; +} diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 745dd4c4a69c..082fe0b0e3c0 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -13,7 +13,6 @@ .text -#define FENTRY_RA_OFFSET 8 #define ABI_SIZE_ON_STACK 80 #define ABI_A0 0 #define ABI_A1 8 @@ -56,21 +55,18 @@ addi sp, sp, ABI_SIZE_ON_STACK .endm -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS - /** * SAVE_ABI_REGS - save regs against the ftrace_regs struct * * After the stack is established, * * 0(sp) stores the PC of the traced function which can be accessed -* by &(fregs)->epc in tracing function. Note that the real -* function entry address should be computed with -FENTRY_RA_OFFSET. +* by &(fregs)->epc in tracing function. * * 8(sp) stores the function return address (i.e. parent IP) that * can be accessed by &(fregs)->ra in tracing function. * -* The other regs are saved at the respective localtion and accessed +* The other regs are saved at the respective location and accessed * by the respective ftrace_regs member. * * Here is the layout of stack for your reference. @@ -86,17 +82,20 @@ * +++++++++ **/ .macro SAVE_ABI_REGS - mv t4, sp // Save original SP in T4 addi sp, sp, -FREGS_SIZE_ON_STACK - REG_S t0, FREGS_EPC(sp) REG_S x1, FREGS_RA(sp) - REG_S t4, FREGS_SP(sp) // Put original SP on stack #ifdef HAVE_FUNCTION_GRAPH_FP_TEST REG_S x8, FREGS_S0(sp) #endif REG_S x6, FREGS_T1(sp) - +#ifdef CONFIG_CC_IS_CLANG + REG_S x7, FREGS_T2(sp) + REG_S x28, FREGS_T3(sp) + REG_S x29, FREGS_T4(sp) + REG_S x30, FREGS_T5(sp) + REG_S x31, FREGS_T6(sp) +#endif // save the arguments REG_S x10, FREGS_A0(sp) REG_S x11, FREGS_A1(sp) @@ -106,16 +105,25 @@ REG_S x15, FREGS_A5(sp) REG_S x16, FREGS_A6(sp) REG_S x17, FREGS_A7(sp) + mv a0, sp + addi a0, a0, FREGS_SIZE_ON_STACK + REG_S a0, FREGS_SP(sp) // Put original SP on stack .endm - .macro RESTORE_ABI_REGS, all=0 + .macro RESTORE_ABI_REGS REG_L t0, FREGS_EPC(sp) REG_L x1, FREGS_RA(sp) #ifdef HAVE_FUNCTION_GRAPH_FP_TEST REG_L x8, FREGS_S0(sp) #endif REG_L x6, FREGS_T1(sp) - +#ifdef CONFIG_CC_IS_CLANG + REG_L x7, FREGS_T2(sp) + REG_L x28, FREGS_T3(sp) + REG_L x29, FREGS_T4(sp) + REG_L x30, FREGS_T5(sp) + REG_L x31, FREGS_T6(sp) +#endif // restore the arguments REG_L x10, FREGS_A0(sp) REG_L x11, FREGS_A1(sp) @@ -130,60 +138,71 @@ .endm .macro PREPARE_ARGS - addi a0, t0, -FENTRY_RA_OFFSET + addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn) +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + mv a1, ra // parent_ip + REG_L a2, -16(t0) // op + REG_L ra, FTRACE_OPS_FUNC(a2) // op->func +#else la a1, function_trace_op - REG_L a2, 0(a1) - mv a1, ra - mv a3, sp + REG_L a2, 0(a1) // op + mv a1, ra // parent_ip +#endif + mv a3, sp // regs .endm -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ - -#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS SYM_FUNC_START(ftrace_caller) - SAVE_ABI - - addi a0, t0, -FENTRY_RA_OFFSET - la a1, function_trace_op - REG_L a2, 0(a1) - mv a1, ra - mv a3, sp +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + /* + * When CALL_OPS is enabled (2 or 4) nops [8B] are placed before the + * function entry, these are later overwritten with the pointer to the + * associated struct ftrace_ops. + * + * -8: &ftrace_ops of the associated tracer function. + *<ftrace enable>: + * 0: auipc t0/ra, 0x? + * 4: jalr t0/ra, ?(t0/ra) + * + * -8: &ftrace_nop_ops + *<ftrace disable>: + * 0: nop + * 4: nop + * + * t0 is set to ip+8 after the jalr is executed at the callsite, + * so we find the associated op at t0-16. + */ + REG_L t1, -16(t0) // op Should be SZ_REG instead of 16 -SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) - call ftrace_stub - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi a0, sp, ABI_RA - REG_L a1, ABI_T0(sp) - addi a1, a1, -FENTRY_RA_OFFSET -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - mv a2, s0 +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + /* + * If the op has a direct call, handle it immediately without + * saving/restoring registers. + */ + REG_L t1, FTRACE_OPS_DIRECT_CALL(t1) + bnez t1, ftrace_caller_direct #endif -SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) - call ftrace_stub #endif - RESTORE_ABI - jr t0 -SYM_FUNC_END(ftrace_caller) - -#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ -SYM_FUNC_START(ftrace_caller) - mv t1, zero SAVE_ABI_REGS PREPARE_ARGS +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS + jalr ra +#else SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) - call ftrace_stub - + REG_L ra, ftrace_call_dest + jalr ra, 0(ra) +#endif RESTORE_ABI_REGS - bnez t1, .Ldirect +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + bnez t1, ftrace_caller_direct +#endif jr t0 -.Ldirect: +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL) jr t1 +#endif SYM_FUNC_END(ftrace_caller) -#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */ - #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) jr t0 diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index 068168046e0e..da4a4000e57e 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -12,8 +12,6 @@ #include <asm/asm-offsets.h> #include <asm/ftrace.h> -#define ABI_SIZE_ON_STACK 80 - .text .macro SAVE_ABI_STATE @@ -28,12 +26,12 @@ * register if a0 was not saved. */ .macro SAVE_RET_ABI_STATE - addi sp, sp, -ABI_SIZE_ON_STACK - REG_S ra, 1*SZREG(sp) - REG_S s0, 8*SZREG(sp) - REG_S a0, 10*SZREG(sp) - REG_S a1, 11*SZREG(sp) - addi s0, sp, ABI_SIZE_ON_STACK + addi sp, sp, -FREGS_SIZE_ON_STACK + REG_S ra, FREGS_RA(sp) + REG_S s0, FREGS_S0(sp) + REG_S a0, FREGS_A0(sp) + REG_S a1, FREGS_A1(sp) + addi s0, sp, FREGS_SIZE_ON_STACK .endm .macro RESTORE_ABI_STATE @@ -43,11 +41,11 @@ .endm .macro RESTORE_RET_ABI_STATE - REG_L ra, 1*SZREG(sp) - REG_L s0, 8*SZREG(sp) - REG_L a0, 10*SZREG(sp) - REG_L a1, 11*SZREG(sp) - addi sp, sp, ABI_SIZE_ON_STACK + REG_L ra, FREGS_RA(sp) + REG_L s0, FREGS_S0(sp) + REG_L a0, FREGS_A0(sp) + REG_L a1, FREGS_A1(sp) + addi sp, sp, FREGS_SIZE_ON_STACK .endm SYM_TYPED_FUNC_START(ftrace_stub) diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c index e264e59e596e..98eaac6f6606 100644 --- a/arch/riscv/kernel/module-sections.c +++ b/arch/riscv/kernel/module-sections.c @@ -9,6 +9,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/moduleloader.h> +#include <linux/sort.h> unsigned long module_emit_got_entry(struct module *mod, unsigned long val) { @@ -55,43 +56,71 @@ unsigned long module_emit_plt_entry(struct module *mod, unsigned long val) return (unsigned long)&plt[i]; } -static int is_rela_equal(const Elf_Rela *x, const Elf_Rela *y) +#define cmp_3way(a, b) ((a) < (b) ? -1 : (a) > (b)) + +static int cmp_rela(const void *a, const void *b) { - return x->r_info == y->r_info && x->r_addend == y->r_addend; + const Elf_Rela *x = a, *y = b; + int i; + + /* sort by type, symbol index and addend */ + i = cmp_3way(x->r_info, y->r_info); + if (i == 0) + i = cmp_3way(x->r_addend, y->r_addend); + return i; } static bool duplicate_rela(const Elf_Rela *rela, int idx) { - int i; - for (i = 0; i < idx; i++) { - if (is_rela_equal(&rela[i], &rela[idx])) - return true; - } - return false; + /* + * Entries are sorted by type, symbol index and addend. That means + * that, if a duplicate entry exists, it must be in the preceding slot. + */ + return idx > 0 && cmp_rela(rela + idx, rela + idx - 1) == 0; } -static void count_max_entries(Elf_Rela *relas, int num, +static void count_max_entries(const Elf_Rela *relas, size_t num, unsigned int *plts, unsigned int *gots) { - unsigned int type, i; - - for (i = 0; i < num; i++) { - type = ELF_RISCV_R_TYPE(relas[i].r_info); - if (type == R_RISCV_CALL_PLT) { - if (!duplicate_rela(relas, i)) - (*plts)++; - } else if (type == R_RISCV_GOT_HI20) { - if (!duplicate_rela(relas, i)) - (*gots)++; + for (size_t i = 0; i < num; i++) { + if (duplicate_rela(relas, i)) + continue; + + switch (ELF_R_TYPE(relas[i].r_info)) { + case R_RISCV_CALL_PLT: + case R_RISCV_PLT32: + (*plts)++; + break; + case R_RISCV_GOT_HI20: + (*gots)++; + break; + default: + unreachable(); } } } +static bool rela_needs_plt_got_entry(const Elf_Rela *rela) +{ + switch (ELF_R_TYPE(rela->r_info)) { + case R_RISCV_CALL_PLT: + case R_RISCV_GOT_HI20: + case R_RISCV_PLT32: + return true; + default: + return false; + } +} + int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, struct module *mod) { + size_t num_scratch_relas = 0; unsigned int num_plts = 0; unsigned int num_gots = 0; + Elf_Rela *scratch = NULL; + Elf_Rela *new_scratch; + size_t scratch_size = 0; int i; /* @@ -119,11 +148,12 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, return -ENOEXEC; } - /* Calculate the maxinum number of entries */ + /* Calculate the maximum number of entries */ for (i = 0; i < ehdr->e_shnum; i++) { + size_t num_relas = sechdrs[i].sh_size / sizeof(Elf_Rela); Elf_Rela *relas = (void *)ehdr + sechdrs[i].sh_offset; - int num_rela = sechdrs[i].sh_size / sizeof(Elf_Rela); Elf_Shdr *dst_sec = sechdrs + sechdrs[i].sh_info; + size_t scratch_size_needed; if (sechdrs[i].sh_type != SHT_RELA) continue; @@ -132,7 +162,31 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, if (!(dst_sec->sh_flags & SHF_EXECINSTR)) continue; - count_max_entries(relas, num_rela, &num_plts, &num_gots); + /* + * apply_relocate_add() relies on HI20 and LO12 relocation pairs being + * close together, so sort a copy of the section to avoid interfering. + */ + scratch_size_needed = (num_scratch_relas + num_relas) * sizeof(*scratch); + if (scratch_size_needed > scratch_size) { + scratch_size = scratch_size_needed; + new_scratch = kvrealloc(scratch, scratch_size, GFP_KERNEL); + if (!new_scratch) { + kvfree(scratch); + return -ENOMEM; + } + scratch = new_scratch; + } + + for (size_t j = 0; j < num_relas; j++) + if (rela_needs_plt_got_entry(&relas[j])) + scratch[num_scratch_relas++] = relas[j]; + } + + if (scratch) { + /* sort the accumulated PLT/GOT relocations so duplicates are adjacent */ + sort(scratch, num_scratch_relas, sizeof(*scratch), cmp_rela, NULL); + count_max_entries(scratch, num_scratch_relas, &num_plts, &num_gots); + kvfree(scratch); } mod->arch.plt.shdr->sh_type = SHT_NOBITS; diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 47d0ebeec93c..1961135689db 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -648,7 +648,7 @@ process_accumulated_relocations(struct module *me, kfree(bucket_iter); } - kfree(*relocation_hashtable); + kvfree(*relocation_hashtable); } static int add_relocation_to_accumulate(struct module *me, int type, @@ -663,7 +663,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, struct used_bucket *bucket; unsigned long hash; - entry = kmalloc(sizeof(*entry), GFP_KERNEL); + entry = kmalloc_obj(*entry); if (!entry) return -ENOMEM; @@ -697,7 +697,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, * relocation_entry. */ if (!found) { - rel_head = kmalloc(sizeof(*rel_head), GFP_KERNEL); + rel_head = kmalloc_obj(*rel_head); if (!rel_head) { kfree(entry); @@ -709,7 +709,7 @@ static int add_relocation_to_accumulate(struct module *me, int type, INIT_HLIST_NODE(&rel_head->node); if (!current_head->first) { bucket = - kmalloc(sizeof(struct used_bucket), GFP_KERNEL); + kmalloc_obj(struct used_bucket); if (!bucket) { kfree(entry); @@ -752,9 +752,9 @@ initialize_relocation_hashtable(unsigned int num_relocations, hashtable_size <<= should_double_size; - *relocation_hashtable = kmalloc_array(hashtable_size, - sizeof(**relocation_hashtable), - GFP_KERNEL); + /* Number of relocations may be large, so kvmalloc it */ + *relocation_hashtable = kvmalloc_objs(**relocation_hashtable, + hashtable_size); if (!*relocation_hashtable) return 0; @@ -859,7 +859,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, } j++; - if (j > sechdrs[relsec].sh_size / sizeof(*rel)) + if (j == num_relocations) j = 0; } while (j_idx != j); diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index fa6b0339a65d..5f56be79cd06 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -16,22 +16,13 @@ #include <linux/printk.h> #include <linux/static_call.h> #include <linux/types.h> +#include <linux/sched/cputime.h> #include <asm/barrier.h> #include <asm/page.h> #include <asm/paravirt.h> #include <asm/sbi.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); - static bool steal_acc = true; static int __init parse_no_stealacc(char *arg) { diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index db13c9ddf9e3..16b243376f36 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -42,19 +42,20 @@ static inline bool is_kernel_exittext(uintptr_t addr) static __always_inline void *patch_map(void *addr, const unsigned int fixmap) { uintptr_t uintaddr = (uintptr_t) addr; - struct page *page; + phys_addr_t phys; - if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) - page = phys_to_page(__pa_symbol(addr)); - else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - page = vmalloc_to_page(addr); - else - return addr; + if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) { + phys = __pa_symbol(addr); + } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { + struct page *page = vmalloc_to_page(addr); - BUG_ON(!page); + BUG_ON(!page); + phys = page_to_phys(page) + offset_in_page(addr); + } else { + return addr; + } - return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + - offset_in_page(addr)); + return (void *)set_fixmap_offset(fixmap, phys); } static void patch_unmap(int fixmap) diff --git a/arch/riscv/kernel/pi/Makefile b/arch/riscv/kernel/pi/Makefile index 81d69d45c06c..bc098edac898 100644 --- a/arch/riscv/kernel/pi/Makefile +++ b/arch/riscv/kernel/pi/Makefile @@ -2,7 +2,7 @@ # This file was copied from arm64/kernel/pi/Makefile. KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \ - -Os -DDISABLE_BRANCH_PROFILING $(DISABLE_STACKLEAK_PLUGIN) \ + -Os -DDISABLE_BRANCH_PROFILING $(DISABLE_KSTACK_ERASE) \ $(call cc-option,-mbranch-protection=none) \ -I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \ -include $(srctree)/include/linux/hidden.h \ @@ -39,4 +39,4 @@ $(obj)/ctype.o: $(srctree)/lib/ctype.c FORCE $(call if_changed_rule,cc_o_c) obj-y := cmdline_early.pi.o fdt_early.pi.o string.pi.o ctype.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o archrandom_early.pi.o -extra-y := $(patsubst %.pi.o,%.o,$(obj-y)) +targets := $(patsubst %.pi.o,%.o,$(obj-y)) diff --git a/arch/riscv/kernel/pi/cmdline_early.c b/arch/riscv/kernel/pi/cmdline_early.c index fbcdc9e4e143..389d086a0718 100644 --- a/arch/riscv/kernel/pi/cmdline_early.c +++ b/arch/riscv/kernel/pi/cmdline_early.c @@ -41,9 +41,9 @@ static char *get_early_cmdline(uintptr_t dtb_pa) static u64 match_noXlvl(char *cmdline) { if (strstr(cmdline, "no4lvl")) - return SATP_MODE_48; + return SATP_MODE_39; else if (strstr(cmdline, "no5lvl")) - return SATP_MODE_57; + return SATP_MODE_48; return 0; } diff --git a/arch/riscv/kernel/pi/fdt_early.c b/arch/riscv/kernel/pi/fdt_early.c index 9bdee2fafe47..a12ff8090f19 100644 --- a/arch/riscv/kernel/pi/fdt_early.c +++ b/arch/riscv/kernel/pi/fdt_early.c @@ -3,6 +3,7 @@ #include <linux/init.h> #include <linux/libfdt.h> #include <linux/ctype.h> +#include <asm/csr.h> #include "pi.h" @@ -183,3 +184,42 @@ bool fdt_early_match_extension_isa(const void *fdt, const char *ext_name) return ret; } + +/** + * set_satp_mode_from_fdt - determine SATP mode based on the MMU type in fdt + * + * @dtb_pa: physical address of the device tree blob + * + * Returns the SATP mode corresponding to the MMU type of the first enabled CPU, + * 0 otherwise + */ +u64 set_satp_mode_from_fdt(uintptr_t dtb_pa) +{ + const void *fdt = (const void *)dtb_pa; + const char *mmu_type; + int node, parent; + + parent = fdt_path_offset(fdt, "/cpus"); + if (parent < 0) + return 0; + + fdt_for_each_subnode(node, fdt, parent) { + if (!fdt_node_name_eq(fdt, node, "cpu")) + continue; + + if (!fdt_device_is_available(fdt, node)) + continue; + + mmu_type = fdt_getprop(fdt, node, "mmu-type", NULL); + if (!mmu_type) + break; + + if (!strcmp(mmu_type, "riscv,sv39")) + return SATP_MODE_39; + else if (!strcmp(mmu_type, "riscv,sv48")) + return SATP_MODE_48; + break; + } + + return 0; +} diff --git a/arch/riscv/kernel/pi/pi.h b/arch/riscv/kernel/pi/pi.h index 21141d84fea6..3fee2cfddf7c 100644 --- a/arch/riscv/kernel/pi/pi.h +++ b/arch/riscv/kernel/pi/pi.h @@ -14,6 +14,7 @@ u64 get_kaslr_seed(uintptr_t dtb_pa); u64 get_kaslr_seed_zkr(const uintptr_t dtb_pa); bool set_nokaslr_from_cmdline(uintptr_t dtb_pa); u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa); +u64 set_satp_mode_from_fdt(uintptr_t dtb_pa); bool fdt_early_match_extension_isa(const void *fdt, const char *ext_name); diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c index c0738d6c6498..9e2afabf94e8 100644 --- a/arch/riscv/kernel/probes/kprobes.c +++ b/arch/riscv/kernel/probes/kprobes.c @@ -49,10 +49,15 @@ static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) post_kprobe_handler(p, kcb, regs); } -static bool __kprobes arch_check_kprobe(struct kprobe *p) +static bool __kprobes arch_check_kprobe(unsigned long addr) { - unsigned long tmp = (unsigned long)p->addr - p->offset; - unsigned long addr = (unsigned long)p->addr; + unsigned long tmp, offset; + + /* start iterating at the closest preceding symbol */ + if (!kallsyms_lookup_size_offset(addr, NULL, &offset)) + return false; + + tmp = addr - offset; while (tmp <= addr) { if (tmp == addr) @@ -71,7 +76,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) if ((unsigned long)insn & 0x1) return -EILSEQ; - if (!arch_check_kprobe(p)) + if (!arch_check_kprobe((unsigned long)p->addr)) return -EILSEQ; /* copy instruction */ @@ -144,7 +149,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p) /* * Interrupts need to be disabled before single-step mode is set, and not - * reenabled until after single-step mode ends. + * re-enabled until after single-step mode ends. * Without disabling interrupt on local CPU, there is a chance of * interrupt occurrence in the period of exception return and start of * out-of-line single-step, that result in wrongly single stepping diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c index 6c166029079c..fa581590c1f8 100644 --- a/arch/riscv/kernel/probes/simulate-insn.c +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -41,19 +41,16 @@ bool __kprobes simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs * 1 10 1 8 5 JAL/J */ bool ret; - u32 imm; - u32 index = (opcode >> 7) & 0x1f; + s32 imm; + u32 index = RV_EXTRACT_RD_REG(opcode); ret = rv_insn_reg_set_val(regs, index, addr + 4); if (!ret) return ret; - imm = ((opcode >> 21) & 0x3ff) << 1; - imm |= ((opcode >> 20) & 0x1) << 11; - imm |= ((opcode >> 12) & 0xff) << 12; - imm |= ((opcode >> 31) & 0x1) << 20; + imm = RV_EXTRACT_JTYPE_IMM(opcode); - instruction_pointer_set(regs, addr + sign_extend32((imm), 20)); + instruction_pointer_set(regs, addr + imm); return ret; } @@ -67,9 +64,9 @@ bool __kprobes simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *reg */ bool ret; unsigned long base_addr; - u32 imm = (opcode >> 20) & 0xfff; - u32 rd_index = (opcode >> 7) & 0x1f; - u32 rs1_index = (opcode >> 15) & 0x1f; + u32 imm = RV_EXTRACT_ITYPE_IMM(opcode); + u32 rd_index = RV_EXTRACT_RD_REG(opcode); + u32 rs1_index = RV_EXTRACT_RS1_REG(opcode); ret = rv_insn_reg_get_val(regs, rs1_index, &base_addr); if (!ret) @@ -84,20 +81,6 @@ bool __kprobes simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *reg return ret; } -#define auipc_rd_idx(opcode) \ - ((opcode >> 7) & 0x1f) - -#define auipc_imm(opcode) \ - ((((opcode) >> 12) & 0xfffff) << 12) - -#if __riscv_xlen == 64 -#define auipc_offset(opcode) sign_extend64(auipc_imm(opcode), 31) -#elif __riscv_xlen == 32 -#define auipc_offset(opcode) auipc_imm(opcode) -#else -#error "Unexpected __riscv_xlen" -#endif - bool __kprobes simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *regs) { /* @@ -107,8 +90,8 @@ bool __kprobes simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *re * 20 5 7 */ - u32 rd_idx = auipc_rd_idx(opcode); - unsigned long rd_val = addr + auipc_offset(opcode); + u32 rd_idx = RV_EXTRACT_RD_REG(opcode); + unsigned long rd_val = addr + (s32)RV_EXTRACT_UTYPE_IMM(opcode); if (!rv_insn_reg_set_val(regs, rd_idx, rd_val)) return false; @@ -118,24 +101,6 @@ bool __kprobes simulate_auipc(u32 opcode, unsigned long addr, struct pt_regs *re return true; } -#define branch_rs1_idx(opcode) \ - (((opcode) >> 15) & 0x1f) - -#define branch_rs2_idx(opcode) \ - (((opcode) >> 20) & 0x1f) - -#define branch_funct3(opcode) \ - (((opcode) >> 12) & 0x7) - -#define branch_imm(opcode) \ - (((((opcode) >> 8) & 0xf ) << 1) | \ - ((((opcode) >> 25) & 0x3f) << 5) | \ - ((((opcode) >> 7) & 0x1 ) << 11) | \ - ((((opcode) >> 31) & 0x1 ) << 12)) - -#define branch_offset(opcode) \ - sign_extend32((branch_imm(opcode)), 12) - bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs) { /* @@ -156,12 +121,12 @@ bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *r unsigned long rs1_val; unsigned long rs2_val; - if (!rv_insn_reg_get_val(regs, branch_rs1_idx(opcode), &rs1_val) || - !rv_insn_reg_get_val(regs, branch_rs2_idx(opcode), &rs2_val)) + if (!rv_insn_reg_get_val(regs, RV_EXTRACT_RS1_REG(opcode), &rs1_val) || + !rv_insn_reg_get_val(regs, RV_EXTRACT_RS2_REG(opcode), &rs2_val)) return false; - offset_tmp = branch_offset(opcode); - switch (branch_funct3(opcode)) { + offset_tmp = RV_EXTRACT_BTYPE_IMM(opcode); + switch (RV_EXTRACT_FUNCT3(opcode)) { case RVG_FUNCT3_BEQ: offset = (rs1_val == rs2_val) ? offset_tmp : 4; break; @@ -191,24 +156,9 @@ bool __kprobes simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *r bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs) { - /* - * 15 13 12 2 1 0 - * | funct3 | offset[11|4|9:8|10|6|7|3:1|5] | opcode | - * 3 11 2 - */ - - s32 offset; + s32 offset = RVC_EXTRACT_JTYPE_IMM(opcode); - offset = ((opcode >> 3) & 0x7) << 1; - offset |= ((opcode >> 11) & 0x1) << 4; - offset |= ((opcode >> 2) & 0x1) << 5; - offset |= ((opcode >> 7) & 0x1) << 6; - offset |= ((opcode >> 6) & 0x1) << 7; - offset |= ((opcode >> 9) & 0x3) << 8; - offset |= ((opcode >> 8) & 0x1) << 10; - offset |= ((opcode >> 12) & 0x1) << 11; - - instruction_pointer_set(regs, addr + sign_extend32(offset, 11)); + instruction_pointer_set(regs, addr + offset); return true; } @@ -224,7 +174,7 @@ static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct unsigned long jump_addr; - u32 rs1 = (opcode >> 7) & 0x1f; + u32 rs1 = RVC_EXTRACT_C2_RS1_REG(opcode); if (rs1 == 0) /* C.JR is only valid when rs1 != x0 */ return false; @@ -268,16 +218,10 @@ static bool __kprobes simulate_c_bnez_beqz(u32 opcode, unsigned long addr, struc if (!rv_insn_reg_get_val(regs, rs1, &rs1_val)) return false; - if ((rs1_val != 0 && is_bnez) || (rs1_val == 0 && !is_bnez)) { - offset = ((opcode >> 3) & 0x3) << 1; - offset |= ((opcode >> 10) & 0x3) << 3; - offset |= ((opcode >> 2) & 0x1) << 5; - offset |= ((opcode >> 5) & 0x3) << 6; - offset |= ((opcode >> 12) & 0x1) << 8; - offset = sign_extend32(offset, 8); - } else { + if ((rs1_val != 0 && is_bnez) || (rs1_val == 0 && !is_bnez)) + offset = RVC_EXTRACT_BTYPE_IMM(opcode); + else offset = 2; - } instruction_pointer_set(regs, addr + offset); diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c index 4b3dc8beaf77..eb177d0ce8ab 100644 --- a/arch/riscv/kernel/probes/uprobes.c +++ b/arch/riscv/kernel/probes/uprobes.c @@ -111,7 +111,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) current->thread.bad_cause = utask->autask.saved_cause; /* - * Task has received a fatal signal, so reset back to probbed + * Task has received a fatal signal, so reset back to probed * address. */ instruction_pointer_set(regs, utask->vaddr); @@ -165,8 +165,9 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, void *src, unsigned long len) { /* Initialize the slot */ - void *kaddr = kmap_atomic(page); + void *kaddr = kmap_local_page(page); void *dst = kaddr + (vaddr & ~PAGE_MASK); + unsigned long start = (unsigned long)dst; memcpy(dst, src, len); @@ -176,13 +177,6 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, *(uprobe_opcode_t *)dst = __BUG_INSN_32; } - kunmap_atomic(kaddr); - - /* - * We probably need flush_icache_user_page() but it needs vma. - * This should work on most of architectures by default. If - * architecture needs to do something different it can define - * its own version of the function. - */ - flush_dcache_page(page); + flush_icache_range(start, start + len); + kunmap_local(kaddr); } diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 7c244de77180..b2df7f72241a 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -17,7 +17,9 @@ #include <linux/ptrace.h> #include <linux/uaccess.h> #include <linux/personality.h> +#include <linux/entry-common.h> +#include <asm/asm-prototypes.h> #include <asm/unistd.h> #include <asm/processor.h> #include <asm/csr.h> @@ -29,6 +31,7 @@ #include <asm/vector.h> #include <asm/cpufeature.h> #include <asm/exec.h> +#include <asm/usercfi.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> @@ -36,7 +39,8 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif -extern asmlinkage void ret_from_fork(void); +extern asmlinkage void ret_from_fork_kernel_asm(void); +extern asmlinkage void ret_from_fork_user_asm(void); void noinstr arch_cpu_idle(void) { @@ -57,7 +61,7 @@ int get_unalign_ctl(struct task_struct *tsk, unsigned long adr) if (!unaligned_ctl_available()) return -EINVAL; - return put_user(tsk->thread.align_ctl, (unsigned long __user *)adr); + return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr); } void __show_regs(struct pt_regs *regs) @@ -89,8 +93,8 @@ void __show_regs(struct pt_regs *regs) regs->s8, regs->s9, regs->s10); pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n", regs->s11, regs->t3, regs->t4); - pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT "\n", - regs->t5, regs->t6); + pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT " ssp : " REG_FMT "\n", + regs->t5, regs->t6, get_active_shstk(current)); pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", regs->status, regs->badaddr, regs->cause); @@ -152,6 +156,21 @@ void start_thread(struct pt_regs *regs, unsigned long pc, regs->epc = pc; regs->sp = sp; + /* + * clear shadow stack state on exec. + * libc will set it later via prctl. + */ + set_shstk_lock(current, false); + set_shstk_status(current, false); + set_shstk_base(current, 0, 0); + set_active_shstk(current, 0); + /* + * disable indirect branch tracking on exec. + * libc will enable it later via prctl. + */ + set_indir_lp_lock(current, false); + set_indir_lp_status(current, false); + #ifdef CONFIG_64BIT regs->status &= ~SR_UXL; @@ -206,11 +225,24 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } +asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs) +{ + fn(fn_arg); + + syscall_exit_to_user_mode(regs); +} + +asmlinkage void ret_from_fork_user(struct pt_regs *regs) +{ + syscall_exit_to_user_mode(regs); +} + int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { - unsigned long clone_flags = args->flags; + u64 clone_flags = args->flags; unsigned long usp = args->stack; unsigned long tls = args->tls; + unsigned long ssp = 0; struct pt_regs *childregs = task_pt_regs(p); /* Ensure all threads in this mm have the same pointer masking mode. */ @@ -228,21 +260,29 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.s[0] = (unsigned long)args->fn; p->thread.s[1] = (unsigned long)args->fn_arg; + p->thread.ra = (unsigned long)ret_from_fork_kernel_asm; } else { + /* allocate new shadow stack if needed. In case of CLONE_VM we have to */ + ssp = shstk_alloc_thread_stack(p, args); + if (IS_ERR_VALUE(ssp)) + return PTR_ERR((void *)ssp); + *childregs = *(current_pt_regs()); /* Turn off status.VS */ riscv_v_vstate_off(childregs); if (usp) /* User fork */ childregs->sp = usp; + /* if needed, set new ssp */ + if (ssp) + set_active_shstk(p, ssp); if (clone_flags & CLONE_SETTLS) childregs->tp = tls; childregs->a0 = 0; /* Return value of fork() */ - p->thread.s[0] = 0; + p->thread.ra = (unsigned long)ret_from_fork_user_asm; } p->thread.riscv_v_flags = 0; if (has_vector() || has_xtheadvector()) riscv_v_thread_alloc(p); - p->thread.ra = (unsigned long)ret_from_fork; p->thread.sp = (unsigned long)childregs; /* kernel sp */ return 0; } @@ -275,6 +315,9 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) unsigned long pmm; u8 pmlen; + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) + return -EINVAL; + if (is_compat_thread(ti)) return -EINVAL; @@ -306,8 +349,10 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) + if (!(arg & PR_TAGGED_ADDR_ENABLE)) { pmlen = PMLEN_0; + pmm = ENVCFG_PMM_PMLEN_0; + } if (mmap_write_lock_killable(mm)) return -EINTR; @@ -330,6 +375,9 @@ long get_tagged_addr_ctrl(struct task_struct *task) struct thread_info *ti = task_thread_info(task); long ret = 0; + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) + return -EINVAL; + if (is_compat_thread(ti)) return -EINVAL; diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index ea67e9fb7a58..793bcee46182 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -19,6 +19,7 @@ #include <linux/regset.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> +#include <asm/usercfi.h> enum riscv_regset { REGSET_X, @@ -31,6 +32,9 @@ enum riscv_regset { #ifdef CONFIG_RISCV_ISA_SUPM REGSET_TAGGED_ADDR_CTRL, #endif +#ifdef CONFIG_RISCV_USER_CFI + REGSET_CFI, +#endif }; static int riscv_gpr_get(struct task_struct *target, @@ -95,9 +99,12 @@ static int riscv_vr_get(struct task_struct *target, struct __riscv_v_ext_state *vstate = &target->thread.vstate; struct __riscv_v_regset_state ptrace_vstate; - if (!riscv_v_vstate_query(task_pt_regs(target))) + if (!(has_vector() || has_xtheadvector())) return -EINVAL; + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -ENODATA; + /* * Ensure the vector registers have been saved to the memory before * copying them to membuf. @@ -121,6 +128,92 @@ static int riscv_vr_get(struct task_struct *target, return membuf_write(&to, vstate->datap, riscv_v_vsize); } +static int invalid_ptrace_v_csr(struct __riscv_v_ext_state *vstate, + struct __riscv_v_regset_state *ptrace) +{ + unsigned long vsew, vlmul, vfrac, vl; + unsigned long elen, vlen; + unsigned long sew, lmul; + unsigned long reserved; + + vlen = vstate->vlenb * 8; + if (vstate->vlenb != ptrace->vlenb) + return 1; + + /* do not allow to set vcsr/vxrm/vxsat reserved bits */ + reserved = ~(CSR_VXSAT_MASK | (CSR_VXRM_MASK << CSR_VXRM_SHIFT)); + if (ptrace->vcsr & reserved) + return 1; + + if (has_vector()) { + /* do not allow to set vtype reserved bits and vill bit */ + reserved = ~(VTYPE_VSEW | VTYPE_VLMUL | VTYPE_VMA | VTYPE_VTA); + if (ptrace->vtype & reserved) + return 1; + + elen = riscv_has_extension_unlikely(RISCV_ISA_EXT_ZVE64X) ? 64 : 32; + vsew = (ptrace->vtype & VTYPE_VSEW) >> VTYPE_VSEW_SHIFT; + sew = 8 << vsew; + + if (sew > elen) + return 1; + + vfrac = (ptrace->vtype & VTYPE_VLMUL_FRAC); + vlmul = (ptrace->vtype & VTYPE_VLMUL); + + /* RVV 1.0 spec 3.4.2: VLMUL(0x4) reserved */ + if (vlmul == 4) + return 1; + + /* RVV 1.0 spec 3.4.2: (LMUL < SEW_min / ELEN) reserved */ + if (vlmul == 5 && elen == 32) + return 1; + + /* for zero vl verify that at least one element is possible */ + vl = ptrace->vl ? ptrace->vl : 1; + + if (vfrac) { + /* integer 1/LMUL: VL =< VLMAX = VLEN / SEW / LMUL */ + lmul = 2 << (3 - (vlmul - vfrac)); + if (vlen < vl * sew * lmul) + return 1; + } else { + /* integer LMUL: VL =< VLMAX = LMUL * VLEN / SEW */ + lmul = 1 << vlmul; + if (vl * sew > lmul * vlen) + return 1; + } + } + + if (has_xtheadvector()) { + /* do not allow to set vtype reserved bits and vill bit */ + reserved = ~(VTYPE_VSEW_THEAD | VTYPE_VLMUL_THEAD | VTYPE_VEDIV_THEAD); + if (ptrace->vtype & reserved) + return 1; + + /* + * THead ISA Extension spec chapter 16: + * divided element extension ('Zvediv') is not part of XTheadVector + */ + if (ptrace->vtype & VTYPE_VEDIV_THEAD) + return 1; + + vsew = (ptrace->vtype & VTYPE_VSEW_THEAD) >> VTYPE_VSEW_THEAD_SHIFT; + sew = 8 << vsew; + + vlmul = (ptrace->vtype & VTYPE_VLMUL_THEAD); + lmul = 1 << vlmul; + + /* for zero vl verify that at least one element is possible */ + vl = ptrace->vl ? ptrace->vl : 1; + + if (vl * sew > lmul * vlen) + return 1; + } + + return 0; +} + static int riscv_vr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, @@ -130,16 +223,19 @@ static int riscv_vr_set(struct task_struct *target, struct __riscv_v_ext_state *vstate = &target->thread.vstate; struct __riscv_v_regset_state ptrace_vstate; - if (!riscv_v_vstate_query(task_pt_regs(target))) + if (!(has_vector() || has_xtheadvector())) return -EINVAL; + if (!riscv_v_vstate_query(task_pt_regs(target))) + return -ENODATA; + /* Copy rest of the vstate except datap */ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ptrace_vstate, 0, sizeof(struct __riscv_v_regset_state)); if (unlikely(ret)) return ret; - if (vstate->vlenb != ptrace_vstate.vlenb) + if (invalid_ptrace_v_csr(vstate, &ptrace_vstate)) return -EINVAL; vstate->vstart = ptrace_vstate.vstart; @@ -153,6 +249,17 @@ static int riscv_vr_set(struct task_struct *target, 0, riscv_v_vsize); return ret; } + +static int riscv_vr_active(struct task_struct *target, const struct user_regset *regset) +{ + if (!(has_vector() || has_xtheadvector())) + return -ENODEV; + + if (!riscv_v_vstate_query(task_pt_regs(target))) + return 0; + + return regset->n; +} #endif #ifdef CONFIG_RISCV_ISA_SUPM @@ -184,9 +291,90 @@ static int tagged_addr_ctrl_set(struct task_struct *target, } #endif -static const struct user_regset riscv_user_regset[] = { +#ifdef CONFIG_RISCV_USER_CFI +static int riscv_cfi_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct user_cfi_state user_cfi; + struct pt_regs *regs; + + memset(&user_cfi, 0, sizeof(user_cfi)); + regs = task_pt_regs(target); + + if (is_indir_lp_enabled(target)) { + user_cfi.cfi_status.cfi_state |= PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE; + user_cfi.cfi_status.cfi_state |= is_indir_lp_locked(target) ? + PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE : 0; + user_cfi.cfi_status.cfi_state |= (regs->status & SR_ELP) ? + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE : 0; + } + + if (is_shstk_enabled(target)) { + user_cfi.cfi_status.cfi_state |= (PTRACE_CFI_SHADOW_STACK_EN_STATE | + PTRACE_CFI_SHADOW_STACK_PTR_STATE); + user_cfi.cfi_status.cfi_state |= is_shstk_locked(target) ? + PTRACE_CFI_SHADOW_STACK_LOCK_STATE : 0; + user_cfi.shstk_ptr = get_active_shstk(target); + } + + return membuf_write(&to, &user_cfi, sizeof(user_cfi)); +} + +/* + * Does it make sense to allow enable / disable of cfi via ptrace? + * We don't allow enable / disable / locking control via ptrace for now. + * Setting the shadow stack pointer is allowed. GDB might use it to unwind or + * some other fixup. Similarly gdb might want to suppress elp and may want + * to reset elp state. + */ +static int riscv_cfi_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct user_cfi_state user_cfi; + struct pt_regs *regs; + + regs = task_pt_regs(target); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_cfi, 0, -1); + if (ret) + return ret; + + /* + * Not allowing enabling or locking shadow stack or landing pad + * There is no disabling of shadow stack or landing pad via ptrace + * rsvd field should be set to zero so that if those fields are needed in future + */ + if ((user_cfi.cfi_status.cfi_state & + (PTRACE_CFI_BRANCH_LANDING_PAD_EN_STATE | PTRACE_CFI_BRANCH_LANDING_PAD_LOCK_STATE | + PTRACE_CFI_SHADOW_STACK_EN_STATE | PTRACE_CFI_SHADOW_STACK_LOCK_STATE)) || + (user_cfi.cfi_status.cfi_state & PTRACE_CFI_STATE_INVALID_MASK)) + return -EINVAL; + + /* If lpad is enabled on target and ptrace requests to set / clear elp, do that */ + if (is_indir_lp_enabled(target)) { + if (user_cfi.cfi_status.cfi_state & + PTRACE_CFI_BRANCH_EXPECTED_LANDING_PAD_STATE) /* set elp state */ + regs->status |= SR_ELP; + else + regs->status &= ~SR_ELP; /* clear elp state */ + } + + /* If shadow stack enabled on target, set new shadow stack pointer */ + if (is_shstk_enabled(target) && + (user_cfi.cfi_status.cfi_state & PTRACE_CFI_SHADOW_STACK_PTR_STATE)) + set_active_shstk(target, user_cfi.shstk_ptr); + + return 0; +} +#endif + +static struct user_regset riscv_user_regset[] __ro_after_init = { [REGSET_X] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(elf_greg_t), .align = sizeof(elf_greg_t), @@ -195,7 +383,7 @@ static const struct user_regset riscv_user_regset[] = { }, #ifdef CONFIG_FPU [REGSET_F] = { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = ELF_NFPREG, .size = sizeof(elf_fpreg_t), .align = sizeof(elf_fpreg_t), @@ -205,18 +393,17 @@ static const struct user_regset riscv_user_regset[] = { #endif #ifdef CONFIG_RISCV_ISA_V [REGSET_V] = { - .core_note_type = NT_RISCV_VECTOR, + USER_REGSET_NOTE_TYPE(RISCV_VECTOR), .align = 16, - .n = ((32 * RISCV_MAX_VLENB) + - sizeof(struct __riscv_v_regset_state)) / sizeof(__u32), .size = sizeof(__u32), .regset_get = riscv_vr_get, .set = riscv_vr_set, + .active = riscv_vr_active, }, #endif #ifdef CONFIG_RISCV_ISA_SUPM [REGSET_TAGGED_ADDR_CTRL] = { - .core_note_type = NT_RISCV_TAGGED_ADDR_CTRL, + USER_REGSET_NOTE_TYPE(RISCV_TAGGED_ADDR_CTRL), .n = 1, .size = sizeof(long), .align = sizeof(long), @@ -224,6 +411,16 @@ static const struct user_regset riscv_user_regset[] = { .set = tagged_addr_ctrl_set, }, #endif +#ifdef CONFIG_RISCV_USER_CFI + [REGSET_CFI] = { + .core_note_type = NT_RISCV_USER_CFI, + .align = sizeof(__u64), + .n = sizeof(struct user_cfi_state) / sizeof(__u64), + .size = sizeof(__u64), + .regset_get = riscv_cfi_get, + .set = riscv_cfi_set, + }, +#endif }; static const struct user_regset_view riscv_user_native_view = { @@ -233,6 +430,14 @@ static const struct user_regset_view riscv_user_native_view = { .n = ARRAY_SIZE(riscv_user_regset), }; +#ifdef CONFIG_RISCV_ISA_V +void __init update_regset_vector_info(unsigned long size) +{ + riscv_user_regset[REGSET_V].n = (size + sizeof(struct __riscv_v_regset_state)) / + sizeof(__u32); +} +#endif + struct pt_regs_offset { const char *name; int offset; @@ -372,15 +577,15 @@ static int compat_riscv_gpr_set(struct task_struct *target, struct compat_user_regs_struct cregs; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &cregs, 0, -1); - - cregs_to_regs(&cregs, task_pt_regs(target)); + if (!ret) + cregs_to_regs(&cregs, task_pt_regs(target)); return ret; } static const struct user_regset compat_riscv_user_regset[] = { [REGSET_X] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(compat_elf_greg_t), .align = sizeof(compat_elf_greg_t), @@ -389,7 +594,7 @@ static const struct user_regset compat_riscv_user_regset[] = { }, #ifdef CONFIG_FPU [REGSET_F] = { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = ELF_NFPREG, .size = sizeof(elf_fpreg_t), .align = sizeof(elf_fpreg_t), diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index 1989b8cade1b..c443337056ab 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -148,7 +148,7 @@ static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, static void sbi_set_power_off(void) { - pm_power_off = sbi_shutdown; + register_platform_power_off(sbi_shutdown); } #else static void __sbi_set_timer_v01(uint64_t stime_value) @@ -299,6 +299,76 @@ static int __sbi_rfence_v02(int fid, const struct cpumask *cpu_mask, return 0; } +static bool sbi_fwft_supported; + +struct fwft_set_req { + u32 feature; + unsigned long value; + unsigned long flags; + atomic_t error; +}; + +static void cpu_sbi_fwft_set(void *arg) +{ + struct fwft_set_req *req = arg; + int ret; + + ret = sbi_fwft_set(req->feature, req->value, req->flags); + if (ret) + atomic_set(&req->error, ret); +} + +/** + * sbi_fwft_set() - Set a feature on the local hart + * @feature: The feature ID to be set + * @value: The feature value to be set + * @flags: FWFT feature set flags + * + * Return: 0 on success, appropriate linux error code otherwise. + */ +int sbi_fwft_set(u32 feature, unsigned long value, unsigned long flags) +{ + struct sbiret ret; + + if (!sbi_fwft_supported) + return -EOPNOTSUPP; + + ret = sbi_ecall(SBI_EXT_FWFT, SBI_EXT_FWFT_SET, + feature, value, flags, 0, 0, 0); + + return sbi_err_map_linux_errno(ret.error); +} + +/** + * sbi_fwft_set_cpumask() - Set a feature for the specified cpumask + * @mask: CPU mask of cpus that need the feature to be set + * @feature: The feature ID to be set + * @value: The feature value to be set + * @flags: FWFT feature set flags + * + * Return: 0 on success, appropriate linux error code otherwise. + */ +int sbi_fwft_set_cpumask(const cpumask_t *mask, u32 feature, + unsigned long value, unsigned long flags) +{ + struct fwft_set_req req = { + .feature = feature, + .value = value, + .flags = flags, + .error = ATOMIC_INIT(0), + }; + + if (!sbi_fwft_supported) + return -EOPNOTSUPP; + + if (feature & SBI_FWFT_GLOBAL_FEATURE_BIT) + return -EINVAL; + + on_each_cpu_mask(mask, cpu_sbi_fwft_set, &req, 1); + + return atomic_read(&req.error); +} + /** * sbi_set_timer() - Program the timer for next timer event. * @stime_value: The value after which next timer event should fire. @@ -578,9 +648,9 @@ int sbi_debug_console_read(char *bytes, unsigned int num_bytes) void __init sbi_init(void) { + bool srst_power_off = false; int ret; - sbi_set_power_off(); ret = sbi_get_spec_version(); if (ret > 0) sbi_spec_version = ret; @@ -609,22 +679,31 @@ void __init sbi_init(void) } else { __sbi_rfence = __sbi_rfence_v01; } - if ((sbi_spec_version >= sbi_mk_version(0, 3)) && + if (sbi_spec_version >= sbi_mk_version(0, 3) && sbi_probe_extension(SBI_EXT_SRST)) { pr_info("SBI SRST extension detected\n"); - pm_power_off = sbi_srst_power_off; + register_platform_power_off(sbi_srst_power_off); + srst_power_off = true; sbi_srst_reboot_nb.notifier_call = sbi_srst_reboot; sbi_srst_reboot_nb.priority = 192; register_restart_handler(&sbi_srst_reboot_nb); } - if ((sbi_spec_version >= sbi_mk_version(2, 0)) && - (sbi_probe_extension(SBI_EXT_DBCN) > 0)) { + if (sbi_spec_version >= sbi_mk_version(2, 0) && + sbi_probe_extension(SBI_EXT_DBCN) > 0) { pr_info("SBI DBCN extension detected\n"); sbi_debug_console_available = true; } + if (sbi_spec_version >= sbi_mk_version(3, 0) && + sbi_probe_extension(SBI_EXT_FWFT)) { + pr_info("SBI FWFT extension detected\n"); + sbi_fwft_supported = true; + } } else { __sbi_set_timer = __sbi_set_timer_v01; __sbi_send_ipi = __sbi_send_ipi_v01; __sbi_rfence = __sbi_rfence_v01; } + + if (!srst_power_off) + sbi_set_power_off(); } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index f1793630fc51..c89cc272440b 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -21,6 +21,8 @@ #include <linux/efi.h> #include <linux/crash_dump.h> #include <linux/panic_notifier.h> +#include <linux/jump_label.h> +#include <linux/gcd.h> #include <asm/acpi.h> #include <asm/alternative.h> @@ -44,12 +46,9 @@ * This is used before the kernel initializes the BSS so it can't be in the * BSS. */ -atomic_t hart_lottery __section(".sdata") -#ifdef CONFIG_XIP_KERNEL -= ATOMIC_INIT(0xC001BEEF) -#endif -; +atomic_t hart_lottery __section(".sdata"); unsigned long boot_cpu_hartid; +EXPORT_SYMBOL_GPL(boot_cpu_hartid); /* * Place kernel memory regions on the resource tree so that @@ -66,6 +65,9 @@ static struct resource bss_res = { .name = "Kernel bss", }; static struct resource elfcorehdr_res = { .name = "ELF Core hdr", }; #endif +static int num_standard_resources; +static struct resource *standard_resources; + static int __init add_resource(struct resource *parent, struct resource *res) { @@ -139,7 +141,7 @@ static void __init init_resources(void) struct resource *res = NULL; struct resource *mem_res = NULL; size_t mem_res_sz = 0; - int num_resources = 0, res_idx = 0; + int num_resources = 0, res_idx = 0, non_resv_res = 0; int ret = 0; /* + 1 as memblock_alloc() might increase memblock.reserved.cnt */ @@ -193,6 +195,7 @@ static void __init init_resources(void) /* Add /memory regions to the resource tree */ for_each_mem_region(region) { res = &mem_res[res_idx--]; + non_resv_res++; if (unlikely(memblock_is_nomap(region))) { res->name = "Reserved"; @@ -210,6 +213,9 @@ static void __init init_resources(void) goto error; } + num_standard_resources = non_resv_res; + standard_resources = &mem_res[res_idx + 1]; + /* Clean-up any unused pre-allocated resources */ if (res_idx >= 0) memblock_free(mem_res, (res_idx + 1) * sizeof(*mem_res)); @@ -221,6 +227,33 @@ static void __init init_resources(void) memblock_free(mem_res, mem_res_sz); } +static int __init reserve_memblock_reserved_regions(void) +{ + u64 i, j; + + for (i = 0; i < num_standard_resources; i++) { + struct resource *mem = &standard_resources[i]; + phys_addr_t r_start, r_end, mem_size = resource_size(mem); + + if (!memblock_is_region_reserved(mem->start, mem_size)) + continue; + + for_each_reserved_mem_range(j, &r_start, &r_end) { + resource_size_t start, end; + + start = max(PFN_PHYS(PFN_DOWN(r_start)), mem->start); + end = min(PFN_PHYS(PFN_UP(r_end)) - 1, mem->end); + + if (start > mem->end || end < mem->start) + continue; + + reserve_region_with_split(mem, start, end, "Reserved"); + } + } + + return 0; +} +arch_initcall(reserve_memblock_reserved_regions); static void __init parse_dtb(void) { @@ -235,11 +268,6 @@ static void __init parse_dtb(void) } else { pr_err("No DTB passed to the kernel\n"); } - -#ifdef CONFIG_CMDLINE_FORCE - strscpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); - pr_info("Forcing kernel command line to: %s\n", boot_command_line); -#endif } #if defined(CONFIG_RISCV_COMBO_SPINLOCKS) @@ -258,6 +286,7 @@ static void __init riscv_spinlock_init(void) if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZACAS) && riscv_isa_extension_available(NULL, ZABHA) && riscv_isa_extension_available(NULL, ZACAS)) { using_ext = "using Zabha"; @@ -298,11 +327,14 @@ void __init setup_arch(char **cmdline_p) /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); + if (acpi_disabled) { #if IS_ENABLED(CONFIG_BUILTIN_DTB) - unflatten_and_copy_device_tree(); + unflatten_and_copy_device_tree(); #else - unflatten_device_tree(); + unflatten_device_tree(); #endif + } + misc_mem_init(); init_resources(); @@ -322,8 +354,8 @@ void __init setup_arch(char **cmdline_p) riscv_init_cbo_blocksizes(); riscv_fill_hwcap(); - init_rt_signal_env(); apply_boot_alternatives(); + init_rt_signal_env(); if (IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM) && riscv_isa_extension_available(NULL, ZICBOM)) @@ -332,6 +364,9 @@ void __init setup_arch(char **cmdline_p) riscv_user_isa_enable(); riscv_spinlock_init(); + + if (!IS_ENABLED(CONFIG_RISCV_ISA_ZBB) || !riscv_isa_extension_available(NULL, ZBB)) + static_branch_disable(&efficient_ffs_key); } bool arch_cpu_is_hotpluggable(int cpu) diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 94e905eea1de..59784dc117e4 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -22,11 +22,13 @@ #include <asm/vector.h> #include <asm/csr.h> #include <asm/cacheflush.h> +#include <asm/usercfi.h> unsigned long signal_minsigstksz __ro_after_init; extern u32 __user_rt_sigreturn[2]; static size_t riscv_v_sc_size __ro_after_init; +static size_t riscv_zicfiss_sc_size __ro_after_init; #define DEBUG_SIG 0 @@ -68,18 +70,19 @@ static long save_fp_state(struct pt_regs *regs, #define restore_fp_state(task, regs) (0) #endif -#ifdef CONFIG_RISCV_ISA_V - -static long save_v_state(struct pt_regs *regs, void __user **sc_vec) +static long save_v_state(struct pt_regs *regs, void __user *sc_vec) { - struct __riscv_ctx_hdr __user *hdr; struct __sc_riscv_v_state __user *state; void __user *datap; long err; - hdr = *sc_vec; - /* Place state to the user's signal context space after the hdr */ - state = (struct __sc_riscv_v_state __user *)(hdr + 1); + if (!IS_ENABLED(CONFIG_RISCV_ISA_V) || + !((has_vector() || has_xtheadvector()) && + riscv_v_vstate_query(regs))) + return 0; + + /* Place state to the user's signal context space */ + state = (struct __sc_riscv_v_state __user *)sc_vec; /* Point datap right after the end of __sc_riscv_v_state */ datap = state + 1; @@ -97,15 +100,11 @@ static long save_v_state(struct pt_regs *regs, void __user **sc_vec) err |= __put_user((__force void *)datap, &state->v_state.datap); /* Copy the whole vector content to user space datap. */ err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize); - /* Copy magic to the user space after saving all vector conetext */ - err |= __put_user(RISCV_V_MAGIC, &hdr->magic); - err |= __put_user(riscv_v_sc_size, &hdr->size); if (unlikely(err)) - return err; + return -EFAULT; - /* Only progress the sv_vec if everything has done successfully */ - *sc_vec += riscv_v_sc_size; - return 0; + /* Only return the size if everything has done successfully */ + return riscv_v_sc_size; } /* @@ -142,10 +141,80 @@ static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec) */ return copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize); } -#else -#define save_v_state(task, regs) (0) -#define __restore_v_state(task, regs) (0) -#endif + +static long save_cfiss_state(struct pt_regs *regs, void __user *sc_cfi) +{ + struct __sc_riscv_cfi_state __user *state = sc_cfi; + unsigned long ss_ptr = 0; + long err = 0; + + if (!is_shstk_enabled(current)) + return 0; + + /* + * Save a pointer to the shadow stack itself on shadow stack as a form of token. + * A token on the shadow stack gives the following properties: + * - Safe save and restore for shadow stack switching. Any save of a shadow stack + * must have saved a token on the shadow stack. Similarly any restore of shadow + * stack must check the token before restore. Since writing to the shadow stack with + * address of the shadow stack itself is not easily allowed, a restore without a save + * is quite difficult for an attacker to perform. + * - A natural break. A token in shadow stack provides a natural break in shadow stack + * So a single linear range can be bucketed into different shadow stack segments. Any + * sspopchk will detect the condition and fault to kernel as a sw check exception. + */ + err |= save_user_shstk(current, &ss_ptr); + err |= __put_user(ss_ptr, &state->ss_ptr); + if (unlikely(err)) + return -EFAULT; + + return riscv_zicfiss_sc_size; +} + +static long __restore_cfiss_state(struct pt_regs *regs, void __user *sc_cfi) +{ + struct __sc_riscv_cfi_state __user *state = sc_cfi; + unsigned long ss_ptr = 0; + long err; + + /* + * Restore shadow stack as a form of token stored on the shadow stack itself as a safe + * way to restore. + * A token on the shadow stack gives the following properties: + * - Safe save and restore for shadow stack switching. Any save of shadow stack + * must have saved a token on shadow stack. Similarly any restore of shadow + * stack must check the token before restore. Since writing to a shadow stack with + * the address of shadow stack itself is not easily allowed, a restore without a save + * is quite difficult for an attacker to perform. + * - A natural break. A token in the shadow stack provides a natural break in shadow stack + * So a single linear range can be bucketed into different shadow stack segments. + * sspopchk will detect the condition and fault to kernel as a sw check exception. + */ + err = __copy_from_user(&ss_ptr, &state->ss_ptr, sizeof(unsigned long)); + + if (unlikely(err)) + return err; + + return restore_user_shstk(current, ss_ptr); +} + +struct arch_ext_priv { + __u32 magic; + long (*save)(struct pt_regs *regs, void __user *sc_vec); +}; + +static struct arch_ext_priv arch_ext_list[] = { + { + .magic = RISCV_V_MAGIC, + .save = &save_v_state, + }, + { + .magic = RISCV_ZICFISS_MAGIC, + .save = &save_cfiss_state, + }, +}; + +static const size_t nr_arch_exts = ARRAY_SIZE(arch_ext_list); static long restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) @@ -195,6 +264,12 @@ static long restore_sigcontext(struct pt_regs *regs, err = __restore_v_state(regs, sc_ext_ptr); break; + case RISCV_ZICFISS_MAGIC: + if (!is_shstk_enabled(current) || size != riscv_zicfiss_sc_size) + return -EINVAL; + + err = __restore_cfiss_state(regs, sc_ext_ptr); + break; default: return -EINVAL; } @@ -215,6 +290,10 @@ static size_t get_rt_frame_size(bool cal_all) if (cal_all || riscv_v_vstate_query(task_pt_regs(current))) total_context_size += riscv_v_sc_size; } + + if (is_shstk_enabled(current)) + total_context_size += riscv_zicfiss_sc_size; + /* * Preserved a __riscv_ctx_hdr for END signal context header if an * extension uses __riscv_extra_ext_header @@ -276,7 +355,8 @@ static long setup_sigcontext(struct rt_sigframe __user *frame, { struct sigcontext __user *sc = &frame->uc.uc_mcontext; struct __riscv_ctx_hdr __user *sc_ext_ptr = &sc->sc_extdesc.hdr; - long err; + struct arch_ext_priv *arch_ext; + long err, i, ext_size; /* sc_regs is structured the same as the start of pt_regs */ err = __copy_to_user(&sc->sc_regs, regs, sizeof(sc->sc_regs)); @@ -284,8 +364,20 @@ static long setup_sigcontext(struct rt_sigframe __user *frame, if (has_fpu()) err |= save_fp_state(regs, &sc->sc_fpregs); /* Save the vector state. */ - if ((has_vector() || has_xtheadvector()) && riscv_v_vstate_query(regs)) - err |= save_v_state(regs, (void __user **)&sc_ext_ptr); + for (i = 0; i < nr_arch_exts; i++) { + arch_ext = &arch_ext_list[i]; + if (!arch_ext->save) + continue; + + ext_size = arch_ext->save(regs, sc_ext_ptr + 1); + if (ext_size <= 0) { + err |= ext_size; + } else { + err |= __put_user(arch_ext->magic, &sc_ext_ptr->magic); + err |= __put_user(ext_size, &sc_ext_ptr->size); + sc_ext_ptr = (void __user *)sc_ext_ptr + ext_size; + } + } /* Write zero to fp-reserved space and check it on restore_sigcontext */ err |= __put_user(0, &sc->sc_extdesc.reserved); /* And put END __riscv_ctx_hdr at the end. */ @@ -345,6 +437,11 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, #ifdef CONFIG_MMU regs->ra = (unsigned long)VDSO_SYMBOL( current->mm->context.vdso, rt_sigreturn); + + /* if bcfi is enabled x1 (ra) and x5 (t0) must match. not sure if we need this? */ + if (is_shstk_enabled(current)) + regs->t0 = regs->ra; + #else /* * For the nommu case we don't have a VDSO. Instead we push two @@ -473,6 +570,9 @@ void __init init_rt_signal_env(void) { riscv_v_sc_size = sizeof(struct __riscv_ctx_hdr) + sizeof(struct __sc_riscv_v_state) + riscv_v_vsize; + + riscv_zicfiss_sc_size = sizeof(struct __riscv_ctx_hdr) + + sizeof(struct __sc_riscv_cfi_state); /* * Determine the stack space required for guaranteed signal delivery. * The signal_minsigstksz will be populated into the AT_MINSIGSTKSZ entry diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index d58b5e751286..5ed5095320e6 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -40,6 +40,17 @@ enum ipi_message_type { IPI_MAX }; +static const char * const ipi_names[] = { + [IPI_RESCHEDULE] = "Rescheduling interrupts", + [IPI_CALL_FUNC] = "Function call interrupts", + [IPI_CPU_STOP] = "CPU stop interrupts", + [IPI_CPU_CRASH_STOP] = "CPU stop (for crash dump) interrupts", + [IPI_IRQ_WORK] = "IRQ work interrupts", + [IPI_TIMER] = "Timer broadcast interrupts", + [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts", + [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts", +}; + unsigned long __cpuid_to_hartid_map[NR_CPUS] __ro_after_init = { [0 ... NR_CPUS-1] = INVALID_HARTID }; @@ -48,6 +59,8 @@ EXPORT_SYMBOL_GPL(__cpuid_to_hartid_map); void __init smp_setup_processor_id(void) { cpuid_to_hartid_map(0) = boot_cpu_hartid; + + pr_info("Booting Linux on hartid %lu\n", boot_cpu_hartid); } static DEFINE_PER_CPU_READ_MOSTLY(int, ipi_dummy_dev); @@ -197,7 +210,7 @@ void riscv_ipi_set_virq_range(int virq, int nr) /* Request IPIs */ for (i = 0; i < nr_ipi; i++) { err = request_percpu_irq(ipi_virq_base + i, handle_IPI, - "IPI", &ipi_dummy_dev); + ipi_names[i], &ipi_dummy_dev); WARN_ON(err); ipi_desc[i] = irq_to_desc(ipi_virq_base + i); @@ -208,17 +221,6 @@ void riscv_ipi_set_virq_range(int virq, int nr) riscv_ipi_enable(); } -static const char * const ipi_names[] = { - [IPI_RESCHEDULE] = "Rescheduling interrupts", - [IPI_CALL_FUNC] = "Function call interrupts", - [IPI_CPU_STOP] = "CPU stop interrupts", - [IPI_CPU_CRASH_STOP] = "CPU stop (for crash dump) interrupts", - [IPI_IRQ_WORK] = "IRQ work interrupts", - [IPI_TIMER] = "Timer broadcast interrupts", - [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts", - [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts", -}; - void show_ipi_stats(struct seq_file *p, int prec) { unsigned int cpu, i; diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index e36d20205bd7..8b628580fe11 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -39,7 +39,9 @@ #include "head.h" +#ifndef CONFIG_HOTPLUG_PARALLEL static DECLARE_COMPLETION(cpu_running); +#endif void __init smp_prepare_cpus(unsigned int max_cpus) { @@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle) return -EOPNOTSUPP; } +#ifdef CONFIG_HOTPLUG_PARALLEL +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle) +{ + return start_secondary_cpu(cpu, tidle); +} +#else int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret = 0; @@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) return ret; } +#endif void __init smp_cpus_done(unsigned int max_cpus) { @@ -225,25 +234,31 @@ asmlinkage __visible void smp_callin(void) mmgrab(mm); current->active_mm = mm; +#ifdef CONFIG_HOTPLUG_PARALLEL + cpuhp_ap_sync_alive(); +#endif + store_cpu_topology(curr_cpuid); notify_cpu_starting(curr_cpuid); riscv_ipi_enable(); numa_add_cpu(curr_cpuid); + + pr_debug("CPU%u: Booted secondary hartid %lu\n", curr_cpuid, + cpuid_to_hartid_map(curr_cpuid)); + set_cpu_online(curr_cpuid, true); /* - * Remote cache and TLB flushes are ignored while the CPU is offline, - * so flush them both right now just in case. + * Remote instruction cache and TLB flushes are ignored while the CPU + * is offline, so flush them both right now just in case. */ local_flush_icache_all(); local_flush_tlb_all(); +#ifndef CONFIG_HOTPLUG_PARALLEL complete(&cpu_running); - /* - * Disable preemption before enabling interrupts, so we don't try to - * schedule a CPU that hasn't actually started yet. - */ +#endif local_irq_enable(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index a0516172a33c..7d69e3b6bab4 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -8,7 +8,7 @@ #include <asm/soc.h> /* - * This is called extremly early, before parse_dtb(), to allow initializing + * This is called extremely early, before parse_dtb(), to allow initializing * SoC hardware before memory or any device driver initialization. */ void __init soc_early_init(void) diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index d4355c770c36..b41b6255751c 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -16,6 +16,22 @@ #ifdef CONFIG_FRAME_POINTER +/* + * This disables KASAN checking when reading a value from another task's stack, + * since the other task could be running on another CPU and could have poisoned + * the stack in the meantime. + */ +#define READ_ONCE_TASK_STACK(task, x) \ +({ \ + unsigned long val; \ + unsigned long addr = x; \ + if ((task) == current) \ + val = READ_ONCE(addr); \ + else \ + val = READ_ONCE_NOCHECK(addr); \ + val; \ +}) + extern asmlinkage void handle_exception(void); extern unsigned long ret_from_exception_end; @@ -69,12 +85,13 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, fp = frame->ra; pc = regs->ra; } else { - fp = frame->fp; - pc = ftrace_graph_ret_addr(current, &graph_idx, frame->ra, + fp = READ_ONCE_TASK_STACK(task, frame->fp); + pc = READ_ONCE_TASK_STACK(task, frame->ra); + pc = ftrace_graph_ret_addr(current, &graph_idx, pc, &frame->ra); if (pc >= (unsigned long)handle_exception && pc < (unsigned long)&ret_from_exception_end) { - if (unlikely(!__kernel_text_address(pc) || !fn(arg, pc))) + if (unlikely(!fn(arg, pc))) break; pc = ((struct pt_regs *)sp)->epc; diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index 9a8a0dc035b2..3efbf7874f3b 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -30,6 +30,13 @@ void suspend_save_csrs(struct suspend_context *context) */ #ifdef CONFIG_MMU + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) { + context->stimecmp = csr_read(CSR_STIMECMP); +#if __riscv_xlen < 64 + context->stimecmph = csr_read(CSR_STIMECMPH); +#endif + } + context->satp = csr_read(CSR_SATP); #endif } @@ -43,6 +50,14 @@ void suspend_restore_csrs(struct suspend_context *context) csr_write(CSR_IE, context->ie); #ifdef CONFIG_MMU + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) { +#if __riscv_xlen < 64 + csr_write(CSR_STIMECMP, ULONG_MAX); + csr_write(CSR_STIMECMPH, context->stimecmph); +#endif + csr_write(CSR_STIMECMP, context->stimecmp); + } + csr_write(CSR_SATP, context->satp); #endif } @@ -63,7 +78,7 @@ int cpu_suspend(unsigned long arg, suspend_save_csrs(&context); /* - * Function graph tracer state gets incosistent when the kernel + * Function graph tracer state gets inconsistent when the kernel * calls functions that never return (aka finishers) hence disable * graph tracing during their execution. */ diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S index 2d54f309c140..d71b55fd6259 100644 --- a/arch/riscv/kernel/suspend_entry.S +++ b/arch/riscv/kernel/suspend_entry.S @@ -10,7 +10,6 @@ #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/csr.h> -#include <asm/xip_fixup.h> .text .altmacro @@ -70,7 +69,6 @@ SYM_TYPED_FUNC_START(__cpu_resume_enter) /* Enable MMU */ la a0, swapper_pg_dir - XIP_FIXUP_OFFSET a0 call relocate_enable_mmu /* Restore A0 and A1 */ diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index bcd3b816306c..1659d31fd288 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -5,6 +5,9 @@ * more details. */ #include <linux/syscalls.h> +#include <linux/completion.h> +#include <linux/atomic.h> +#include <linux/once.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> #include <asm/hwprobe.h> @@ -15,10 +18,20 @@ #include <asm/uaccess.h> #include <asm/unistd.h> #include <asm/vector.h> +#include <asm/vendor_extensions/mips_hwprobe.h> +#include <asm/vendor_extensions/sifive_hwprobe.h> #include <asm/vendor_extensions/thead_hwprobe.h> #include <vdso/vsyscall.h> +#define EXT_KEY(isa_arg, ext, pv, missing) \ + do { \ + if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext)) \ + pv |= RISCV_HWPROBE_EXT_##ext; \ + else \ + missing |= RISCV_HWPROBE_EXT_##ext; \ + } while (false) + static void hwprobe_arch_id(struct riscv_hwprobe *pair, const struct cpumask *cpus) { @@ -26,6 +39,11 @@ static void hwprobe_arch_id(struct riscv_hwprobe *pair, bool first = true; int cpu; + if (pair->key != RISCV_HWPROBE_KEY_MVENDORID && + pair->key != RISCV_HWPROBE_KEY_MIMPID && + pair->key != RISCV_HWPROBE_KEY_MARCHID) + goto out; + for_each_cpu(cpu, cpus) { u64 cpu_id; @@ -56,6 +74,7 @@ static void hwprobe_arch_id(struct riscv_hwprobe *pair, } } +out: pair->value = id; } @@ -82,86 +101,117 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, for_each_cpu(cpu, cpus) { struct riscv_isainfo *isainfo = &hart_isa[cpu]; -#define EXT_KEY(ext) \ - do { \ - if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext)) \ - pair->value |= RISCV_HWPROBE_EXT_##ext; \ - else \ - missing |= RISCV_HWPROBE_EXT_##ext; \ - } while (false) - /* * Only use EXT_KEY() for extensions which can be exposed to userspace, * regardless of the kernel's configuration, as no other checks, besides * presence in the hart_isa bitmap, are made. */ - EXT_KEY(ZACAS); - EXT_KEY(ZAWRS); - EXT_KEY(ZBA); - EXT_KEY(ZBB); - EXT_KEY(ZBC); - EXT_KEY(ZBKB); - EXT_KEY(ZBKC); - EXT_KEY(ZBKX); - EXT_KEY(ZBS); - EXT_KEY(ZCA); - EXT_KEY(ZCB); - EXT_KEY(ZCMOP); - EXT_KEY(ZICBOZ); - EXT_KEY(ZICOND); - EXT_KEY(ZIHINTNTL); - EXT_KEY(ZIHINTPAUSE); - EXT_KEY(ZIMOP); - EXT_KEY(ZKND); - EXT_KEY(ZKNE); - EXT_KEY(ZKNH); - EXT_KEY(ZKSED); - EXT_KEY(ZKSH); - EXT_KEY(ZKT); - EXT_KEY(ZTSO); + EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing); + EXT_KEY(isainfo->isa, ZABHA, pair->value, missing); + EXT_KEY(isainfo->isa, ZACAS, pair->value, missing); + EXT_KEY(isainfo->isa, ZALASR, pair->value, missing); + EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing); + EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing); + EXT_KEY(isainfo->isa, ZBA, pair->value, missing); + EXT_KEY(isainfo->isa, ZBB, pair->value, missing); + EXT_KEY(isainfo->isa, ZBC, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKB, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKC, pair->value, missing); + EXT_KEY(isainfo->isa, ZBKX, pair->value, missing); + EXT_KEY(isainfo->isa, ZBS, pair->value, missing); + EXT_KEY(isainfo->isa, ZCA, pair->value, missing); + EXT_KEY(isainfo->isa, ZCB, pair->value, missing); + EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing); + EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing); + EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing); + EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing); + EXT_KEY(isainfo->isa, ZICOND, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing); + EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing); + EXT_KEY(isainfo->isa, ZILSD, pair->value, missing); + EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing); + EXT_KEY(isainfo->isa, ZKND, pair->value, missing); + EXT_KEY(isainfo->isa, ZKNE, pair->value, missing); + EXT_KEY(isainfo->isa, ZKNH, pair->value, missing); + EXT_KEY(isainfo->isa, ZKSED, pair->value, missing); + EXT_KEY(isainfo->isa, ZKSH, pair->value, missing); + EXT_KEY(isainfo->isa, ZKT, pair->value, missing); + EXT_KEY(isainfo->isa, ZTSO, pair->value, missing); /* * All the following extensions must depend on the kernel * support of V. */ if (has_vector()) { - EXT_KEY(ZVBB); - EXT_KEY(ZVBC); - EXT_KEY(ZVE32F); - EXT_KEY(ZVE32X); - EXT_KEY(ZVE64D); - EXT_KEY(ZVE64F); - EXT_KEY(ZVE64X); - EXT_KEY(ZVFH); - EXT_KEY(ZVFHMIN); - EXT_KEY(ZVKB); - EXT_KEY(ZVKG); - EXT_KEY(ZVKNED); - EXT_KEY(ZVKNHA); - EXT_KEY(ZVKNHB); - EXT_KEY(ZVKSED); - EXT_KEY(ZVKSH); - EXT_KEY(ZVKT); + EXT_KEY(isainfo->isa, ZVBB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVBC, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing); + EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFH, pair->value, missing); + EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKG, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing); + EXT_KEY(isainfo->isa, ZVKT, pair->value, missing); } - if (has_fpu()) { - EXT_KEY(ZCD); - EXT_KEY(ZCF); - EXT_KEY(ZFA); - EXT_KEY(ZFH); - EXT_KEY(ZFHMIN); - } + EXT_KEY(isainfo->isa, ZCD, pair->value, missing); + EXT_KEY(isainfo->isa, ZCF, pair->value, missing); + EXT_KEY(isainfo->isa, ZFA, pair->value, missing); + EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing); + EXT_KEY(isainfo->isa, ZFH, pair->value, missing); + EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing); if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM)) - EXT_KEY(SUPM); -#undef EXT_KEY + EXT_KEY(isainfo->isa, SUPM, pair->value, missing); + } + + /* Now turn off reporting features if any CPU is missing it. */ + pair->value &= ~missing; +} + +static void hwprobe_isa_ext1(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + int cpu; + u64 missing = 0; + + pair->value = 0; + + /* + * Loop through and record extensions that 1) anyone has, and 2) anyone + * doesn't have. + */ + for_each_cpu(cpu, cpus) { + struct riscv_isainfo *isainfo = &hart_isa[cpu]; + + /* + * Only use EXT_KEY() for extensions which can be + * exposed to userspace, regardless of the kernel's + * configuration, as no other checks, besides presence + * in the hart_isa bitmap, are made. + */ + EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing); } /* Now turn off reporting features if any CPU is missing it. */ pair->value &= ~missing; } -static bool hwprobe_ext0_has(const struct cpumask *cpus, unsigned long ext) +static bool hwprobe_ext0_has(const struct cpumask *cpus, u64 ext) { struct riscv_hwprobe pair; @@ -265,6 +315,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, hwprobe_isa_ext0(pair, cpus); break; + case RISCV_HWPROBE_KEY_IMA_EXT_1: + hwprobe_isa_ext1(pair, cpus); + break; + case RISCV_HWPROBE_KEY_CPUPERF_0: case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: pair->value = hwprobe_misaligned(cpus); @@ -279,6 +333,16 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) pair->value = riscv_cboz_block_size; break; + case RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE: + pair->value = 0; + if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOM)) + pair->value = riscv_cbom_block_size; + break; + case RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE: + pair->value = 0; + if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOP)) + pair->value = riscv_cbop_block_size; + break; case RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS: pair->value = user_max_virt_addr(); break; @@ -287,9 +351,16 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, pair->value = riscv_timebase; break; + case RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0: + hwprobe_isa_vendor_ext_sifive_0(pair, cpus); + break; + case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0: hwprobe_isa_vendor_ext_thead_0(pair, cpus); break; + case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0: + hwprobe_isa_vendor_ext_mips_0(pair, cpus); + break; /* * For forward compatibility, unknown keys don't fail the whole @@ -433,29 +504,32 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, return 0; } -static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, - size_t pair_count, size_t cpusetsize, - unsigned long __user *cpus_user, - unsigned int flags) -{ - if (flags & RISCV_HWPROBE_WHICH_CPUS) - return hwprobe_get_cpus(pairs, pair_count, cpusetsize, - cpus_user, flags); +#ifdef CONFIG_MMU - return hwprobe_get_values(pairs, pair_count, cpusetsize, - cpus_user, flags); +static DECLARE_COMPLETION(boot_probes_done); +static atomic_t pending_boot_probes = ATOMIC_INIT(1); + +void riscv_hwprobe_register_async_probe(void) +{ + atomic_inc(&pending_boot_probes); } -#ifdef CONFIG_MMU +void riscv_hwprobe_complete_async_probe(void) +{ + if (atomic_dec_and_test(&pending_boot_probes)) + complete(&boot_probes_done); +} -static int __init init_hwprobe_vdso_data(void) +static int complete_hwprobe_vdso_data(void) { - struct vdso_data *vd = __arch_get_k_vdso_data(); - struct arch_vdso_time_data *avd = &vd->arch_data; + struct vdso_arch_data *avd = vdso_k_arch_data; u64 id_bitsmash = 0; struct riscv_hwprobe pair; int key; + if (unlikely(!atomic_dec_and_test(&pending_boot_probes))) + wait_for_completion(&boot_probes_done); + /* * Initialize vDSO data with the answers for the "all CPUs" case, to * save a syscall in the common case. @@ -483,13 +557,52 @@ static int __init init_hwprobe_vdso_data(void) * vDSO should defer to the kernel for exotic cpu masks. */ avd->homogeneous_cpus = id_bitsmash != 0 && id_bitsmash != -1; + + /* + * Make sure all the VDSO values are visible before we look at them. + * This pairs with the implicit "no speculativly visible accesses" + * barrier in the VDSO hwprobe code. + */ + smp_wmb(); + avd->ready = true; + return 0; +} + +static int __init init_hwprobe_vdso_data(void) +{ + struct vdso_arch_data *avd = vdso_k_arch_data; + + /* + * Prevent the vDSO cached values from being used, as they're not ready + * yet. + */ + avd->ready = false; return 0; } arch_initcall_sync(init_hwprobe_vdso_data); +#else + +static int complete_hwprobe_vdso_data(void) { return 0; } + #endif /* CONFIG_MMU */ +static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs, + size_t pair_count, size_t cpusetsize, + unsigned long __user *cpus_user, + unsigned int flags) +{ + DO_ONCE_SLEEPABLE(complete_hwprobe_vdso_data); + + if (flags & RISCV_HWPROBE_WHICH_CPUS) + return hwprobe_get_cpus(pairs, pair_count, cpusetsize, + cpus_user, flags); + + return hwprobe_get_values(pairs, pair_count, cpusetsize, + cpus_user, flags); +} + SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs, size_t, pair_count, size_t, cpusetsize, unsigned long __user *, cpus, unsigned int, flags) diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index d77afe05578f..22fc9b3268be 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -7,15 +7,25 @@ #include <linux/syscalls.h> #include <asm/cacheflush.h> +#include <asm-generic/mman-common.h> static long riscv_sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, off_t offset, + unsigned long fd, unsigned long offset, unsigned long page_shift_offset) { if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) return -EINVAL; + /* + * If PROT_WRITE is specified then extend that to PROT_READ + * protection_map[VM_WRITE] is now going to select shadow stack encodings. + * So specifying PROT_WRITE actually should select protection_map [VM_WRITE | VM_READ] + * If user wants to create shadow stack then they should use `map_shadow_stack` syscall. + */ + if (unlikely((prot & PROT_WRITE) && !(prot & PROT_READ))) + prot |= PROT_READ; + return ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> (PAGE_SHIFT - page_shift_offset)); } diff --git a/arch/riscv/kernel/tests/Kconfig.debug b/arch/riscv/kernel/tests/Kconfig.debug index 78cea5d2c270..40f8dafffa0a 100644 --- a/arch/riscv/kernel/tests/Kconfig.debug +++ b/arch/riscv/kernel/tests/Kconfig.debug @@ -30,6 +30,18 @@ config RISCV_MODULE_LINKING_KUNIT If unsure, say N. +config RISCV_KPROBES_KUNIT + tristate "KUnit test for riscv kprobes" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on KPROBES + default KUNIT_ALL_TESTS + help + Enable testing for riscv kprobes. Useful for riscv and/or kprobes + development. The test verifies that kprobes do not change the behaviour + of some sample functions. + + If unsure, say N. + endif # RUNTIME_TESTING_MENU endmenu # "arch/riscv/kernel runtime Testing" diff --git a/arch/riscv/kernel/tests/Makefile b/arch/riscv/kernel/tests/Makefile index 7d6c76cffe20..407e7e6c28dc 100644 --- a/arch/riscv/kernel/tests/Makefile +++ b/arch/riscv/kernel/tests/Makefile @@ -1 +1,2 @@ obj-$(CONFIG_RISCV_MODULE_LINKING_KUNIT) += module_test/ +obj-$(CONFIG_RISCV_KPROBES_KUNIT) += kprobes/ diff --git a/arch/riscv/kernel/tests/kprobes/Makefile b/arch/riscv/kernel/tests/kprobes/Makefile new file mode 100644 index 000000000000..df7256f62313 --- /dev/null +++ b/arch/riscv/kernel/tests/kprobes/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_RISCV_KPROBES_KUNIT) += kprobes_riscv_kunit.o + +kprobes_riscv_kunit-objs := test-kprobes.o test-kprobes-asm.o diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S new file mode 100644 index 000000000000..f16deee9e091 --- /dev/null +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include "test-kprobes.h" + +SYM_FUNC_START(test_kprobes_add) + li a1, KPROBE_TEST_MAGIC_UPPER + li a2, KPROBE_TEST_MAGIC_LOWER +test_kprobes_add_addr1: + add a1, a1, a2 +test_kprobes_add_addr2: + add a0, a1, x0 + ret +SYM_FUNC_END(test_kprobes_add) + +SYM_FUNC_START(test_kprobes_jal) + li a0, 0 + mv a1, ra + .option push + .option norvc +test_kprobes_jal_addr1: + jal x0, 2f + ret + .option pop +1: li a0, KPROBE_TEST_MAGIC_UPPER + ret + .option push + .option norvc +test_kprobes_jal_addr2: +2: jal 1b + .option pop + li a2, KPROBE_TEST_MAGIC_LOWER + add a0, a0, a2 + jr a1 +SYM_FUNC_END(test_kprobes_jal) + +SYM_FUNC_START(test_kprobes_jalr) + la a0, 1f + mv a1, ra + .option push + .option norvc +test_kprobes_jalr_addr: + jalr a0 + .option pop + li t0, KPROBE_TEST_MAGIC_UPPER + add a0, a0, t0 + jr a1 +1: li a0, KPROBE_TEST_MAGIC_LOWER + ret +SYM_FUNC_END(test_kprobes_jalr) + +SYM_FUNC_START(test_kprobes_auipc) +test_kprobes_auipc_addr: + auipc a0, KPROBE_TEST_MAGIC_LOWER + la a1, test_kprobes_auipc_addr + sub a0, a0, a1 + srli a0, a0, 12 + li a1, KPROBE_TEST_MAGIC_UPPER + add a0, a0, a1 + ret +SYM_FUNC_END(test_kprobes_auipc) + +SYM_FUNC_START(test_kprobes_branch) + .option push + .option norvc + li a0, 0 + li a1, 1 + li a2, 2 +test_kprobes_branch_addr1: + beqz a0, 1f + ret +1: +test_kprobes_branch_addr2: + beqz a1, 3f +test_kprobes_branch_addr3: + bnez a0, 3f +test_kprobes_branch_addr4: + bnez a2, 1f + ret +1: +test_kprobes_branch_addr5: + bge a1, a2, 3f +test_kprobes_branch_addr6: + bge a2, a1, 2f + ret +1: + li t0, KPROBE_TEST_MAGIC_UPPER + add a0, a0, t0 + ret +2: +test_kprobes_branch_addr7: + blt a2, a1, 3f + li a0, KPROBE_TEST_MAGIC_LOWER +test_kprobes_branch_addr8: + blt a1, a2, 1b +3: + li a0, 0 + ret + .option pop +SYM_FUNC_END(test_kprobes_branch) + +#ifdef CONFIG_RISCV_ISA_C + +SYM_FUNC_START(test_kprobes_c_j) + li a0, 0 +test_kprobes_branch_c_j_addr1: + c.j 2f +1: + li a1, KPROBE_TEST_MAGIC_UPPER + add a0, a0, a1 + ret +2: li a0, KPROBE_TEST_MAGIC_LOWER +test_kprobes_branch_c_j_addr2: + c.j 1b +SYM_FUNC_END(test_kprobes_c_j) + +SYM_FUNC_START(test_kprobes_c_jr) + la a0, 2f +test_kprobes_c_jr_addr1: + c.jr a0 + ret +1: li a1, KPROBE_TEST_MAGIC_LOWER + add a0, a0, a1 + ret +2: + li a0, KPROBE_TEST_MAGIC_UPPER + la a1, 1b +test_kprobes_c_jr_addr2: + c.jr a1 +SYM_FUNC_END(test_kprobes_c_jr) + +SYM_FUNC_START(test_kprobes_c_jalr) + mv a1, ra + la a0, 1f +test_kprobes_c_jalr_addr: + c.jalr a0 + li a2, KPROBE_TEST_MAGIC_UPPER + add a0, a0, a2 + jr a1 +1: li a0, KPROBE_TEST_MAGIC_LOWER + ret +SYM_FUNC_END(test_kprobes_c_jalr) + +SYM_FUNC_START(test_kprobes_c_beqz) + li a0, 0 + li a1, 1 +test_kprobes_c_beqz_addr1: + c.beqz a0, 2f + ret +1: li a1, KPROBE_TEST_MAGIC_UPPER + add a0, a0, a1 + ret +test_kprobes_c_beqz_addr2: +2: c.beqz a1, 3f + li a0, KPROBE_TEST_MAGIC_LOWER + mv a1, x0 +test_kprobes_c_beqz_addr3: + c.beqz a1, 1b +3: li a0, 0 + ret +SYM_FUNC_END(test_kprobes_c_beqz) + +SYM_FUNC_START(test_kprobes_c_bnez) + li a0, 0 + li a1, 1 +test_kprobes_c_bnez_addr1: + c.bnez a1, 2f + ret +1: li a1, KPROBE_TEST_MAGIC_UPPER + add a0, a0, a1 + ret +test_kprobes_c_bnez_addr2: +2: c.bnez a0, 3f + li a0, KPROBE_TEST_MAGIC_LOWER +test_kprobes_c_bnez_addr3: + c.bnez a0, 1b +3: li a0, 0 + ret +SYM_FUNC_END(test_kprobes_c_bnez) + +#endif /* CONFIG_RISCV_ISA_C */ + +.section .rodata +SYM_DATA_START(test_kprobes_addresses) + RISCV_PTR test_kprobes_add_addr1 + RISCV_PTR test_kprobes_add_addr2 + RISCV_PTR test_kprobes_jal_addr1 + RISCV_PTR test_kprobes_jal_addr2 + RISCV_PTR test_kprobes_jalr_addr + RISCV_PTR test_kprobes_auipc_addr + RISCV_PTR test_kprobes_branch_addr1 + RISCV_PTR test_kprobes_branch_addr2 + RISCV_PTR test_kprobes_branch_addr3 + RISCV_PTR test_kprobes_branch_addr4 + RISCV_PTR test_kprobes_branch_addr5 + RISCV_PTR test_kprobes_branch_addr6 + RISCV_PTR test_kprobes_branch_addr7 + RISCV_PTR test_kprobes_branch_addr8 +#ifdef CONFIG_RISCV_ISA_C + RISCV_PTR test_kprobes_branch_c_j_addr1 + RISCV_PTR test_kprobes_branch_c_j_addr2 + RISCV_PTR test_kprobes_c_jr_addr1 + RISCV_PTR test_kprobes_c_jr_addr2 + RISCV_PTR test_kprobes_c_jalr_addr + RISCV_PTR test_kprobes_c_beqz_addr1 + RISCV_PTR test_kprobes_c_beqz_addr2 + RISCV_PTR test_kprobes_c_beqz_addr3 + RISCV_PTR test_kprobes_c_bnez_addr1 + RISCV_PTR test_kprobes_c_bnez_addr2 + RISCV_PTR test_kprobes_c_bnez_addr3 +#endif /* CONFIG_RISCV_ISA_C */ + RISCV_PTR 0 +SYM_DATA_END(test_kprobes_addresses) + +.section .rodata +SYM_DATA_START(test_kprobes_functions) + RISCV_PTR test_kprobes_add + RISCV_PTR test_kprobes_jal + RISCV_PTR test_kprobes_jalr + RISCV_PTR test_kprobes_auipc + RISCV_PTR test_kprobes_branch +#ifdef CONFIG_RISCV_ISA_C + RISCV_PTR test_kprobes_c_j + RISCV_PTR test_kprobes_c_jr + RISCV_PTR test_kprobes_c_jalr + RISCV_PTR test_kprobes_c_beqz + RISCV_PTR test_kprobes_c_bnez +#endif /* CONFIG_RISCV_ISA_C */ + RISCV_PTR 0 +SYM_DATA_END(test_kprobes_functions) diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes.c b/arch/riscv/kernel/tests/kprobes/test-kprobes.c new file mode 100644 index 000000000000..027424a3ff7b --- /dev/null +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/kernel.h> +#include <linux/kprobes.h> +#include <kunit/test.h> +#include "test-kprobes.h" + +static int kprobe_dummy_handler(struct kprobe *kp, struct pt_regs *regs) +{ + return 0; +} + +static void test_kprobe_riscv(struct kunit *test) +{ + unsigned int num_kprobe = 0; + long (*func)(void); + struct kprobe *kp; + int i; + + while (test_kprobes_addresses[num_kprobe]) + num_kprobe++; + + kp = kzalloc_objs(*kp, num_kprobe); + KUNIT_EXPECT_TRUE(test, kp); + if (!kp) + return; + + for (i = 0; i < num_kprobe; ++i) { + kp[i].addr = test_kprobes_addresses[i]; + kp[i].pre_handler = kprobe_dummy_handler; + KUNIT_EXPECT_EQ(test, 0, register_kprobe(&kp[i])); + } + + for (i = 0;; ++i) { + func = test_kprobes_functions[i]; + if (!func) + break; + KUNIT_EXPECT_EQ_MSG(test, KPROBE_TEST_MAGIC, func(), "function %d broken", i); + } + + for (i = 0; i < num_kprobe; ++i) + unregister_kprobe(&kp[i]); + kfree(kp); +} + +static struct kunit_case kprobes_testcases[] = { + KUNIT_CASE(test_kprobe_riscv), + {} +}; + +static struct kunit_suite kprobes_test_suite = { + .name = "kprobes_riscv", + .test_cases = kprobes_testcases, +}; + +kunit_test_suites(&kprobes_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit test for riscv kprobes"); diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes.h b/arch/riscv/kernel/tests/kprobes/test-kprobes.h new file mode 100644 index 000000000000..537f44aa9d3f --- /dev/null +++ b/arch/riscv/kernel/tests/kprobes/test-kprobes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef TEST_KPROBES_H +#define TEST_KPROBES_H + +/* + * The magic value that all the functions in the test_kprobes_functions array return. The test + * installs kprobes into these functions, and verify that the functions still correctly return this + * value. + */ +#define KPROBE_TEST_MAGIC 0xcafebabe +#define KPROBE_TEST_MAGIC_LOWER 0x0000babe +#define KPROBE_TEST_MAGIC_UPPER 0xcafe0000 + +#ifndef __ASSEMBLER__ + +/* array of addresses to install kprobes */ +extern void *test_kprobes_addresses[]; + +/* array of functions that return KPROBE_TEST_MAGIC */ +extern long (*test_kprobes_functions[])(void); + +#endif /* __ASSEMBLER__ */ + +#endif /* TEST_KPROBES_H */ diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 8ff8e8b36524..8c62c771a656 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -6,6 +6,7 @@ #include <linux/cpu.h> #include <linux/kernel.h> #include <linux/init.h> +#include <linux/irqflags.h> #include <linux/randomize_kstack.h> #include <linux/sched.h> #include <linux/sched/debug.h> @@ -141,17 +142,15 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code, } } -#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) -#define __trap_section __noinstr_section(".xip.traps") -#else #define __trap_section noinstr -#endif #define DO_ERROR_INFO(name, signo, code, str) \ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ { \ if (user_mode(regs)) { \ irqentry_enter_from_user_mode(regs); \ + local_irq_enable(); \ do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ + local_irq_disable(); \ irqentry_exit_to_user_mode(regs); \ } else { \ irqentry_state_t state = irqentry_nmi_enter(regs); \ @@ -162,6 +161,8 @@ asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ DO_ERROR_INFO(do_trap_unknown, SIGILL, ILL_ILLTRP, "unknown exception"); +DO_ERROR_INFO(do_trap_hardware_error, + SIGBUS, BUS_MCEERR_AR, "hardware error"); DO_ERROR_INFO(do_trap_insn_misaligned, SIGBUS, BUS_ADRALN, "instruction address misaligned"); DO_ERROR_INFO(do_trap_insn_fault, @@ -173,17 +174,14 @@ asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *re if (user_mode(regs)) { irqentry_enter_from_user_mode(regs); - local_irq_enable(); handled = riscv_v_first_use_handler(regs); - - local_irq_disable(); - if (!handled) do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, "Oops - illegal instruction"); + local_irq_disable(); irqentry_exit_to_user_mode(regs); } else { irqentry_state_t state = irqentry_nmi_enter(regs); @@ -198,47 +196,57 @@ asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *re DO_ERROR_INFO(do_trap_load_fault, SIGSEGV, SEGV_ACCERR, "load access fault"); -asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) +enum misaligned_access_type { + MISALIGNED_STORE, + MISALIGNED_LOAD, +}; +static const struct { + const char *type_str; + int (*handler)(struct pt_regs *regs); +} misaligned_handler[] = { + [MISALIGNED_STORE] = { + .type_str = "Oops - store (or AMO) address misaligned", + .handler = handle_misaligned_store, + }, + [MISALIGNED_LOAD] = { + .type_str = "Oops - load address misaligned", + .handler = handle_misaligned_load, + }, +}; + +static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type) { + irqentry_state_t state; + if (user_mode(regs)) { irqentry_enter_from_user_mode(regs); + local_irq_enable(); + } else { + state = irqentry_nmi_enter(regs); + } - if (handle_misaligned_load(regs)) - do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, - "Oops - load address misaligned"); + if (misaligned_handler[type].handler(regs)) + do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, + misaligned_handler[type].type_str); + if (user_mode(regs)) { + local_irq_disable(); irqentry_exit_to_user_mode(regs); } else { - irqentry_state_t state = irqentry_nmi_enter(regs); - - if (handle_misaligned_load(regs)) - do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, - "Oops - load address misaligned"); - irqentry_nmi_exit(regs, state); } } -asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) +asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) { - if (user_mode(regs)) { - irqentry_enter_from_user_mode(regs); - - if (handle_misaligned_store(regs)) - do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, - "Oops - store (or AMO) address misaligned"); - - irqentry_exit_to_user_mode(regs); - } else { - irqentry_state_t state = irqentry_nmi_enter(regs); - - if (handle_misaligned_store(regs)) - do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, - "Oops - store (or AMO) address misaligned"); + do_trap_misaligned(regs, MISALIGNED_LOAD); +} - irqentry_nmi_exit(regs, state); - } +asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) +{ + do_trap_misaligned(regs, MISALIGNED_STORE); } + DO_ERROR_INFO(do_trap_store_fault, SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); DO_ERROR_INFO(do_trap_ecall_s, @@ -298,9 +306,11 @@ asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) { if (user_mode(regs)) { irqentry_enter_from_user_mode(regs); + local_irq_enable(); handle_break(regs); + local_irq_disable(); irqentry_exit_to_user_mode(regs); } else { irqentry_state_t state = irqentry_nmi_enter(regs); @@ -327,20 +337,10 @@ void do_trap_ecall_u(struct pt_regs *regs) add_random_kstack_offset(); - if (syscall >= 0 && syscall < NR_syscalls) + if (syscall >= 0 && syscall < NR_syscalls) { + syscall = array_index_nospec(syscall, NR_syscalls); syscall_handler(regs, syscall); - - /* - * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), - * so the maximum stack offset is 1k bytes (10 bits). - * - * The actual entropy will be further reduced by the compiler when - * applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned - * for RV32I or RV64I. - * - * The resulting 6 bits of entropy is seen in SP[9:4]. - */ - choose_random_kstack_offset(get_random_u16()); + } syscall_exit_to_user_mode(regs); } else { @@ -354,6 +354,60 @@ void do_trap_ecall_u(struct pt_regs *regs) } +#define CFI_TVAL_FCFI_CODE 2 +#define CFI_TVAL_BCFI_CODE 3 +/* handle cfi violations */ +bool handle_user_cfi_violation(struct pt_regs *regs) +{ + unsigned long tval = csr_read(CSR_TVAL); + bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr()); + bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack()); + + /* + * Handle uprobe event first. The probe point can be a valid target + * of indirect jumps or calls, in this case, forward cfi violation + * will be triggered instead of breakpoint exception. Clear ELP flag + * on sstatus image as well to avoid recurring fault. + */ + if (is_fcfi && probe_breakpoint_handler(regs)) { + regs->status &= ~SR_ELP; + return true; + } + + if (is_fcfi || is_bcfi) { + do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc, + "Oops - control flow violation"); + return true; + } + + return false; +} + +/* + * software check exception is defined with risc-v cfi spec. Software check + * exception is raised when: + * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad` + * instruction or `label` value programmed in `lpad` instr doesn't + * match with value setup in `x7`. reported code in `xtval` is 2. + * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp) + * and x1/x5. reported code in `xtval` is 3. + */ +asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs) +{ + if (user_mode(regs)) { + irqentry_enter_from_user_mode(regs); + + /* not a cfi violation, then merge into flow of unknown trap handler */ + if (!handle_user_cfi_violation(regs)) + do_trap_unknown(regs); + + irqentry_exit_to_user_mode(regs); + } else { + /* sw check exception coming from kernel is a bug in kernel */ + die(regs, "Kernel BUG"); + } +} + #ifdef CONFIG_MMU asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) { diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c index 7cc108aed74e..81b7682e6c6d 100644 --- a/arch/riscv/kernel/traps_misaligned.c +++ b/arch/riscv/kernel/traps_misaligned.c @@ -16,143 +16,9 @@ #include <asm/entry-common.h> #include <asm/hwprobe.h> #include <asm/cpufeature.h> +#include <asm/sbi.h> #include <asm/vector.h> - -#define INSN_MATCH_LB 0x3 -#define INSN_MASK_LB 0x707f -#define INSN_MATCH_LH 0x1003 -#define INSN_MASK_LH 0x707f -#define INSN_MATCH_LW 0x2003 -#define INSN_MASK_LW 0x707f -#define INSN_MATCH_LD 0x3003 -#define INSN_MASK_LD 0x707f -#define INSN_MATCH_LBU 0x4003 -#define INSN_MASK_LBU 0x707f -#define INSN_MATCH_LHU 0x5003 -#define INSN_MASK_LHU 0x707f -#define INSN_MATCH_LWU 0x6003 -#define INSN_MASK_LWU 0x707f -#define INSN_MATCH_SB 0x23 -#define INSN_MASK_SB 0x707f -#define INSN_MATCH_SH 0x1023 -#define INSN_MASK_SH 0x707f -#define INSN_MATCH_SW 0x2023 -#define INSN_MASK_SW 0x707f -#define INSN_MATCH_SD 0x3023 -#define INSN_MASK_SD 0x707f - -#define INSN_MATCH_FLW 0x2007 -#define INSN_MASK_FLW 0x707f -#define INSN_MATCH_FLD 0x3007 -#define INSN_MASK_FLD 0x707f -#define INSN_MATCH_FLQ 0x4007 -#define INSN_MASK_FLQ 0x707f -#define INSN_MATCH_FSW 0x2027 -#define INSN_MASK_FSW 0x707f -#define INSN_MATCH_FSD 0x3027 -#define INSN_MASK_FSD 0x707f -#define INSN_MATCH_FSQ 0x4027 -#define INSN_MASK_FSQ 0x707f - -#define INSN_MATCH_C_LD 0x6000 -#define INSN_MASK_C_LD 0xe003 -#define INSN_MATCH_C_SD 0xe000 -#define INSN_MASK_C_SD 0xe003 -#define INSN_MATCH_C_LW 0x4000 -#define INSN_MASK_C_LW 0xe003 -#define INSN_MATCH_C_SW 0xc000 -#define INSN_MASK_C_SW 0xe003 -#define INSN_MATCH_C_LDSP 0x6002 -#define INSN_MASK_C_LDSP 0xe003 -#define INSN_MATCH_C_SDSP 0xe002 -#define INSN_MASK_C_SDSP 0xe003 -#define INSN_MATCH_C_LWSP 0x4002 -#define INSN_MASK_C_LWSP 0xe003 -#define INSN_MATCH_C_SWSP 0xc002 -#define INSN_MASK_C_SWSP 0xe003 - -#define INSN_MATCH_C_FLD 0x2000 -#define INSN_MASK_C_FLD 0xe003 -#define INSN_MATCH_C_FLW 0x6000 -#define INSN_MASK_C_FLW 0xe003 -#define INSN_MATCH_C_FSD 0xa000 -#define INSN_MASK_C_FSD 0xe003 -#define INSN_MATCH_C_FSW 0xe000 -#define INSN_MASK_C_FSW 0xe003 -#define INSN_MATCH_C_FLDSP 0x2002 -#define INSN_MASK_C_FLDSP 0xe003 -#define INSN_MATCH_C_FSDSP 0xa002 -#define INSN_MASK_C_FSDSP 0xe003 -#define INSN_MATCH_C_FLWSP 0x6002 -#define INSN_MASK_C_FLWSP 0xe003 -#define INSN_MATCH_C_FSWSP 0xe002 -#define INSN_MASK_C_FSWSP 0xe003 - -#define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) - -#if defined(CONFIG_64BIT) -#define LOG_REGBYTES 3 -#define XLEN 64 -#else -#define LOG_REGBYTES 2 -#define XLEN 32 -#endif -#define REGBYTES (1 << LOG_REGBYTES) -#define XLEN_MINUS_16 ((XLEN) - 16) - -#define SH_RD 7 -#define SH_RS1 15 -#define SH_RS2 20 -#define SH_RS2C 2 - -#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) -#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ - (RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 1) << 6)) -#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 2) << 6)) -#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 2) << 6)) -#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 3) << 6)) -#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ - (RV_X(x, 7, 2) << 6)) -#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 7, 3) << 6)) -#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) -#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) -#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) - -#define SHIFT_RIGHT(x, y) \ - ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) - -#define REG_MASK \ - ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) - -#define REG_OFFSET(insn, pos) \ - (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) - -#define REG_PTR(insn, pos, regs) \ - (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) - -#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) -#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) -#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) -#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) -#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) -#define GET_SP(regs) (*REG_PTR(2, 0, regs)) -#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) -#define IMM_I(insn) ((s32)(insn) >> 20) -#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ - (s32)(((insn) >> 7) & 0x1f)) -#define MASK_FUNCT3 0x7000 - -#define GET_PRECISION(insn) (((insn) >> 25) & 3) -#define GET_RM(insn) (((insn) >> 12) & 7) -#define PRECISION_S 0 -#define PRECISION_D 1 +#include <asm/insn.h> #ifdef CONFIG_FPU @@ -268,7 +134,7 @@ static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, int __ret; \ \ if (user_mode(regs)) { \ - __ret = __get_user(insn, (type __user *) insn_addr); \ + __ret = get_user(insn, (type __user *) insn_addr); \ } else { \ insn = *(type *)insn_addr; \ __ret = 0; \ @@ -361,9 +227,7 @@ static int handle_scalar_misaligned_load(struct pt_regs *regs) perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; -#endif if (!unaligned_enabled) return -1; @@ -431,6 +295,13 @@ static int handle_scalar_misaligned_load(struct pt_regs *regs) fp = 1; len = 4; #endif + } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { + len = 2; + insn = RVC_RS2S(insn) << SH_RD; + } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { + len = 2; + shift = 8 * (sizeof(ulong) - len); + insn = RVC_RS2S(insn) << SH_RD; } else { regs->epc = epc; return -1; @@ -448,7 +319,7 @@ static int handle_scalar_misaligned_load(struct pt_regs *regs) } if (!fp) - SET_RD(insn, regs, val.data_ulong << shift >> shift); + SET_RD(insn, regs, (long)(val.data_ulong << shift) >> shift); else if (len == 8) set_f64_rd(insn, regs, val.data_u64); else @@ -530,6 +401,9 @@ static int handle_scalar_misaligned_store(struct pt_regs *regs) len = 4; val.data_ulong = GET_F32_RS2C(insn, regs); #endif + } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { + len = 2; + val.data_ulong = GET_RS2S(insn, regs); } else { regs->epc = epc; return -1; @@ -605,16 +479,14 @@ void check_vector_unaligned_access_emulated(struct work_struct *work __always_un kernel_vector_end(); } -bool check_vector_unaligned_access_emulated_all_cpus(void) +bool __init check_vector_unaligned_access_emulated_all_cpus(void) { int cpu; - if (!has_vector()) { - for_each_online_cpu(cpu) - per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; - return false; - } - + /* + * While being documented as very slow, schedule_on_each_cpu() is used since + * kernel_vector_begin() expects irqs to be enabled or it will panic() + */ schedule_on_each_cpu(check_vector_unaligned_access_emulated); for_each_online_cpu(cpu) @@ -625,17 +497,29 @@ bool check_vector_unaligned_access_emulated_all_cpus(void) return true; } #else -bool check_vector_unaligned_access_emulated_all_cpus(void) +bool __init check_vector_unaligned_access_emulated_all_cpus(void) { return false; } #endif +static bool all_cpus_unaligned_scalar_access_emulated(void) +{ + int cpu; + + for_each_online_cpu(cpu) + if (per_cpu(misaligned_access_speed, cpu) != + RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) + return false; + + return true; +} + #ifdef CONFIG_RISCV_SCALAR_MISALIGNED static bool unaligned_ctl __read_mostly; -void check_unaligned_access_emulated(struct work_struct *work __always_unused) +static void check_unaligned_access_emulated(void *arg __always_unused) { int cpu = smp_processor_id(); long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); @@ -646,6 +530,13 @@ void check_unaligned_access_emulated(struct work_struct *work __always_unused) __asm__ __volatile__ ( " "REG_L" %[tmp], 1(%[ptr])\n" : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); +} + +static int cpu_online_check_unaligned_access_emulated(unsigned int cpu) +{ + long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); + + check_unaligned_access_emulated(NULL); /* * If unaligned_ctl is already set, this means that we detected that all @@ -654,26 +545,23 @@ void check_unaligned_access_emulated(struct work_struct *work __always_unused) */ if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) { pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); - while (true) - cpu_relax(); + return -EINVAL; } + + return 0; } -bool check_unaligned_access_emulated_all_cpus(void) +bool __init check_unaligned_access_emulated_all_cpus(void) { - int cpu; - /* * We can only support PR_UNALIGN controls if all CPUs have misaligned * accesses emulated since tasks requesting such control can run on any * CPU. */ - schedule_on_each_cpu(check_unaligned_access_emulated); + on_each_cpu(check_unaligned_access_emulated, NULL, 1); - for_each_online_cpu(cpu) - if (per_cpu(misaligned_access_speed, cpu) - != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) - return false; + if (!all_cpus_unaligned_scalar_access_emulated()) + return false; unaligned_ctl = true; return true; @@ -684,8 +572,77 @@ bool unaligned_ctl_available(void) return unaligned_ctl; } #else -bool check_unaligned_access_emulated_all_cpus(void) +bool __init check_unaligned_access_emulated_all_cpus(void) { return false; } +static int cpu_online_check_unaligned_access_emulated(unsigned int cpu) +{ + return 0; +} +#endif + +static bool misaligned_traps_delegated; + +#if defined(CONFIG_RISCV_SBI_FWFT_DELEGATE_MISALIGNED) + +static int cpu_online_sbi_unaligned_setup(unsigned int cpu) +{ + if (sbi_fwft_set(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0) && + misaligned_traps_delegated) { + pr_crit("Misaligned trap delegation non homogeneous (expected delegated)"); + return -EINVAL; + } + + return 0; +} + +void __init unaligned_access_init(void) +{ + int ret; + + ret = sbi_fwft_set_online_cpus(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0); + if (ret) + return; + + misaligned_traps_delegated = true; + pr_info("SBI misaligned access exception delegation ok\n"); + /* + * Note that we don't have to take any specific action here, if + * the delegation is successful, then + * check_unaligned_access_emulated() will verify that indeed the + * platform traps on misaligned accesses. + */ +} +#else +void __init unaligned_access_init(void) {} + +static int cpu_online_sbi_unaligned_setup(unsigned int cpu __always_unused) +{ + return 0; +} + #endif + +int cpu_online_unaligned_access_init(unsigned int cpu) +{ + int ret; + + ret = cpu_online_sbi_unaligned_setup(cpu); + if (ret) + return ret; + + return cpu_online_check_unaligned_access_emulated(cpu); +} + +bool misaligned_traps_can_delegate(void) +{ + /* + * Either we successfully requested misaligned traps delegation for all + * CPUs, or the SBI does not implement the FWFT extension but delegated + * the exception by default. + */ + return misaligned_traps_delegated || + all_cpus_unaligned_scalar_access_emulated(); +} +EXPORT_SYMBOL_GPL(misaligned_traps_can_delegate); diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 91f189cf1611..485ab1d105d3 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -16,7 +16,7 @@ #include "copy-unaligned.h" -#define MISALIGNED_ACCESS_JIFFIES_LG2 1 +#define MISALIGNED_ACCESS_NS 8000000 #define MISALIGNED_BUFFER_SIZE 0x4000 #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) @@ -24,111 +24,171 @@ DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; -#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS +static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; +static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; + static cpumask_t fast_misaligned_access; -static int check_unaligned_access(void *param) -{ - int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; - struct page *page = param; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; - if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) - return 0; +static u64 __maybe_unused +measure_cycles(void (*func)(void *dst, const void *src, size_t len), + void *dst, void *src, size_t len) +{ + u64 start_cycles, end_cycles, cycles = -1ULL; + u64 start_ns; - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; /* Do a warmup. */ - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); + preempt_disable(); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); /* * For a fixed amount of time, repeatedly try the function, and take * the best time in cycles as the measurement. */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { + start_ns = ktime_get_mono_fast_ns(); + while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) { start_cycles = get_cycles64(); /* Ensure the CSR read can't reorder WRT to the copy. */ mb(); - __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); + func(dst, src, len); /* Ensure the copy ends before the end time is snapped. */ mb(); end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; + if ((end_cycles - start_cycles) < cycles) + cycles = end_cycles - start_cycles; } - byte_cycles = -1ULL; - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); + preempt_enable(); - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - mb(); - __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + return cycles; +} - preempt_enable(); +/* + * Return: + * 1 if unaligned accesses are fast + * 0 if unaligned accesses are slow + * -1 if check cannot be done + */ +static int __maybe_unused +compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len), + void (*byte_copy)(void *dst, const void *src, size_t len), + void *buf, const char *type) +{ + int cpu = smp_processor_id(); + u64 word_cycles; + u64 byte_cycles; + void *dst, *src; + bool fast; + int ratio; + + /* Make an unaligned destination buffer. */ + dst = (void *)((unsigned long)buf | 0x1); + /* Unalign src as well, but differently (off by 1 + 2 = 3). */ + src = dst + (MISALIGNED_BUFFER_SIZE / 2); + src += 2; + + word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE); + byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE); /* Don't divide by zero. */ if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", - cpu); + pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n", + cpu, type); - return 0; + return -1; } - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; + fast = word_cycles < byte_cycles; ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", + pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n", cpu, + type, ratio / 100, ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) ? "fast" : "slow"); + fast ? "fast" : "slow"); - per_cpu(misaligned_access_speed, cpu) = speed; + return fast; +} + +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS +static int check_unaligned_access(struct page *page) +{ + void *buf = page_address(page); + int cpu = smp_processor_id(); + int ret; + + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + return 0; + + ret = compare_unaligned_access(__riscv_copy_words_unaligned, + __riscv_copy_bytes_unaligned, + buf, "scalar"); + if (ret < 0) + return 0; /* * Set the value of fast_misaligned_access of a CPU. These operations * are atomic to avoid race conditions. */ - if (speed == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) + if (ret) { + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; cpumask_set_cpu(cpu, &fast_misaligned_access); - else + } else { + per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; cpumask_clear_cpu(cpu, &fast_misaligned_access); + } return 0; } -static void check_unaligned_access_nonboot_cpu(void *param) +static void __init _check_unaligned_access(void *param) { unsigned int cpu = smp_processor_id(); struct page **pages = param; - if (smp_processor_id() != 0) - check_unaligned_access(pages[cpu]); + check_unaligned_access(pages[cpu]); +} + +/* Measure unaligned access speed on all CPUs present at boot in parallel. */ +static void __init check_unaligned_access_speed_all_cpus(void) +{ + unsigned int cpu; + unsigned int cpu_count = num_possible_cpus(); + struct page **bufs = kzalloc_objs(*bufs, cpu_count); + + if (!bufs) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + return; + } + + /* + * Allocate separate buffers for each CPU so there's no fighting over + * cache lines. + */ + for_each_cpu(cpu, cpu_online_mask) { + bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); + if (!bufs[cpu]) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + goto out; + } + } + + on_each_cpu(_check_unaligned_access, bufs, 1); + +out: + for_each_cpu(cpu, cpu_online_mask) { + if (bufs[cpu]) + __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); + } + + kfree(bufs); +} +#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ +static void __init check_unaligned_access_speed_all_cpus(void) +{ } +#endif DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); @@ -175,7 +235,7 @@ static void set_unaligned_access_static_branches(void) modify_unaligned_access_branches(&fast_and_online, num_online_cpus()); } -static int lock_and_set_unaligned_access_static_branch(void) +static int __init lock_and_set_unaligned_access_static_branch(void) { cpus_read_lock(); set_unaligned_access_static_branches(); @@ -188,21 +248,33 @@ arch_initcall_sync(lock_and_set_unaligned_access_static_branch); static int riscv_online_cpu(unsigned int cpu) { - static struct page *buf; + int ret = cpu_online_unaligned_access_init(cpu); + + if (ret) + return ret; /* We are already set since the last check */ - if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) + if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { + goto exit; + } else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { + per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; goto exit; - - check_unaligned_access_emulated(NULL); - buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); - if (!buf) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - return -ENOMEM; } - check_unaligned_access(buf); - __free_pages(buf, MISALIGNED_BUFFER_ORDER); +#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS + { + static struct page *buf; + + buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); + if (!buf) { + pr_warn("Allocation failure, not measuring misaligned performance\n"); + return -ENOMEM; + } + + check_unaligned_access(buf); + __free_pages(buf, MISALIGNED_BUFFER_ORDER); + } +#endif exit: set_unaligned_access_static_branches(); @@ -217,72 +289,12 @@ static int riscv_offline_cpu(unsigned int cpu) return 0; } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int check_unaligned_access_speed_all_cpus(void) -{ - unsigned int cpu; - unsigned int cpu_count = num_possible_cpus(); - struct page **bufs = kcalloc(cpu_count, sizeof(*bufs), GFP_KERNEL); - - if (!bufs) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - return 0; - } - - /* - * Allocate separate buffers for each CPU so there's no fighting over - * cache lines. - */ - for_each_cpu(cpu, cpu_online_mask) { - bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); - if (!bufs[cpu]) { - pr_warn("Allocation failure, not measuring misaligned performance\n"); - goto out; - } - } - - /* Check everybody except 0, who stays behind to tend jiffies. */ - on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); - - /* Check core 0. */ - smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); - - /* - * Setup hotplug callbacks for any new CPUs that come online or go - * offline. - */ - cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", - riscv_online_cpu, riscv_offline_cpu); - -out: - for_each_cpu(cpu, cpu_online_mask) { - if (bufs[cpu]) - __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); - } - - kfree(bufs); - return 0; -} -#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ -static int check_unaligned_access_speed_all_cpus(void) -{ - return 0; -} -#endif - #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS static void check_vector_unaligned_access(struct work_struct *work __always_unused) { int cpu = smp_processor_id(); - u64 start_cycles, end_cycles; - u64 word_cycles; - u64 byte_cycles; - int ratio; - unsigned long start_jiffies, now; struct page *page; - void *dst; - void *src; - long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; + int ret; if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return; @@ -293,129 +305,142 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus return; } - /* Make an unaligned destination buffer. */ - dst = (void *)((unsigned long)page_address(page) | 0x1); - /* Unalign src as well, but differently (off by 1 + 2 = 3). */ - src = dst + (MISALIGNED_BUFFER_SIZE / 2); - src += 2; - word_cycles = -1ULL; - - /* Do a warmup. */ kernel_vector_begin(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - /* - * For a fixed amount of time, repeatedly try the function, and take - * the best time in cycles as the measurement. - */ - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < word_cycles) - word_cycles = end_cycles - start_cycles; - } - - byte_cycles = -1ULL; - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - start_jiffies = jiffies; - while ((now = jiffies) == start_jiffies) - cpu_relax(); - - while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { - start_cycles = get_cycles64(); - /* Ensure the CSR read can't reorder WRT to the copy. */ - mb(); - __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); - /* Ensure the copy ends before the end time is snapped. */ - mb(); - end_cycles = get_cycles64(); - if ((end_cycles - start_cycles) < byte_cycles) - byte_cycles = end_cycles - start_cycles; - } + ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned, + __riscv_copy_vec_bytes_unaligned, + page_address(page), "vector"); kernel_vector_end(); - /* Don't divide by zero. */ - if (!word_cycles || !byte_cycles) { - pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n", - cpu); + if (ret < 0) + goto free; - return; - } + if (ret) + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; + else + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; - if (word_cycles < byte_cycles) - speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; +free: + __free_pages(page, MISALIGNED_BUFFER_ORDER); +} - ratio = div_u64((byte_cycles * 100), word_cycles); - pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n", - cpu, - ratio / 100, - ratio % 100, - (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow"); +/* Measure unaligned access speed on all CPUs present at boot in parallel. */ +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +{ + schedule_on_each_cpu(check_vector_unaligned_access); + riscv_hwprobe_complete_async_probe(); - per_cpu(vector_misaligned_access, cpu) = speed; + return 0; } +#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ +static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +{ + return 0; +} +#endif static int riscv_online_cpu_vec(unsigned int cpu) { - if (!has_vector()) + if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { + per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; return 0; + } - if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED) +#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS + if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) return 0; check_vector_unaligned_access_emulated(NULL); check_vector_unaligned_access(NULL); +#endif + return 0; } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); +static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" }; - /* - * Setup hotplug callbacks for any new CPUs that come online or go - * offline. - */ - cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", - riscv_online_cpu_vec, NULL); +static int __init set_unaligned_scalar_speed_param(char *str) +{ + if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED])) + unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED; + else + return -EINVAL; - return 0; + return 1; } -#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +__setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param); + +static int __init set_unaligned_vector_speed_param(char *str) { - return 0; + if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; + else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED])) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + else + return -EINVAL; + + return 1; } -#endif +__setup("unaligned_vector_speed=", set_unaligned_vector_speed_param); -static int check_unaligned_access_all_cpus(void) +static int __init check_unaligned_access_all_cpus(void) { - bool all_cpus_emulated, all_cpus_vec_unsupported; + int cpu; - all_cpus_emulated = check_unaligned_access_emulated_all_cpus(); - all_cpus_vec_unsupported = check_vector_unaligned_access_emulated_all_cpus(); + unaligned_access_init(); - if (!all_cpus_vec_unsupported && - IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"); + if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { + pr_info("scalar unaligned access speed set to '%s' (%lu) by command line\n", + speed_str[unaligned_scalar_speed_param], unaligned_scalar_speed_param); + for_each_online_cpu(cpu) + per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; + } else if (!check_unaligned_access_emulated_all_cpus()) { + check_unaligned_access_speed_all_cpus(); } - if (!all_cpus_emulated) - return check_unaligned_access_speed_all_cpus(); + if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { + if (!has_vector() && + unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED) { + pr_warn("vector support is not available, ignoring unaligned_vector_speed=%s\n", + speed_str[unaligned_vector_speed_param]); + } else { + pr_info("vector unaligned access speed set to '%s' (%lu) by command line\n", + speed_str[unaligned_vector_speed_param], unaligned_vector_speed_param); + } + } + + if (!has_vector()) + unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; + + if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { + for_each_online_cpu(cpu) + per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; + } else if (!check_vector_unaligned_access_emulated_all_cpus() && + IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { + riscv_hwprobe_register_async_probe(); + if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, + NULL, "vec_check_unaligned_access_speed_all_cpus"))) { + pr_warn("Failed to create vec_unalign_check kthread\n"); + riscv_hwprobe_complete_async_probe(); + } + } + + /* + * Setup hotplug callbacks for any new CPUs that come online or go + * offline. + */ + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", + riscv_online_cpu, riscv_offline_cpu); + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", + riscv_online_cpu_vec, NULL); return 0; } -arch_initcall(check_unaligned_access_all_cpus); +late_initcall(check_unaligned_access_all_cpus); diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c new file mode 100644 index 000000000000..cbfb4e495e9f --- /dev/null +++ b/arch/riscv/kernel/usercfi.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Rivos, Inc. + * Deepak Gupta <debug@rivosinc.com> + */ + +#include <linux/sched.h> +#include <linux/bitops.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/uaccess.h> +#include <linux/sizes.h> +#include <linux/user.h> +#include <linux/syscalls.h> +#include <linux/prctl.h> +#include <asm/csr.h> +#include <asm/usercfi.h> + +unsigned long riscv_nousercfi __read_mostly; + +#define SHSTK_ENTRY_SIZE sizeof(void *) + +bool is_shstk_enabled(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ubcfi_en; +} + +bool is_shstk_allocated(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.shdw_stk_base; +} + +bool is_shstk_locked(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ubcfi_locked; +} + +void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size) +{ + task->thread_info.user_cfi_state.shdw_stk_base = shstk_addr; + task->thread_info.user_cfi_state.shdw_stk_size = size; +} + +unsigned long get_shstk_base(struct task_struct *task, unsigned long *size) +{ + if (size) + *size = task->thread_info.user_cfi_state.shdw_stk_size; + return task->thread_info.user_cfi_state.shdw_stk_base; +} + +void set_active_shstk(struct task_struct *task, unsigned long shstk_addr) +{ + task->thread_info.user_cfi_state.user_shdw_stk = shstk_addr; +} + +unsigned long get_active_shstk(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.user_shdw_stk; +} + +void set_shstk_status(struct task_struct *task, bool enable) +{ + if (!is_user_shstk_enabled()) + return; + + task->thread_info.user_cfi_state.ubcfi_en = enable ? 1 : 0; + + if (enable) + task->thread.envcfg |= ENVCFG_SSE; + else + task->thread.envcfg &= ~ENVCFG_SSE; + + csr_write(CSR_ENVCFG, task->thread.envcfg); +} + +void set_shstk_lock(struct task_struct *task, bool lock) +{ + task->thread_info.user_cfi_state.ubcfi_locked = lock; +} + +bool is_indir_lp_enabled(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ufcfi_en; +} + +bool is_indir_lp_locked(struct task_struct *task) +{ + return task->thread_info.user_cfi_state.ufcfi_locked; +} + +void set_indir_lp_status(struct task_struct *task, bool enable) +{ + if (!is_user_lpad_enabled()) + return; + + task->thread_info.user_cfi_state.ufcfi_en = enable ? 1 : 0; + + if (enable) + task->thread.envcfg |= ENVCFG_LPE; + else + task->thread.envcfg &= ~ENVCFG_LPE; + + csr_write(CSR_ENVCFG, task->thread.envcfg); +} + +void set_indir_lp_lock(struct task_struct *task, bool lock) +{ + task->thread_info.user_cfi_state.ufcfi_locked = lock; +} +/* + * The shadow stack only stores the return address and not any variables + * this should be more than sufficient for most applications. + * Else PAGE_ALIGN it and return back + */ +static unsigned long calc_shstk_size(unsigned long size) +{ + if (size) + return PAGE_ALIGN(size); + + return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 2, SZ_2G)); +} + +/* + * Writes on shadow stack can either be `sspush` or `ssamoswap`. `sspush` can happen + * implicitly on current shadow stack pointed to by CSR_SSP. `ssamoswap` takes pointer to + * shadow stack. To keep it simple, we plan to use `ssamoswap` to perform writes on shadow + * stack. + */ +static noinline unsigned long amo_user_shstk(unsigned long __user *addr, unsigned long val) +{ + /* + * Never expect -1 on shadow stack. Expect return addresses and zero + */ + unsigned long swap = -1; + + __enable_user_access(); + asm goto(".option push\n" + ".option arch, +zicfiss\n" + "1: ssamoswap.d %[swap], %[val], %[addr]\n" + _ASM_EXTABLE(1b, %l[fault]) + ".option pop\n" + : [swap] "=r" (swap), [addr] "+A" (*(__force unsigned long *)addr) + : [val] "r" (val) + : "memory" + : fault + ); + __disable_user_access(); + return swap; +fault: + __disable_user_access(); + return -1; +} + +/* + * Create a restore token on the shadow stack. A token is always XLEN wide + * and aligned to XLEN. + */ +static int create_rstor_token(unsigned long ssp, unsigned long *token_addr) +{ + unsigned long addr; + + /* Token must be aligned */ + if (!IS_ALIGNED(ssp, SHSTK_ENTRY_SIZE)) + return -EINVAL; + + /* On RISC-V we're constructing token to be function of address itself */ + addr = ssp - SHSTK_ENTRY_SIZE; + + if (amo_user_shstk((unsigned long __user *)addr, (unsigned long)ssp) == -1) + return -EFAULT; + + if (token_addr) + *token_addr = addr; + + return 0; +} + +/* + * Save user shadow stack pointer on the shadow stack itself and return a pointer to saved location. + * Returns -EFAULT if unsuccessful. + */ +int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr) +{ + unsigned long ss_ptr = 0; + unsigned long token_loc = 0; + int ret = 0; + + if (!saved_shstk_ptr) + return -EINVAL; + + ss_ptr = get_active_shstk(tsk); + ret = create_rstor_token(ss_ptr, &token_loc); + + if (!ret) { + *saved_shstk_ptr = token_loc; + set_active_shstk(tsk, token_loc); + } + + return ret; +} + +/* + * Restores the user shadow stack pointer from the token on the shadow stack for task 'tsk'. + * Returns -EFAULT if unsuccessful. + */ +int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr) +{ + unsigned long token = 0; + + token = amo_user_shstk((unsigned long __user *)shstk_ptr, 0); + + if (token == -1) + return -EFAULT; + + /* invalid token, return EINVAL */ + if ((token - shstk_ptr) != SHSTK_ENTRY_SIZE) { + pr_info_ratelimited("%s[%d]: bad restore token in %s: pc=%p sp=%p, token=%p, shstk_ptr=%p\n", + tsk->comm, task_pid_nr(tsk), __func__, + (void *)(task_pt_regs(tsk)->epc), + (void *)(task_pt_regs(tsk)->sp), + (void *)token, (void *)shstk_ptr); + return -EINVAL; + } + + /* all checks passed, set active shstk and return success */ + set_active_shstk(tsk, token); + return 0; +} + +static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size, + unsigned long token_offset, bool set_tok) +{ + addr = vm_mmap_shadow_stack(addr, size, 0); + + if (!set_tok || IS_ERR_VALUE(addr)) + goto out; + + if (create_rstor_token(addr + token_offset, NULL)) { + vm_munmap(addr, size); + return -EINVAL; + } + +out: + return addr; +} + +SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags) +{ + bool set_tok = flags & SHADOW_STACK_SET_TOKEN; + unsigned long aligned_size = 0; + + if (!is_user_shstk_enabled()) + return -EOPNOTSUPP; + + /* Anything other than set token should result in invalid param */ + if (flags & ~SHADOW_STACK_SET_TOKEN) + return -EINVAL; + + /* + * Unlike other architectures, on RISC-V, SSP pointer is held in CSR_SSP and is an available + * CSR in all modes. CSR accesses are performed using 12bit index programmed in instruction + * itself. This provides static property on register programming and writes to CSR can't + * be unintentional from programmer's perspective. As long as programmer has guarded areas + * which perform writes to CSR_SSP properly, shadow stack pivoting is not possible. Since + * CSR_SSP is writable by user mode, it itself can setup a shadow stack token subsequent + * to allocation. Although in order to provide portablity with other architectures (because + * `map_shadow_stack` is arch agnostic syscall), RISC-V will follow expectation of a token + * flag in flags and if provided in flags, will setup a token at the base. + */ + + /* If there isn't space for a token */ + if (set_tok && size < SHSTK_ENTRY_SIZE) + return -ENOSPC; + + if (addr && (addr & (PAGE_SIZE - 1))) + return -EINVAL; + + aligned_size = PAGE_ALIGN(size); + if (aligned_size < size) + return -EOVERFLOW; + + return allocate_shadow_stack(addr, aligned_size, size, set_tok); +} + +/* + * This gets called during clone/clone3/fork. And is needed to allocate a shadow stack for + * cases where CLONE_VM is specified and thus a different stack is specified by user. We + * thus need a separate shadow stack too. How a separate shadow stack is specified by + * user is still being debated. Once that's settled, remove this part of the comment. + * This function simply returns 0 if shadow stacks are not supported or if separate shadow + * stack allocation is not needed (like in case of !CLONE_VM) + */ +unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, + const struct kernel_clone_args *args) +{ + unsigned long addr, size; + + /* If shadow stack is not supported, return 0 */ + if (!is_user_shstk_enabled()) + return 0; + + /* + * If shadow stack is not enabled on the new thread, skip any + * switch to a new shadow stack. + */ + if (!is_shstk_enabled(tsk)) + return 0; + + /* + * For CLONE_VFORK the child will share the parents shadow stack. + * Set base = 0 and size = 0, this is special means to track this state + * so the freeing logic run for child knows to leave it alone. + */ + if (args->flags & CLONE_VFORK) { + set_shstk_base(tsk, 0, 0); + return 0; + } + + /* + * For !CLONE_VM the child will use a copy of the parents shadow + * stack. + */ + if (!(args->flags & CLONE_VM)) + return 0; + + /* + * reaching here means, CLONE_VM was specified and thus a separate shadow + * stack is needed for new cloned thread. Note: below allocation is happening + * using current mm. + */ + size = calc_shstk_size(args->stack_size); + addr = allocate_shadow_stack(0, size, 0, false); + if (IS_ERR_VALUE(addr)) + return addr; + + set_shstk_base(tsk, addr, size); + + return addr + size; +} + +void shstk_release(struct task_struct *tsk) +{ + unsigned long base = 0, size = 0; + /* If shadow stack is not supported or not enabled, nothing to release */ + if (!is_user_shstk_enabled() || !is_shstk_enabled(tsk)) + return; + + /* + * When fork() with CLONE_VM fails, the child (tsk) already has a + * shadow stack allocated, and exit_thread() calls this function to + * free it. In this case the parent (current) and the child share + * the same mm struct. Move forward only when they're same. + */ + if (!tsk->mm || tsk->mm != current->mm) + return; + + /* + * We know shadow stack is enabled but if base is NULL, then + * this task is not managing its own shadow stack (CLONE_VFORK). So + * skip freeing it. + */ + base = get_shstk_base(tsk, &size); + if (!base) + return; + + vm_munmap(base, size); + set_shstk_base(tsk, 0, 0); +} + +int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status) +{ + unsigned long bcfi_status = 0; + + if (!is_user_shstk_enabled()) + return -EINVAL; + + /* this means shadow stack is enabled on the task */ + bcfi_status |= (is_shstk_enabled(t) ? PR_SHADOW_STACK_ENABLE : 0); + + return copy_to_user(status, &bcfi_status, sizeof(bcfi_status)) ? -EFAULT : 0; +} + +int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status) +{ + unsigned long size = 0, addr = 0; + bool enable_shstk = false; + + if (!is_user_shstk_enabled()) + return -EINVAL; + + /* Reject unknown flags */ + if (status & ~PR_SHADOW_STACK_SUPPORTED_STATUS_MASK) + return -EINVAL; + + /* bcfi status is locked and further can't be modified by user */ + if (is_shstk_locked(t)) + return -EINVAL; + + enable_shstk = status & PR_SHADOW_STACK_ENABLE; + /* Request is to enable shadow stack and shadow stack is not enabled already */ + if (enable_shstk && !is_shstk_enabled(t)) { + /* shadow stack was allocated and enable request again + * no need to support such usecase and return EINVAL. + */ + if (is_shstk_allocated(t)) + return -EINVAL; + + size = calc_shstk_size(0); + addr = allocate_shadow_stack(0, size, 0, false); + if (IS_ERR_VALUE(addr)) + return -ENOMEM; + set_shstk_base(t, addr, size); + set_active_shstk(t, addr + size); + } + + /* + * If a request to disable shadow stack happens, let's go ahead and release it + * Although, if CLONE_VFORKed child did this, then in that case we will end up + * not releasing the shadow stack (because it might be needed in parent). Although + * we will disable it for VFORKed child. And if VFORKed child tries to enable again + * then in that case, it'll get entirely new shadow stack because following condition + * are true + * - shadow stack was not enabled for vforked child + * - shadow stack base was anyways pointing to 0 + * This shouldn't be a big issue because we want parent to have availability of shadow + * stack whenever VFORKed child releases resources via exit or exec but at the same + * time we want VFORKed child to break away and establish new shadow stack if it desires + * + */ + if (!enable_shstk) + shstk_release(t); + + set_shstk_status(t, enable_shstk); + return 0; +} + +int arch_lock_shadow_stack_status(struct task_struct *task, + unsigned long arg) +{ + /* If shtstk not supported or not enabled on task, nothing to lock here */ + if (!is_user_shstk_enabled() || + !is_shstk_enabled(task) || arg != 0) + return -EINVAL; + + set_shstk_lock(task, true); + + return 0; +} + +int arch_prctl_get_branch_landing_pad_state(struct task_struct *t, + unsigned long __user *state) +{ + unsigned long fcfi_status = 0; + + if (!is_user_lpad_enabled()) + return -EINVAL; + + fcfi_status = (is_indir_lp_enabled(t) ? PR_CFI_ENABLE : PR_CFI_DISABLE); + fcfi_status |= (is_indir_lp_locked(t) ? PR_CFI_LOCK : 0); + + return copy_to_user(state, &fcfi_status, sizeof(fcfi_status)) ? -EFAULT : 0; +} + +int arch_prctl_set_branch_landing_pad_state(struct task_struct *t, unsigned long state) +{ + if (!is_user_lpad_enabled()) + return -EINVAL; + + /* indirect branch tracking is locked and further can't be modified by user */ + if (is_indir_lp_locked(t)) + return -EINVAL; + + if (!(state & (PR_CFI_ENABLE | PR_CFI_DISABLE))) + return -EINVAL; + + if (state & PR_CFI_ENABLE && state & PR_CFI_DISABLE) + return -EINVAL; + + set_indir_lp_status(t, !!(state & PR_CFI_ENABLE)); + + return 0; +} + +int arch_prctl_lock_branch_landing_pad_state(struct task_struct *task) +{ + /* + * If indirect branch tracking is not supported or not enabled on task, + * nothing to lock here + */ + if (!is_user_lpad_enabled() || + !is_indir_lp_enabled(task)) + return -EINVAL; + + set_indir_lp_lock(task, true); + + return 0; +} + +bool is_user_shstk_enabled(void) +{ + return (cpu_supports_shadow_stack() && + !(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI)); +} + +bool is_user_lpad_enabled(void) +{ + return (cpu_supports_indirect_br_lp_instr() && + !(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI)); +} + +static int __init setup_global_riscv_enable(char *str) +{ + if (strcmp(str, "all") == 0) + riscv_nousercfi = CMDLINE_DISABLE_RISCV_USERCFI; + + if (strcmp(str, "fcfi") == 0) + riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_FCFI; + + if (strcmp(str, "bcfi") == 0) + riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_BCFI; + + if (riscv_nousercfi) + pr_info("RISC-V user CFI disabled via cmdline - shadow stack status : %s, landing pad status : %s\n", + (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI) ? "disabled" : + "enabled", (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI) ? + "disabled" : "enabled"); + + return 1; +} + +__setup("riscv_nousercfi=", setup_global_riscv_enable); diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 3ca3ae4277e1..9c2f5e442338 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -13,20 +13,11 @@ #include <linux/err.h> #include <asm/page.h> #include <asm/vdso.h> -#include <linux/time_namespace.h> +#include <linux/vdso_datastore.h> #include <vdso/datapage.h> #include <vdso/vsyscall.h> -enum vvar_pages { - VVAR_DATA_PAGE_OFFSET, - VVAR_TIMENS_PAGE_OFFSET, - VVAR_NR_PAGES, -}; - -#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) - -static union vdso_data_store vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = vdso_data_store.data; +#define VVAR_SIZE (VDSO_NR_PAGES << PAGE_SHIFT) struct __vdso_info { const char *name; @@ -64,9 +55,7 @@ static void __init __vdso_init(struct __vdso_info *vdso_info) vdso_info->vdso_code_start) >> PAGE_SHIFT; - vdso_pagelist = kcalloc(vdso_info->vdso_pages, - sizeof(struct page *), - GFP_KERNEL); + vdso_pagelist = kzalloc_objs(struct page *, vdso_info->vdso_pages); if (vdso_pagelist == NULL) panic("vDSO kcalloc failed!\n"); @@ -79,78 +68,6 @@ static void __init __vdso_init(struct __vdso_info *vdso_info) vdso_info->cm->pages = vdso_pagelist; } -#ifdef CONFIG_TIME_NS -struct vdso_data *arch_get_vdso_data(void *vvar_page) -{ - return (struct vdso_data *)(vvar_page); -} - -static const struct vm_special_mapping rv_vvar_map; - -/* - * The vvar mapping contains data for a specific time namespace, so when a task - * changes namespace we must unmap its vvar data for the old namespace. - * Subsequent faults will map in data for the new namespace. - * - * For more details see timens_setup_vdso_data(). - */ -int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) -{ - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; - VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); - - for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &rv_vvar_map)) - zap_vma_pages(vma); - } - - mmap_read_unlock(mm); - return 0; -} -#endif - -static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, - struct vm_area_struct *vma, struct vm_fault *vmf) -{ - struct page *timens_page = find_timens_vvar_page(vma); - unsigned long pfn; - - switch (vmf->pgoff) { - case VVAR_DATA_PAGE_OFFSET: - if (timens_page) - pfn = page_to_pfn(timens_page); - else - pfn = sym_to_pfn(vdso_data); - break; -#ifdef CONFIG_TIME_NS - case VVAR_TIMENS_PAGE_OFFSET: - /* - * If a task belongs to a time namespace then a namespace - * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and - * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET - * offset. - * See also the comment near timens_setup_vdso_data(). - */ - if (!timens_page) - return VM_FAULT_SIGBUS; - pfn = sym_to_pfn(vdso_data); - break; -#endif /* CONFIG_TIME_NS */ - default: - return VM_FAULT_SIGBUS; - } - - return vmf_insert_pfn(vma, vmf->address, pfn); -} - -static const struct vm_special_mapping rv_vvar_map = { - .name = "[vvar]", - .fault = vvar_fault, -}; - static struct vm_special_mapping rv_vdso_map __ro_after_init = { .name = "[vdso]", .mremap = vdso_mremap, @@ -179,6 +96,13 @@ static struct __vdso_info compat_vdso_info __ro_after_init = { static int __init vdso_init(void) { + /* Hart implements zimop, expose cfi compiled vdso */ + if (IS_ENABLED(CONFIG_RISCV_USER_CFI) && + riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP)) { + vdso_info.vdso_code_start = vdso_cfi_start; + vdso_info.vdso_code_end = vdso_cfi_end; + } + __vdso_init(&vdso_info); #ifdef CONFIG_COMPAT __vdso_init(&compat_vdso_info); @@ -196,7 +120,7 @@ static int __setup_additional_pages(struct mm_struct *mm, unsigned long vdso_base, vdso_text_len, vdso_mapping_len; void *ret; - BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); + BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES); vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT; /* Be sure to map the data page */ @@ -208,8 +132,7 @@ static int __setup_additional_pages(struct mm_struct *mm, goto up_fail; } - ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE, - (VM_READ | VM_MAYREAD | VM_PFNMAP), &rv_vvar_map); + ret = vdso_install_vvar_mapping(mm, vdso_base); if (IS_ERR(ret)) goto up_fail; @@ -218,7 +141,7 @@ static int __setup_additional_pages(struct mm_struct *mm, ret = _install_special_mapping(mm, vdso_base, vdso_text_len, - (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC | VM_SEALED_SYSMAP), vdso_info->cm); if (IS_ERR(ret)) diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 9a1b555e8733..a842dc034571 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -2,7 +2,7 @@ # Copied from arch/tile/kernel/vdso/Makefile # Include the generic Makefile to check the built vdso. -include $(srctree)/lib/vdso/Makefile +include $(srctree)/lib/vdso/Makefile.include # Symbols present in the vdso vdso-syms = rt_sigreturn ifdef CONFIG_64BIT @@ -13,41 +13,74 @@ vdso-syms += flush_icache vdso-syms += hwprobe vdso-syms += sys_hwprobe +ifdef CONFIG_VDSO_GETRANDOM +vdso-syms += getrandom +endif + +ifdef VDSO_CFI_BUILD +CFI_MARCH = _zicfilp_zicfiss +CFI_FULL = -fcf-protection=full +CFI_SUFFIX = -cfi +OFFSET_SUFFIX = _cfi +ccflags-y += -DVDSO_CFI=1 +asflags-y += -DVDSO_CFI=1 +endif + # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o +ifdef CONFIG_VDSO_GETRANDOM +obj-vdso += vgetrandom-chacha.o +endif + ccflags-y := -fno-stack-protector ccflags-y += -DDISABLE_BRANCH_PROFILING ccflags-y += -fno-builtin +ccflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) +ccflags-y += $(CFI_FULL) +asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) +asflags-y += $(CFI_FULL) ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y) endif +ifneq ($(c-getrandom-y),) + CFLAGS_getrandom.o += -fPIC -include $(c-getrandom-y) +endif + CFLAGS_hwprobe.o += -fPIC # Build rules -targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds +vdso_offsets := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),)-offsets.h +vdso_o := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o +vdso_so := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so +vdso_so_dbg := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so.dbg +vdso_lds := vdso.lds + +targets := $(obj-vdso) $(vdso_so) $(vdso_so_dbg) $(vdso_lds) + obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) -obj-y += vdso.o -CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +obj-y += vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o +CPPFLAGS_$(vdso_lds) += -P -C -U$(ARCH) ifneq ($(filter vgettimeofday, $(vdso-syms)),) -CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY +CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY endif # Disable -pg to prevent insert call site CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) +CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) # Force dependency -$(obj)/vdso.o: $(obj)/vdso.so +$(obj)/$(vdso_o): $(obj)/$(vdso_so) # link rule for the .so file, .lds has to be first -$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE +$(obj)/$(vdso_so_dbg): $(obj)/$(vdso_lds) $(obj-vdso) FORCE $(call if_changed,vdsold_and_check) -LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \ - --build-id=sha1 --hash-style=both --eh-frame-hdr +LDFLAGS_$(vdso_so_dbg) = -shared -soname=linux-vdso.so.1 \ + --build-id=sha1 --eh-frame-hdr # strip rule for the .so file $(obj)/%.so: OBJCOPYFLAGS := -S @@ -57,16 +90,16 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE # Generate VDSO offsets using helper script gen-vdsosym := $(src)/gen_vdso_offsets.sh quiet_cmd_vdsosym = VDSOSYM $@ - cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@ + cmd_vdsosym = $(NM) $< | $(gen-vdsosym) $(OFFSET_SUFFIX) | LC_ALL=C sort > $@ -include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE +include/generated/$(vdso_offsets): $(obj)/$(vdso_so_dbg) FORCE $(call if_changed,vdsosym) # actual build commands # The DSO images are built using a special linker script # Make sure only to export the intended __vdso_xxx symbol offsets. quiet_cmd_vdsold_and_check = VDSOLD $@ - cmd_vdsold_and_check = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ + cmd_vdsold_and_check = $(LD) $(CFI_FULL) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \ $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ rm $@.tmp && \ $(cmd_vdso_check) diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S index 8f884227e8bc..e4c56970905e 100644 --- a/arch/riscv/kernel/vdso/flush_icache.S +++ b/arch/riscv/kernel/vdso/flush_icache.S @@ -5,11 +5,13 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text /* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */ SYM_FUNC_START(__vdso_flush_icache) .cfi_startproc + vdso_lpad #ifdef CONFIG_SMP li a7, __NR_riscv_flush_icache ecall @@ -20,3 +22,5 @@ SYM_FUNC_START(__vdso_flush_icache) ret .cfi_endproc SYM_FUNC_END(__vdso_flush_icache) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh index c2e5613f3495..bd5d5afaaa14 100755 --- a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh +++ b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh @@ -2,4 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 LC_ALL=C -sed -n -e 's/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2_offset\t0x\1/p' +SUFFIX=${1:-""} +sed -n -e \ +'s/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2'$SUFFIX'_offset\t0x\1/p' diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S index 9c1bd531907f..5c1ecc4e1465 100644 --- a/arch/riscv/kernel/vdso/getcpu.S +++ b/arch/riscv/kernel/vdso/getcpu.S @@ -5,14 +5,18 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text /* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */ SYM_FUNC_START(__vdso_getcpu) .cfi_startproc + vdso_lpad /* For now, just do the syscall. */ li a7, __NR_getcpu ecall ret .cfi_endproc SYM_FUNC_END(__vdso_getcpu) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/getrandom.c b/arch/riscv/kernel/vdso/getrandom.c new file mode 100644 index 000000000000..f21922e8cebd --- /dev/null +++ b/arch/riscv/kernel/vdso/getrandom.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved. + */ +#include <linux/types.h> + +ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len) +{ + return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len); +} diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c index a158c029344f..8f45500d0a6e 100644 --- a/arch/riscv/kernel/vdso/hwprobe.c +++ b/arch/riscv/kernel/vdso/hwprobe.c @@ -16,8 +16,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, size_t cpusetsize, unsigned long *cpus, unsigned int flags) { - const struct vdso_data *vd = __arch_get_vdso_data(); - const struct arch_vdso_time_data *avd = &vd->arch_data; + const struct vdso_arch_data *avd = &vdso_u_arch_data; bool all_cpus = !cpusetsize && !cpus; struct riscv_hwprobe *p = pairs; struct riscv_hwprobe *end = pairs + pair_count; @@ -28,7 +27,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count, * homogeneous, then this function can handle requests for arbitrary * masks. */ - if ((flags != 0) || (!all_cpus && !avd->homogeneous_cpus)) + if (flags != 0 || (!all_cpus && !avd->homogeneous_cpus) || unlikely(!avd->ready)) return riscv_hwprobe(pairs, pair_count, cpusetsize, cpus, flags); /* This is something we can handle, fill out the pairs. */ @@ -51,8 +50,7 @@ static int riscv_vdso_get_cpus(struct riscv_hwprobe *pairs, size_t pair_count, size_t cpusetsize, unsigned long *cpus, unsigned int flags) { - const struct vdso_data *vd = __arch_get_vdso_data(); - const struct arch_vdso_time_data *avd = &vd->arch_data; + const struct vdso_arch_data *avd = &vdso_u_arch_data; struct riscv_hwprobe *p = pairs; struct riscv_hwprobe *end = pairs + pair_count; unsigned char *c = (unsigned char *)cpus; diff --git a/arch/riscv/kernel/vdso/note.S b/arch/riscv/kernel/vdso/note.S index 2a956c942211..3d92cc956b95 100644 --- a/arch/riscv/kernel/vdso/note.S +++ b/arch/riscv/kernel/vdso/note.S @@ -6,7 +6,10 @@ #include <linux/elfnote.h> #include <linux/version.h> +#include <asm/assembler.h> ELFNOTE_START(Linux, 0, "a") .long LINUX_VERSION_CODE ELFNOTE_END + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S index 3dc022aa8931..e82987dc3739 100644 --- a/arch/riscv/kernel/vdso/rt_sigreturn.S +++ b/arch/riscv/kernel/vdso/rt_sigreturn.S @@ -5,12 +5,16 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text SYM_FUNC_START(__vdso_rt_sigreturn) .cfi_startproc .cfi_signal_frame + vdso_lpad li a7, __NR_rt_sigreturn ecall .cfi_endproc SYM_FUNC_END(__vdso_rt_sigreturn) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/sys_hwprobe.S b/arch/riscv/kernel/vdso/sys_hwprobe.S index 77e57f830521..f1694451a60c 100644 --- a/arch/riscv/kernel/vdso/sys_hwprobe.S +++ b/arch/riscv/kernel/vdso/sys_hwprobe.S @@ -3,13 +3,17 @@ #include <linux/linkage.h> #include <asm/unistd.h> +#include <asm/assembler.h> .text SYM_FUNC_START(riscv_hwprobe) .cfi_startproc + vdso_lpad li a7, __NR_riscv_hwprobe ecall ret .cfi_endproc SYM_FUNC_END(riscv_hwprobe) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index cbe2a179331d..c29ef12a63bb 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -4,15 +4,14 @@ */ #include <asm/page.h> #include <asm/vdso.h> +#include <vdso/datapage.h> OUTPUT_ARCH(riscv) SECTIONS { - PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE); -#ifdef CONFIG_TIME_NS - PROVIDE(_timens_data = _vdso_data + PAGE_SIZE); -#endif + VDSO_VVAR_SYMS + . = SIZEOF_HEADERS; .hash : { *(.hash) } :text @@ -31,7 +30,7 @@ SECTIONS *(.data .data.* .gnu.linkonce.d.*) *(.dynbss) *(.bss .bss.* .gnu.linkonce.b.*) - } + } :text .note : { *(.note.*) } :text :note @@ -81,6 +80,9 @@ VERSION #ifndef COMPAT_VDSO __vdso_riscv_hwprobe; #endif +#if defined(CONFIG_VDSO_GETRANDOM) && !defined(COMPAT_VDSO) + __vdso_getrandom; +#endif local: *; }; } diff --git a/arch/riscv/kernel/vdso/vgetrandom-chacha.S b/arch/riscv/kernel/vdso/vgetrandom-chacha.S new file mode 100644 index 000000000000..916ab30a88f7 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgetrandom-chacha.S @@ -0,0 +1,252 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved. + * + * Based on arch/loongarch/vdso/vgetrandom-chacha.S. + */ + +#include <asm/asm.h> +#include <linux/linkage.h> +#include <asm/assembler.h> + +.text + +.macro ROTRI rd rs imm + slliw t0, \rs, 32 - \imm + srliw \rd, \rs, \imm + or \rd, \rd, t0 +.endm + +.macro OP_4REG op d0 d1 d2 d3 s0 s1 s2 s3 + \op \d0, \d0, \s0 + \op \d1, \d1, \s1 + \op \d2, \d2, \s2 + \op \d3, \d3, \s3 +.endm + +/* + * a0: output bytes + * a1: 32-byte key input + * a2: 8-byte counter input/output + * a3: number of 64-byte blocks to write to output + */ +SYM_FUNC_START(__arch_chacha20_blocks_nostack) + +#define output a0 +#define key a1 +#define counter a2 +#define nblocks a3 +#define i a4 +#define state0 s0 +#define state1 s1 +#define state2 s2 +#define state3 s3 +#define state4 s4 +#define state5 s5 +#define state6 s6 +#define state7 s7 +#define state8 s8 +#define state9 s9 +#define state10 s10 +#define state11 s11 +#define state12 a5 +#define state13 a6 +#define state14 a7 +#define state15 t1 +#define cnt t2 +#define copy0 t3 +#define copy1 t4 +#define copy2 t5 +#define copy3 t6 + +/* Packs to be used with OP_4REG */ +#define line0 state0, state1, state2, state3 +#define line1 state4, state5, state6, state7 +#define line2 state8, state9, state10, state11 +#define line3 state12, state13, state14, state15 + +#define line1_perm state5, state6, state7, state4 +#define line2_perm state10, state11, state8, state9 +#define line3_perm state15, state12, state13, state14 + +#define copy copy0, copy1, copy2, copy3 + +#define _16 16, 16, 16, 16 +#define _20 20, 20, 20, 20 +#define _24 24, 24, 24, 24 +#define _25 25, 25, 25, 25 + vdso_lpad + /* + * The ABI requires s0-s9 saved. + * This does not violate the stack-less requirement: no sensitive data + * is spilled onto the stack. + */ + addi sp, sp, -12*SZREG + REG_S s0, (sp) + REG_S s1, SZREG(sp) + REG_S s2, 2*SZREG(sp) + REG_S s3, 3*SZREG(sp) + REG_S s4, 4*SZREG(sp) + REG_S s5, 5*SZREG(sp) + REG_S s6, 6*SZREG(sp) + REG_S s7, 7*SZREG(sp) + REG_S s8, 8*SZREG(sp) + REG_S s9, 9*SZREG(sp) + REG_S s10, 10*SZREG(sp) + REG_S s11, 11*SZREG(sp) + + ld cnt, (counter) + + li copy0, 0x61707865 + li copy1, 0x3320646e + li copy2, 0x79622d32 + li copy3, 0x6b206574 + +.Lblock: + /* state[0,1,2,3] = "expand 32-byte k" */ + mv state0, copy0 + mv state1, copy1 + mv state2, copy2 + mv state3, copy3 + + /* state[4,5,..,11] = key */ + lw state4, (key) + lw state5, 4(key) + lw state6, 8(key) + lw state7, 12(key) + lw state8, 16(key) + lw state9, 20(key) + lw state10, 24(key) + lw state11, 28(key) + + /* state[12,13] = counter */ + mv state12, cnt + srli state13, cnt, 32 + + /* state[14,15] = 0 */ + mv state14, zero + mv state15, zero + + li i, 10 +.Lpermute: + /* odd round */ + OP_4REG addw line0, line1 + OP_4REG xor line3, line0 + OP_4REG ROTRI line3, _16 + + OP_4REG addw line2, line3 + OP_4REG xor line1, line2 + OP_4REG ROTRI line1, _20 + + OP_4REG addw line0, line1 + OP_4REG xor line3, line0 + OP_4REG ROTRI line3, _24 + + OP_4REG addw line2, line3 + OP_4REG xor line1, line2 + OP_4REG ROTRI line1, _25 + + /* even round */ + OP_4REG addw line0, line1_perm + OP_4REG xor line3_perm, line0 + OP_4REG ROTRI line3_perm, _16 + + OP_4REG addw line2_perm, line3_perm + OP_4REG xor line1_perm, line2_perm + OP_4REG ROTRI line1_perm, _20 + + OP_4REG addw line0, line1_perm + OP_4REG xor line3_perm, line0 + OP_4REG ROTRI line3_perm, _24 + + OP_4REG addw line2_perm, line3_perm + OP_4REG xor line1_perm, line2_perm + OP_4REG ROTRI line1_perm, _25 + + addi i, i, -1 + bnez i, .Lpermute + + /* output[0,1,2,3] = copy[0,1,2,3] + state[0,1,2,3] */ + OP_4REG addw line0, copy + sw state0, (output) + sw state1, 4(output) + sw state2, 8(output) + sw state3, 12(output) + + /* from now on state[0,1,2,3] are scratch registers */ + + /* state[0,1,2,3] = lo(key) */ + lw state0, (key) + lw state1, 4(key) + lw state2, 8(key) + lw state3, 12(key) + + /* output[4,5,6,7] = state[0,1,2,3] + state[4,5,6,7] */ + OP_4REG addw line1, line0 + sw state4, 16(output) + sw state5, 20(output) + sw state6, 24(output) + sw state7, 28(output) + + /* state[0,1,2,3] = hi(key) */ + lw state0, 16(key) + lw state1, 20(key) + lw state2, 24(key) + lw state3, 28(key) + + /* output[8,9,10,11] = tmp[0,1,2,3] + state[8,9,10,11] */ + OP_4REG addw line2, line0 + sw state8, 32(output) + sw state9, 36(output) + sw state10, 40(output) + sw state11, 44(output) + + /* output[12,13,14,15] = state[12,13,14,15] + [cnt_lo, cnt_hi, 0, 0] */ + addw state12, state12, cnt + srli state0, cnt, 32 + addw state13, state13, state0 + sw state12, 48(output) + sw state13, 52(output) + sw state14, 56(output) + sw state15, 60(output) + + /* ++counter */ + addi cnt, cnt, 1 + + /* output += 64 */ + addi output, output, 64 + /* --nblocks */ + addi nblocks, nblocks, -1 + bnez nblocks, .Lblock + + /* counter = [cnt_lo, cnt_hi] */ + sd cnt, (counter) + + /* Zero out the potentially sensitive regs, in case nothing uses these + * again. As at now copy[0,1,2,3] just contains "expand 32-byte k" and + * state[0,...,11] are s0-s11 those we'll restore in the epilogue, we + * only need to zero state[12,...,15]. + */ + mv state12, zero + mv state13, zero + mv state14, zero + mv state15, zero + + REG_L s0, (sp) + REG_L s1, SZREG(sp) + REG_L s2, 2*SZREG(sp) + REG_L s3, 3*SZREG(sp) + REG_L s4, 4*SZREG(sp) + REG_L s5, 5*SZREG(sp) + REG_L s6, 6*SZREG(sp) + REG_L s7, 7*SZREG(sp) + REG_L s8, 8*SZREG(sp) + REG_L s9, 9*SZREG(sp) + REG_L s10, 10*SZREG(sp) + REG_L s11, 11*SZREG(sp) + addi sp, sp, 12*SZREG + + ret +SYM_FUNC_END(__arch_chacha20_blocks_nostack) + +emit_riscv_feature_1_and diff --git a/arch/riscv/kernel/vdso_cfi/.gitignore b/arch/riscv/kernel/vdso_cfi/.gitignore new file mode 100644 index 000000000000..220b6ebece4f --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/.gitignore @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copied source files from the main vdso directory +*.c +*.S +!vdso-cfi.S +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/riscv/kernel/vdso_cfi/Makefile b/arch/riscv/kernel/vdso_cfi/Makefile new file mode 100644 index 000000000000..10292498b765 --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/Makefile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only +# RISC-V VDSO CFI Makefile +# This Makefile builds the VDSO with CFI support when CONFIG_RISCV_USER_CFI is enabled + +# setting VDSO_CFI_BUILD triggers build for vdso differently +VDSO_CFI_BUILD := 1 + +# Set the source directory to the main vdso directory +src := $(srctree)/arch/riscv/kernel/vdso + +# Copy all .S and .c files from vdso directory to vdso_cfi object build directory +vdso_c_sources := $(wildcard $(src)/*.c) +vdso_S_sources := $(wildcard $(src)/*.S) +vdso_c_objects := $(addprefix $(obj)/, $(notdir $(vdso_c_sources))) +vdso_S_objects := $(addprefix $(obj)/, $(notdir $(vdso_S_sources))) + +$(vdso_S_objects): $(obj)/%.S: $(src)/%.S + $(Q)cp $< $@ + +$(vdso_c_objects): $(obj)/%.c: $(src)/%.c + $(Q)cp $< $@ + +# Include the main VDSO Makefile which contains all the build rules and sources +# The VDSO_CFI_BUILD variable will be passed to it to enable CFI compilation +include $(src)/Makefile + +# Clean rules - remove the copied source files +clean-files += $(notdir $(vdso_c_sources)) $(notdir $(vdso_S_sources)) diff --git a/arch/riscv/kernel/vdso_cfi/vdso-cfi.S b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S new file mode 100644 index 000000000000..d426f6accb35 --- /dev/null +++ b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2025 Rivos, Inc + */ + +#define vdso_start vdso_cfi_start +#define vdso_end vdso_cfi_end + +#define __VDSO_PATH "arch/riscv/kernel/vdso_cfi/vdso-cfi.so" + +#include "../vdso/vdso.S" diff --git a/arch/riscv/kernel/vec-copy-unaligned.S b/arch/riscv/kernel/vec-copy-unaligned.S index d16f19f1b3b6..361039f7b944 100644 --- a/arch/riscv/kernel/vec-copy-unaligned.S +++ b/arch/riscv/kernel/vec-copy-unaligned.S @@ -2,6 +2,7 @@ /* Copyright (C) 2024 Rivos Inc. */ #include <linux/args.h> +#include <linux/cfi_types.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -11,12 +12,12 @@ #define WORD_SEW CONCATENATE(e, WORD_EEW) #define VEC_L CONCATENATE(vle, WORD_EEW).v -#define VEC_S CONCATENATE(vle, WORD_EEW).v +#define VEC_S CONCATENATE(vse, WORD_EEW).v /* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using word loads and stores. */ /* Note: The size is truncated to a multiple of WORD_EEW */ -SYM_FUNC_START(__riscv_copy_vec_words_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_vec_words_unaligned) andi a4, a2, ~(WORD_EEW-1) beqz a4, 2f add a3, a1, a4 @@ -38,7 +39,7 @@ SYM_FUNC_END(__riscv_copy_vec_words_unaligned) /* void __riscv_copy_vec_bytes_unaligned(void *, const void *, size_t) */ /* Performs a memcpy without aligning buffers, using only byte accesses. */ /* Note: The size is truncated to a multiple of 8 */ -SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned) +SYM_TYPED_FUNC_START(__riscv_copy_vec_bytes_unaligned) andi a4, a2, ~(8-1) beqz a4, 2f add a3, a1, a4 diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 184f780c932d..b112166d51e9 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -66,6 +66,8 @@ void __init riscv_v_setup_ctx_cache(void) if (!(has_vector() || has_xtheadvector())) return; + update_regset_vector_info(riscv_v_vsize); + riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx", riscv_v_vsize, 16, SLAB_PANIC, 0, riscv_v_vsize, NULL); @@ -93,7 +95,7 @@ bool insn_is_vector(u32 insn_buf) return true; case RVV_OPCODE_VL: case RVV_OPCODE_VS: - width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); + width = RVV_EXTRACT_VL_VS_WIDTH(insn_buf); if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) return true; @@ -109,8 +111,8 @@ bool insn_is_vector(u32 insn_buf) return false; } -static int riscv_v_thread_zalloc(struct kmem_cache *cache, - struct __riscv_v_ext_state *ctx) +static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, + struct __riscv_v_ext_state *ctx) { void *datap; @@ -120,13 +122,15 @@ static int riscv_v_thread_zalloc(struct kmem_cache *cache, ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); + ctx->vlenb = riscv_v_vsize / 32; + return 0; } void riscv_v_thread_alloc(struct task_struct *tsk) { #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE - riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); + riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); #endif } @@ -212,12 +216,14 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ - if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { + if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; } + riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs); + return true; } diff --git a/arch/riscv/kernel/vendor_extensions.c b/arch/riscv/kernel/vendor_extensions.c index a31ff84740eb..bb4a75923685 100644 --- a/arch/riscv/kernel/vendor_extensions.c +++ b/arch/riscv/kernel/vendor_extensions.c @@ -6,6 +6,8 @@ #include <asm/vendorid_list.h> #include <asm/vendor_extensions.h> #include <asm/vendor_extensions/andes.h> +#include <asm/vendor_extensions/mips.h> +#include <asm/vendor_extensions/sifive.h> #include <asm/vendor_extensions/thead.h> #include <linux/array_size.h> @@ -15,6 +17,12 @@ struct riscv_isa_vendor_ext_data_list *riscv_isa_vendor_ext_list[] = { #ifdef CONFIG_RISCV_ISA_VENDOR_EXT_ANDES &riscv_isa_vendor_ext_list_andes, #endif +#ifdef CONFIG_RISCV_ISA_VENDOR_EXT_MIPS + &riscv_isa_vendor_ext_list_mips, +#endif +#ifdef CONFIG_RISCV_ISA_VENDOR_EXT_SIFIVE + &riscv_isa_vendor_ext_list_sifive, +#endif #ifdef CONFIG_RISCV_ISA_VENDOR_EXT_THEAD &riscv_isa_vendor_ext_list_thead, #endif @@ -45,6 +53,18 @@ bool __riscv_isa_vendor_extension_available(int cpu, unsigned long vendor, unsig cpu_bmap = riscv_isa_vendor_ext_list_andes.per_hart_isa_bitmap; break; #endif + #ifdef CONFIG_RISCV_ISA_VENDOR_EXT_MIPS + case MIPS_VENDOR_ID: + bmap = &riscv_isa_vendor_ext_list_mips.all_harts_isa_bitmap; + cpu_bmap = riscv_isa_vendor_ext_list_mips.per_hart_isa_bitmap; + break; + #endif + #ifdef CONFIG_RISCV_ISA_VENDOR_EXT_SIFIVE + case SIFIVE_VENDOR_ID: + bmap = &riscv_isa_vendor_ext_list_sifive.all_harts_isa_bitmap; + cpu_bmap = riscv_isa_vendor_ext_list_sifive.per_hart_isa_bitmap; + break; + #endif #ifdef CONFIG_RISCV_ISA_VENDOR_EXT_THEAD case THEAD_VENDOR_ID: bmap = &riscv_isa_vendor_ext_list_thead.all_harts_isa_bitmap; @@ -61,6 +81,6 @@ bool __riscv_isa_vendor_extension_available(int cpu, unsigned long vendor, unsig if (bit >= RISCV_ISA_VENDOR_EXT_MAX) return false; - return test_bit(bit, bmap->isa) ? true : false; + return test_bit(bit, bmap->isa); } EXPORT_SYMBOL_GPL(__riscv_isa_vendor_extension_available); diff --git a/arch/riscv/kernel/vendor_extensions/Makefile b/arch/riscv/kernel/vendor_extensions/Makefile index 866414c81a9f..bf116c82b6bd 100644 --- a/arch/riscv/kernel/vendor_extensions/Makefile +++ b/arch/riscv/kernel/vendor_extensions/Makefile @@ -1,5 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_ANDES) += andes.o +obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_MIPS) += mips.o +obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_MIPS) += mips_hwprobe.o +obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_SIFIVE) += sifive.o +obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_SIFIVE) += sifive_hwprobe.o obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_THEAD) += thead.o obj-$(CONFIG_RISCV_ISA_VENDOR_EXT_THEAD) += thead_hwprobe.o diff --git a/arch/riscv/kernel/vendor_extensions/mips.c b/arch/riscv/kernel/vendor_extensions/mips.c new file mode 100644 index 000000000000..f691129f96c2 --- /dev/null +++ b/arch/riscv/kernel/vendor_extensions/mips.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 MIPS. + */ + +#include <asm/cpufeature.h> +#include <asm/vendor_extensions.h> +#include <asm/vendor_extensions/mips.h> + +#include <linux/array_size.h> +#include <linux/cpumask.h> +#include <linux/types.h> + +/* All MIPS vendor extensions supported in Linux */ +static const struct riscv_isa_ext_data riscv_isa_vendor_ext_mips[] = { + __RISCV_ISA_EXT_DATA(xmipsexectl, RISCV_ISA_VENDOR_EXT_XMIPSEXECTL), +}; + +struct riscv_isa_vendor_ext_data_list riscv_isa_vendor_ext_list_mips = { + .ext_data_count = ARRAY_SIZE(riscv_isa_vendor_ext_mips), + .ext_data = riscv_isa_vendor_ext_mips, +}; diff --git a/arch/riscv/kernel/vendor_extensions/mips_hwprobe.c b/arch/riscv/kernel/vendor_extensions/mips_hwprobe.c new file mode 100644 index 000000000000..dc213a2ca70d --- /dev/null +++ b/arch/riscv/kernel/vendor_extensions/mips_hwprobe.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 MIPS. + */ + +#include <asm/vendor_extensions.h> +#include <asm/vendor_extensions/mips.h> +#include <asm/vendor_extensions/mips_hwprobe.h> +#include <asm/vendor_extensions/vendor_hwprobe.h> + +#include <linux/cpumask.h> +#include <linux/types.h> + +#include <uapi/asm/hwprobe.h> +#include <uapi/asm/vendor/mips.h> + +void hwprobe_isa_vendor_ext_mips_0(struct riscv_hwprobe *pair, + const struct cpumask *cpus) +{ + VENDOR_EXTENSION_SUPPORTED(pair, cpus, + riscv_isa_vendor_ext_list_mips.per_hart_isa_bitmap, + { VENDOR_EXT_KEY(XMIPSEXECTL); }); +} diff --git a/arch/riscv/kernel/vendor_extensions/sifive.c b/arch/riscv/kernel/vendor_extensions/sifive.c new file mode 100644 index 000000000000..8fcf67e8c07f --- /dev/null +++ b/arch/riscv/kernel/vendor_extensions/sifive.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <asm/cpufeature.h> +#include <asm/vendor_extensions.h> +#include <asm/vendor_extensions/sifive.h> + +#include <linux/array_size.h> +#include <linux/types.h> + +/* All SiFive vendor extensions supported in Linux */ +static const struct riscv_isa_ext_data riscv_isa_vendor_ext_sifive[] = { + __RISCV_ISA_EXT_DATA(xsfvfnrclipxfqf, RISCV_ISA_VENDOR_EXT_XSFVFNRCLIPXFQF), + __RISCV_ISA_EXT_DATA(xsfvfwmaccqqq, RISCV_ISA_VENDOR_EXT_XSFVFWMACCQQQ), + __RISCV_ISA_EXT_DATA(xsfvqmaccdod, RISCV_ISA_VENDOR_EXT_XSFVQMACCDOD), + __RISCV_ISA_EXT_DATA(xsfvqmaccqoq, RISCV_ISA_VENDOR_EXT_XSFVQMACCQOQ), +}; + +struct riscv_isa_vendor_ext_data_list riscv_isa_vendor_ext_list_sifive = { + .ext_data_count = ARRAY_SIZE(riscv_isa_vendor_ext_sifive), + .ext_data = riscv_isa_vendor_ext_sifive, +}; diff --git a/arch/riscv/kernel/vendor_extensions/sifive_hwprobe.c b/arch/riscv/kernel/vendor_extensions/sifive_hwprobe.c new file mode 100644 index 000000000000..1f77f6309763 --- /dev/null +++ b/arch/riscv/kernel/vendor_extensions/sifive_hwprobe.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <asm/vendor_extensions/sifive.h> +#include <asm/vendor_extensions/sifive_hwprobe.h> +#include <asm/vendor_extensions/vendor_hwprobe.h> + +#include <linux/cpumask.h> +#include <linux/types.h> + +#include <uapi/asm/hwprobe.h> +#include <uapi/asm/vendor/sifive.h> + +void hwprobe_isa_vendor_ext_sifive_0(struct riscv_hwprobe *pair, const struct cpumask *cpus) +{ + VENDOR_EXTENSION_SUPPORTED(pair, cpus, + riscv_isa_vendor_ext_list_sifive.per_hart_isa_bitmap, { + VENDOR_EXT_KEY(XSFVQMACCDOD); + VENDOR_EXT_KEY(XSFVQMACCQOQ); + VENDOR_EXT_KEY(XSFVFNRCLIPXFQF); + VENDOR_EXT_KEY(XSFVFWMACCQQQ); + }); +} diff --git a/arch/riscv/kernel/vmcore_info.c b/arch/riscv/kernel/vmcore_info.c index d5e448aa90e7..c27efceec3cc 100644 --- a/arch/riscv/kernel/vmcore_info.c +++ b/arch/riscv/kernel/vmcore_info.c @@ -3,6 +3,11 @@ #include <linux/vmcore_info.h> #include <linux/pagemap.h> +static inline u64 get_satp_value(void) +{ + return csr_read(CSR_SATP); +} + void arch_crash_save_vmcoreinfo(void) { VMCOREINFO_NUMBER(phys_ram_base); @@ -19,13 +24,8 @@ void arch_crash_save_vmcoreinfo(void) #endif #endif vmcoreinfo_append_str("NUMBER(KERNEL_LINK_ADDR)=0x%lx\n", KERNEL_LINK_ADDR); -#ifdef CONFIG_XIP_KERNEL - /* TODO: Communicate with crash-utility developers on the information to - * export. The XIP case is more complicated, because the virtual-physical - * address offset depends on whether the address is in ROM or in RAM. - */ -#else vmcoreinfo_append_str("NUMBER(va_kernel_pa_offset)=0x%lx\n", kernel_map.va_kernel_pa_offset); -#endif + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); + vmcoreinfo_append_str("NUMBER(satp)=0x%llx\n", get_satp_value()); } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 002ca58dd998..1f4f8496941a 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -7,10 +7,6 @@ #define RO_EXCEPTION_TABLE_ALIGN 4 #define RUNTIME_DISCARD_EXIT -#ifdef CONFIG_XIP_KERNEL -#include "vmlinux-xip.lds.S" -#else - #include <asm/pgtable.h> #define LOAD_OFFSET KERNEL_LINK_ADDR @@ -97,6 +93,9 @@ SECTIONS { EXIT_DATA } + + RUNTIME_CONST_VARIABLES + PERCPU_SECTION(L1_CACHE_BYTES) .rel.dyn : { @@ -167,9 +166,9 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .riscv.attributes 0 : { *(.riscv.attributes) } DISCARDS } -#endif /* CONFIG_XIP_KERNEL */ diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 0c3cbb0915ff..ec2cee0a39e0 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -18,19 +18,18 @@ menuconfig VIRTUALIZATION if VIRTUALIZATION config KVM - tristate "Kernel-based Virtual Machine (KVM) support (EXPERIMENTAL)" + tristate "Kernel-based Virtual Machine (KVM) support" depends on RISCV_SBI && MMU select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_MSI - select HAVE_KVM_VCPU_ASYNC_IOCTL select HAVE_KVM_READONLY_MEM + select HAVE_KVM_DIRTY_RING_ACQ_REL select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING select KVM_MMIO - select KVM_XFER_TO_GUEST_WORK - select KVM_GENERIC_MMU_NOTIFIER + select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS help diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile index 4e0bba91d284..296c2ba05089 100644 --- a/arch/riscv/kvm/Makefile +++ b/arch/riscv/kvm/Makefile @@ -14,11 +14,14 @@ kvm-y += aia.o kvm-y += aia_aplic.o kvm-y += aia_device.o kvm-y += aia_imsic.o +kvm-y += gstage.o +kvm-y += isa.o kvm-y += main.o kvm-y += mmu.o kvm-y += nacl.o kvm-y += tlb.o kvm-y += vcpu.o +kvm-y += vcpu_config.o kvm-y += vcpu_exit.o kvm-y += vcpu_fp.o kvm-y += vcpu_insn.o @@ -26,6 +29,8 @@ kvm-y += vcpu_onereg.o kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o kvm-y += vcpu_sbi.o kvm-y += vcpu_sbi_base.o +kvm-y += vcpu_sbi_forward.o +kvm-y += vcpu_sbi_fwft.o kvm-y += vcpu_sbi_hsm.o kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_sbi_pmu.o kvm-y += vcpu_sbi_replace.o diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index 19afd1f23537..5ec503288555 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -13,6 +13,7 @@ #include <linux/irqchip/riscv-imsic.h> #include <linux/irqdomain.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/percpu.h> #include <linux/spinlock.h> #include <asm/cpufeature.h> @@ -30,28 +31,6 @@ unsigned int kvm_riscv_aia_nr_hgei; unsigned int kvm_riscv_aia_max_ids; DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); -static int aia_find_hgei(struct kvm_vcpu *owner) -{ - int i, hgei; - unsigned long flags; - struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); - - raw_spin_lock_irqsave(&hgctrl->lock, flags); - - hgei = -1; - for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) { - if (hgctrl->owners[i] == owner) { - hgei = i; - break; - } - } - - raw_spin_unlock_irqrestore(&hgctrl->lock, flags); - - put_cpu_ptr(&aia_hgei); - return hgei; -} - static inline unsigned long aia_hvictl_value(bool ext_irq_pending) { unsigned long hvictl; @@ -95,7 +74,6 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { - int hgei; unsigned long seip; if (!kvm_riscv_aia_available()) @@ -114,11 +92,7 @@ bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) return false; - hgei = aia_find_hgei(vcpu); - if (hgei > 0) - return !!(ncsr_read(CSR_HGEIP) & BIT(hgei)); - - return false; + return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu); } void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) @@ -164,6 +138,9 @@ void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) csr_write(CSR_HVIPRIO2H, csr->hviprio2h); #endif } + + if (kvm_riscv_aia_initialized(vcpu->kvm)) + kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu); } void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) @@ -174,6 +151,9 @@ void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) if (!kvm_riscv_aia_available()) return; + if (kvm_riscv_aia_initialized(vcpu->kvm)) + kvm_riscv_vcpu_aia_imsic_put(vcpu); + if (kvm_riscv_nacl_available()) { nsh = nacl_shmem(); csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT); @@ -203,9 +183,14 @@ int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) + if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); *out_val = 0; if (kvm_riscv_aia_available()) @@ -219,9 +204,14 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, unsigned long val) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) + if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); if (kvm_riscv_aia_available()) { ((unsigned long *)csr)[reg_num] = val; @@ -472,22 +462,6 @@ void kvm_riscv_aia_free_hgei(int cpu, int hgei) raw_spin_unlock_irqrestore(&hgctrl->lock, flags); } -void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable) -{ - int hgei; - - if (!kvm_riscv_aia_available()) - return; - - hgei = aia_find_hgei(owner); - if (hgei > 0) { - if (enable) - csr_set(CSR_HGEIE, BIT(hgei)); - else - csr_clear(CSR_HGEIE, BIT(hgei)); - } -} - static irqreturn_t hgei_interrupt(int irq, void *dev_id) { int i; @@ -667,7 +641,7 @@ int kvm_riscv_aia_init(void) */ if (gc) kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei, - BIT(gc->guest_index_bits) - 1); + gc->nr_guest_files); else kvm_riscv_aia_nr_hgei = 0; diff --git a/arch/riscv/kvm/aia_aplic.c b/arch/riscv/kvm/aia_aplic.c index f59d1c0c8c43..3464f3351df7 100644 --- a/arch/riscv/kvm/aia_aplic.c +++ b/arch/riscv/kvm/aia_aplic.c @@ -10,6 +10,7 @@ #include <linux/irqchip/riscv-aplic.h> #include <linux/kvm_host.h> #include <linux/math.h> +#include <linux/nospec.h> #include <linux/spinlock.h> #include <linux/swab.h> #include <kvm/iodev.h> @@ -45,7 +46,7 @@ static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return 0; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = irqd->sourcecfg; @@ -61,7 +62,7 @@ static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; if (val & APLIC_SOURCECFG_D) val = 0; @@ -81,7 +82,7 @@ static u32 aplic_read_target(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return 0; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = irqd->target; @@ -97,7 +98,7 @@ static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; val &= APLIC_TARGET_EIID_MASK | (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) | @@ -116,7 +117,7 @@ static bool aplic_read_pending(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false; @@ -132,7 +133,7 @@ static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -170,7 +171,7 @@ static bool aplic_read_enabled(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false; @@ -186,7 +187,7 @@ static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled) if (!irq || aplic->nr_irqs <= irq) return; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); if (enabled) @@ -205,7 +206,7 @@ static bool aplic_read_input(struct aplic *aplic, u32 irq) if (!irq || aplic->nr_irqs <= irq) return false; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -254,7 +255,7 @@ static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last) for (irq = first; irq <= last; irq++) { if (!irq || aplic->nr_irqs <= irq) continue; - irqd = &aplic->irqs[irq]; + irqd = &aplic->irqs[array_index_nospec(irq, aplic->nr_irqs)]; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -283,7 +284,7 @@ int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level) if (!aplic || !source || (aplic->nr_irqs <= source)) return -ENODEV; - irqd = &aplic->irqs[source]; + irqd = &aplic->irqs[array_index_nospec(source, aplic->nr_irqs)]; ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false; raw_spin_lock_irqsave(&irqd->lock, flags); @@ -580,7 +581,7 @@ int kvm_riscv_aia_aplic_init(struct kvm *kvm) return 0; /* Allocate APLIC global state */ - aplic = kzalloc(sizeof(*aplic), GFP_KERNEL); + aplic = kzalloc_obj(*aplic); if (!aplic) return -ENOMEM; kvm->arch.aia.aplic_state = aplic; @@ -588,8 +589,7 @@ int kvm_riscv_aia_aplic_init(struct kvm *kvm) /* Setup APLIC IRQs */ aplic->nr_irqs = kvm->arch.aia.nr_sources + 1; aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32); - aplic->irqs = kcalloc(aplic->nr_irqs, - sizeof(*aplic->irqs), GFP_KERNEL); + aplic->irqs = kzalloc_objs(*aplic->irqs, aplic->nr_irqs); if (!aplic->irqs) { ret = -ENOMEM; goto fail_free_aplic; diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c index 39cd26af5a69..3d1e81e2a36b 100644 --- a/arch/riscv/kvm/aia_device.c +++ b/arch/riscv/kvm/aia_device.c @@ -11,36 +11,7 @@ #include <linux/irqchip/riscv-imsic.h> #include <linux/kvm_host.h> #include <linux/uaccess.h> - -static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx) -{ - struct kvm_vcpu *tmp_vcpu; - - for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { - tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx); - mutex_unlock(&tmp_vcpu->mutex); - } -} - -static void unlock_all_vcpus(struct kvm *kvm) -{ - unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1); -} - -static bool lock_all_vcpus(struct kvm *kvm) -{ - struct kvm_vcpu *tmp_vcpu; - unsigned long c; - - kvm_for_each_vcpu(c, tmp_vcpu, kvm) { - if (!mutex_trylock(&tmp_vcpu->mutex)) { - unlock_vcpus(kvm, c - 1); - return false; - } - } - - return true; -} +#include <asm/kvm_isa.h> static int aia_create(struct kvm_device *dev, u32 type) { @@ -52,8 +23,11 @@ static int aia_create(struct kvm_device *dev, u32 type) if (irqchip_in_kernel(kvm)) return -EEXIST; + if (kvm_riscv_isa_check_host(SSAIA)) + return -ENODEV; + ret = -EBUSY; - if (!lock_all_vcpus(kvm)) + if (kvm_trylock_all_vcpus(kvm)) return ret; kvm_for_each_vcpu(i, vcpu, kvm) { @@ -65,7 +39,7 @@ static int aia_create(struct kvm_device *dev, u32 type) kvm->arch.aia.in_kernel = true; out_unlock: - unlock_all_vcpus(kvm); + kvm_unlock_all_vcpus(kvm); return ret; } @@ -467,7 +441,7 @@ static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { - int nr_vcpus; + int nr_vcpus, r = -ENXIO; switch (attr->group) { case KVM_DEV_RISCV_AIA_GRP_CONFIG: @@ -496,12 +470,18 @@ static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) } break; case KVM_DEV_RISCV_AIA_GRP_APLIC: - return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr); + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr); + mutex_unlock(&dev->kvm->lock); + break; case KVM_DEV_RISCV_AIA_GRP_IMSIC: - return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr); + mutex_lock(&dev->kvm->lock); + r = kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr); + mutex_unlock(&dev->kvm->lock); + break; } - return -ENXIO; + return r; } struct kvm_device_ops kvm_riscv_aia_device_ops = { @@ -526,12 +506,10 @@ int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu) void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; - struct kvm_vcpu_aia_csr *reset_csr = - &vcpu->arch.aia_context.guest_reset_csr; if (!kvm_riscv_aia_available()) return; - memcpy(csr, reset_csr, sizeof(*csr)); + memset(csr, 0, sizeof(*csr)); /* Proceed only if AIA was initialized successfully */ if (!kvm_riscv_aia_initialized(vcpu->kvm)) @@ -541,12 +519,12 @@ void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_aia_imsic_reset(vcpu); } -int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu) +void kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context; if (!kvm_riscv_aia_available()) - return 0; + return; /* * We don't do any memory allocations over here because these @@ -558,8 +536,6 @@ int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu) /* Initialize default values in AIA vcpu context */ vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR; vaia->hart_index = vcpu->vcpu_idx; - - return 0; } void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index a8085cd8215e..8786f52cf65a 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -16,6 +16,7 @@ #include <linux/swab.h> #include <kvm/iodev.h> #include <asm/csr.h> +#include <asm/kvm_mmu.h> #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64)) @@ -676,6 +677,60 @@ static void imsic_swfile_update(struct kvm_vcpu *vcpu, imsic_swfile_extirq_update(vcpu); } +bool kvm_riscv_vcpu_aia_imsic_has_interrupt(struct kvm_vcpu *vcpu) +{ + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + unsigned long flags; + bool ret = false; + + /* + * The IMSIC SW-file directly injects interrupt via hvip so + * only check for interrupt when IMSIC VS-file is being used. + */ + + read_lock_irqsave(&imsic->vsfile_lock, flags); + if (imsic->vsfile_cpu > -1) { + /* + * This function is typically called from kvm_vcpu_block() via + * kvm_arch_vcpu_runnable() upon WFI trap. The kvm_vcpu_block() + * can be preempted and the blocking VCPU might resume on a + * different CPU. This means it is possible that current CPU + * does not match the imsic->vsfile_cpu hence this function + * must check imsic->vsfile_cpu before accessing HGEIP CSR. + */ + if (imsic->vsfile_cpu != vcpu->cpu) + ret = true; + else + ret = !!(csr_read(CSR_HGEIP) & BIT(imsic->vsfile_hgei)); + } + read_unlock_irqrestore(&imsic->vsfile_lock, flags); + + return ret; +} + +void kvm_riscv_vcpu_aia_imsic_load(struct kvm_vcpu *vcpu, int cpu) +{ + /* + * No need to explicitly clear HGEIE CSR bits because the + * hgei interrupt handler (aka hgei_interrupt()) will always + * clear it for us. + */ +} + +void kvm_riscv_vcpu_aia_imsic_put(struct kvm_vcpu *vcpu) +{ + struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + unsigned long flags; + + if (!kvm_vcpu_is_blocking(vcpu)) + return; + + read_lock_irqsave(&imsic->vsfile_lock, flags); + if (imsic->vsfile_cpu > -1) + csr_set(CSR_HGEIE, BIT(imsic->vsfile_hgei)); + read_unlock_irqrestore(&imsic->vsfile_lock, flags); +} + void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) { unsigned long flags; @@ -703,9 +758,8 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) */ /* Purge the G-stage mapping */ - kvm_riscv_gstage_iounmap(vcpu->kvm, - vcpu->arch.aia_context.imsic_addr, - IMSIC_MMIO_PAGE_SZ); + kvm_riscv_mmu_iounmap(vcpu->kvm, vcpu->arch.aia_context.imsic_addr, + IMSIC_MMIO_PAGE_SZ); /* TODO: Purge the IOMMU mapping ??? */ @@ -743,6 +797,10 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL) return 1; + /* IMSIC vCPU state may not be initialized yet */ + if (!imsic) + return 1; + /* Read old IMSIC VS-file details */ read_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; @@ -760,7 +818,7 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) /* For HW acceleration mode, we can't continue */ if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) { run->fail_entry.hardware_entry_failure_reason = - CSR_HSTATUS; + KVM_EXIT_FAIL_ENTRY_NO_VSFILE; run->fail_entry.cpu = vcpu->cpu; run->exit_reason = KVM_EXIT_FAIL_ENTRY; return 0; @@ -781,13 +839,16 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) * producers to the new IMSIC VS-file. */ + /* Ensure HGEIE CSR bit is zero before using the new IMSIC VS-file */ + csr_clear(CSR_HGEIE, BIT(new_vsfile_hgei)); + /* Zero-out new IMSIC VS-file */ imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix); /* Update G-stage mapping for the new IMSIC VS-file */ - ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr, - new_vsfile_pa, IMSIC_MMIO_PAGE_SZ, - true, true); + ret = kvm_riscv_mmu_ioremap(kvm, vcpu->arch.aia_context.imsic_addr, + new_vsfile_pa, IMSIC_MMIO_PAGE_SZ, + true, true); if (ret) goto fail_free_vsfile_hgei; @@ -847,6 +908,10 @@ int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel, int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC; struct imsic *imsic = vcpu->arch.aia_context.imsic_state; + /* If IMSIC vCPU state not initialized then forward to user space */ + if (!imsic) + return KVM_INSN_EXIT_TO_USER_SPACE; + if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) { /* Read pending and enabled interrupt with highest priority */ topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix, @@ -895,8 +960,10 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, if (!vcpu) return -ENODEV; - isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); imsic = vcpu->arch.aia_context.imsic_state; + if (!imsic) + return -ENODEV; + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); read_lock_irqsave(&imsic->vsfile_lock, flags); @@ -936,8 +1003,11 @@ int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type) if (!vcpu) return -ENODEV; - isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); imsic = vcpu->arch.aia_context.imsic_state; + if (!imsic) + return -ENODEV; + + isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); return imsic_mrif_isel_check(imsic->nr_eix, isel); } @@ -974,7 +1044,6 @@ int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu, if (imsic->vsfile_cpu >= 0) { writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE); - kvm_vcpu_kick(vcpu); } else { eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)]; set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip); @@ -1030,7 +1099,7 @@ int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) return -EINVAL; /* Allocate IMSIC context */ - imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); + imsic = kzalloc_obj(*imsic); if (!imsic) return -ENOMEM; vcpu->arch.aia_context.imsic_state = imsic; diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c new file mode 100644 index 000000000000..d9fe8be2a151 --- /dev/null +++ b/arch/riscv/kvm/gstage.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#include <linux/bitops.h> +#include <linux/errno.h> +#include <linux/kvm_host.h> +#include <linux/module.h> +#include <linux/pgtable.h> +#include <asm/kvm_gstage.h> + +#ifdef CONFIG_64BIT +unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3; +#else +unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 2; +#endif + +#define gstage_pte_leaf(__ptep) \ + (pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)) + +static inline unsigned long gstage_pte_index(struct kvm_gstage *gstage, + gpa_t addr, u32 level) +{ + unsigned long mask; + unsigned long shift = HGATP_PAGE_SHIFT + (kvm_riscv_gstage_index_bits * level); + + if (level == gstage->pgd_levels - 1) + mask = (PTRS_PER_PTE * (1UL << kvm_riscv_gstage_pgd_xbits)) - 1; + else + mask = PTRS_PER_PTE - 1; + + return (addr >> shift) & mask; +} + +static inline unsigned long gstage_pte_page_vaddr(pte_t pte) +{ + return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte))); +} + +static int gstage_page_size_to_level(struct kvm_gstage *gstage, unsigned long page_size, + u32 *out_level) +{ + u32 i; + unsigned long psz = 1UL << 12; + + for (i = 0; i < gstage->pgd_levels; i++) { + if (page_size == (psz << (i * kvm_riscv_gstage_index_bits))) { + *out_level = i; + return 0; + } + } + + return -EINVAL; +} + +static int gstage_level_to_page_order(struct kvm_gstage *gstage, u32 level, + unsigned long *out_pgorder) +{ + if (gstage->pgd_levels < level) + return -EINVAL; + + *out_pgorder = 12 + (level * kvm_riscv_gstage_index_bits); + return 0; +} + +static int gstage_level_to_page_size(struct kvm_gstage *gstage, u32 level, + unsigned long *out_pgsize) +{ + int rc; + unsigned long page_order = PAGE_SHIFT; + + rc = gstage_level_to_page_order(gstage, level, &page_order); + if (rc) + return rc; + + *out_pgsize = BIT(page_order); + return 0; +} + +bool kvm_riscv_gstage_get_leaf(struct kvm_gstage *gstage, gpa_t addr, + pte_t **ptepp, u32 *ptep_level) +{ + pte_t *ptep; + u32 current_level = gstage->pgd_levels - 1; + + *ptep_level = current_level; + ptep = (pte_t *)gstage->pgd; + ptep = &ptep[gstage_pte_index(gstage, addr, current_level)]; + while (ptep && pte_val(ptep_get(ptep))) { + if (gstage_pte_leaf(ptep)) { + *ptep_level = current_level; + *ptepp = ptep; + return true; + } + + if (current_level) { + current_level--; + *ptep_level = current_level; + ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); + ptep = &ptep[gstage_pte_index(gstage, addr, current_level)]; + } else { + ptep = NULL; + } + } + + return false; +} + +static void gstage_tlb_flush(struct kvm_gstage *gstage, u32 level, gpa_t addr) +{ + unsigned long order = PAGE_SHIFT; + + if (gstage_level_to_page_order(gstage, level, &order)) + return; + addr &= ~(BIT(order) - 1); + + if (gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) + kvm_riscv_local_hfence_gvma_vmid_gpa(gstage->vmid, addr, BIT(order), order); + else + kvm_riscv_hfence_gvma_vmid_gpa(gstage->kvm, -1UL, 0, addr, BIT(order), order, + gstage->vmid); +} + +int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + const struct kvm_gstage_mapping *map) +{ + u32 current_level = gstage->pgd_levels - 1; + pte_t *next_ptep = (pte_t *)gstage->pgd; + pte_t *ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)]; + + if (current_level < map->level) + return -EINVAL; + + while (current_level != map->level) { + if (gstage_pte_leaf(ptep)) + return -EEXIST; + + if (!pte_val(ptep_get(ptep))) { + if (!pcache) + return -ENOMEM; + next_ptep = kvm_mmu_memory_cache_alloc(pcache); + if (!next_ptep) + return -ENOMEM; + set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), + __pgprot(_PAGE_TABLE))); + } else { + if (gstage_pte_leaf(ptep)) + return -EEXIST; + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); + } + + current_level--; + ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)]; + } + + if (pte_val(*ptep) != pte_val(map->pte)) { + set_pte(ptep, map->pte); + if (gstage_pte_leaf(ptep)) + gstage_tlb_flush(gstage, current_level, map->addr); + } + + return 0; +} + +static void kvm_riscv_gstage_update_pte_prot(struct kvm_gstage *gstage, u32 level, + gpa_t addr, pte_t *ptep, pgprot_t prot) +{ + pte_t new_pte; + + if (pgprot_val(pte_pgprot(ptep_get(ptep))) == pgprot_val(prot)) + return; + + new_pte = pfn_pte(pte_pfn(ptep_get(ptep)), prot); + new_pte = pte_mkdirty(new_pte); + + set_pte(ptep, new_pte); + + gstage_tlb_flush(gstage, level, addr); +} + +int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t gpa, phys_addr_t hpa, unsigned long page_size, + bool page_rdonly, bool page_exec, + struct kvm_gstage_mapping *out_map) +{ + bool found_leaf; + u32 ptep_level; + pgprot_t prot; + pte_t *ptep; + int ret; + + out_map->addr = gpa; + out_map->level = 0; + + ret = gstage_page_size_to_level(gstage, page_size, &out_map->level); + if (ret) + return ret; + + /* + * A RISC-V implementation can choose to either: + * 1) Update 'A' and 'D' PTE bits in hardware + * 2) Generate page fault when 'A' and/or 'D' bits are not set + * PTE so that software can update these bits. + * + * We support both options mentioned above. To achieve this, we + * always set 'A' and 'D' PTE bits at time of creating G-stage + * mapping. To support KVM dirty page logging with both options + * mentioned above, we will write-protect G-stage PTEs to track + * dirty pages. + */ + + if (page_exec) { + if (page_rdonly) + prot = PAGE_READ_EXEC; + else + prot = PAGE_WRITE_EXEC; + } else { + if (page_rdonly) + prot = PAGE_READ; + else + prot = PAGE_WRITE; + } + + found_leaf = kvm_riscv_gstage_get_leaf(gstage, gpa, &ptep, &ptep_level); + if (found_leaf) { + /* + * ptep_level is the current gstage mapping level of addr, out_map->level + * is the required mapping level during fault handling. + * + * 1) ptep_level > out_map->level + * This happens when dirty logging is enabled and huge pages are used. + * KVM must track the pages at 4K level, and split the huge mapping + * into 4K mappings. + * + * 2) ptep_level < out_map->level + * This happens when dirty logging is disabled and huge pages are used. + * The gstage is split into 4K mappings, but the out_map level is now + * back to the huge page level. Ignore the out_map level this time, and + * just update the pte prot here. Otherwise, we would fall back to mapping + * the gstage at huge page level in `kvm_riscv_gstage_set_pte`, with the + * overhead of freeing the page tables(not support now), which would slow + * down the vCPUs' performance. + * + * It is better to recover the huge page mapping in the ioctl context when + * disabling dirty logging. + * + * 3) ptep_level == out_map->level + * We already have the ptep, just update the pte prot if the pfn not change. + * There is no need to invoke `kvm_riscv_gstage_set_pte` again. + */ + if (ptep_level > out_map->level) { + kvm_riscv_gstage_split_huge(gstage, pcache, gpa, + out_map->level, true); + } else if (ALIGN_DOWN(PFN_PHYS(pte_pfn(ptep_get(ptep))), page_size) == hpa) { + kvm_riscv_gstage_update_pte_prot(gstage, ptep_level, gpa, ptep, prot); + return 0; + } + } + + out_map->pte = pfn_pte(PFN_DOWN(hpa), prot); + out_map->pte = pte_mkdirty(out_map->pte); + + return kvm_riscv_gstage_set_pte(gstage, pcache, out_map); +} + +static inline unsigned long make_child_pte(unsigned long huge_pte, int index, + unsigned long child_page_size) +{ + unsigned long child_pte = huge_pte; + unsigned long child_pfn_offset; + + /* + * The child_pte already has the base address of the huge page being + * split. So we just have to OR in the offset to the page at the next + * lower level for the given index. + */ + child_pfn_offset = index * (child_page_size / PAGE_SIZE); + child_pte |= pte_val(pfn_pte(child_pfn_offset, __pgprot(0))); + + return child_pte; +} + +int kvm_riscv_gstage_split_huge(struct kvm_gstage *gstage, + struct kvm_mmu_memory_cache *pcache, + gpa_t addr, u32 target_level, bool flush) +{ + u32 current_level = gstage->pgd_levels - 1; + pte_t *next_ptep = (pte_t *)gstage->pgd; + unsigned long huge_pte, child_pte; + unsigned long child_page_size; + pte_t *ptep; + int i, ret; + + if (!pcache) + return -ENOMEM; + + while(current_level > target_level) { + ptep = (pte_t *)&next_ptep[gstage_pte_index(gstage, addr, current_level)]; + + if (!pte_val(ptep_get(ptep))) + break; + + if (!gstage_pte_leaf(ptep)) { + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); + current_level--; + continue; + } + + huge_pte = pte_val(ptep_get(ptep)); + + ret = gstage_level_to_page_size(gstage, current_level - 1, &child_page_size); + if (ret) + return ret; + + next_ptep = kvm_mmu_memory_cache_alloc(pcache); + if (!next_ptep) + return -ENOMEM; + + for (i = 0; i < PTRS_PER_PTE; i++) { + child_pte = make_child_pte(huge_pte, i, child_page_size); + set_pte((pte_t *)&next_ptep[i], __pte(child_pte)); + } + + set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), + __pgprot(_PAGE_TABLE))); + + if (flush) + gstage_tlb_flush(gstage, current_level, addr); + + current_level--; + } + + return 0; +} + +void kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, + pte_t *ptep, u32 ptep_level, enum kvm_riscv_gstage_op op) +{ + int i, ret; + pte_t old_pte, *next_ptep; + u32 next_ptep_level; + unsigned long next_page_size, page_size; + + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); + if (ret) + return; + + WARN_ON(addr & (page_size - 1)); + + if (!pte_val(ptep_get(ptep))) + return; + + if (ptep_level && !gstage_pte_leaf(ptep)) { + next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); + next_ptep_level = ptep_level - 1; + ret = gstage_level_to_page_size(gstage, next_ptep_level, &next_page_size); + if (ret) + return; + + if (op == GSTAGE_OP_CLEAR) + set_pte(ptep, __pte(0)); + for (i = 0; i < PTRS_PER_PTE; i++) + kvm_riscv_gstage_op_pte(gstage, addr + i * next_page_size, + &next_ptep[i], next_ptep_level, op); + if (op == GSTAGE_OP_CLEAR) + put_page(virt_to_page(next_ptep)); + } else { + old_pte = *ptep; + if (op == GSTAGE_OP_CLEAR) + set_pte(ptep, __pte(0)); + else if (op == GSTAGE_OP_WP) + set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE)); + if (pte_val(*ptep) != pte_val(old_pte)) + gstage_tlb_flush(gstage, ptep_level, addr); + } +} + +void kvm_riscv_gstage_unmap_range(struct kvm_gstage *gstage, + gpa_t start, gpa_t size, bool may_block) +{ + int ret; + pte_t *ptep; + u32 ptep_level; + bool found_leaf; + unsigned long page_size; + gpa_t addr = start, end = start + size; + + while (addr < end) { + found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); + if (ret) + break; + + if (!found_leaf) + goto next; + + if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) + kvm_riscv_gstage_op_pte(gstage, addr, ptep, + ptep_level, GSTAGE_OP_CLEAR); + +next: + addr += page_size; + + /* + * If the range is too large, release the kvm->mmu_lock + * to prevent starvation and lockup detector warnings. + */ + if (!(gstage->flags & KVM_GSTAGE_FLAGS_LOCAL) && may_block && addr < end) + cond_resched_lock(&gstage->kvm->mmu_lock); + } +} + +void kvm_riscv_gstage_wp_range(struct kvm_gstage *gstage, gpa_t start, gpa_t end) +{ + int ret; + pte_t *ptep; + u32 ptep_level; + bool found_leaf; + gpa_t addr = start; + unsigned long page_size; + + while (addr < end) { + found_leaf = kvm_riscv_gstage_get_leaf(gstage, addr, &ptep, &ptep_level); + ret = gstage_level_to_page_size(gstage, ptep_level, &page_size); + if (ret) + break; + + if (!found_leaf) + goto next; + + addr = ALIGN_DOWN(addr, page_size); + kvm_riscv_gstage_op_pte(gstage, addr, ptep, + ptep_level, GSTAGE_OP_WP); +next: + addr += page_size; + } +} + +void __init kvm_riscv_gstage_mode_detect(void) +{ +#ifdef CONFIG_64BIT + /* Try Sv57x4 G-stage mode */ + csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) { + kvm_riscv_gstage_max_pgd_levels = 5; + goto done; + } + + /* Try Sv48x4 G-stage mode */ + csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { + kvm_riscv_gstage_max_pgd_levels = 4; + goto done; + } + + /* Try Sv39x4 G-stage mode */ + csr_write(CSR_HGATP, HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV39X4) { + kvm_riscv_gstage_max_pgd_levels = 3; + goto done; + } +#else /* CONFIG_32BIT */ + /* Try Sv32x4 G-stage mode */ + csr_write(CSR_HGATP, HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV32X4) { + kvm_riscv_gstage_max_pgd_levels = 2; + goto done; + } +#endif + + /* KVM depends on !HGATP_MODE_OFF */ + kvm_riscv_gstage_max_pgd_levels = 0; + +done: + csr_write(CSR_HGATP, 0); + kvm_riscv_local_hfence_gvma_all(); +} diff --git a/arch/riscv/kvm/isa.c b/arch/riscv/kvm/isa.c new file mode 100644 index 000000000000..1132d909cc25 --- /dev/null +++ b/arch/riscv/kvm/isa.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#include <linux/cpufeature.h> +#include <linux/errno.h> +#include <linux/kvm_host.h> +#include <linux/nospec.h> +#include <linux/pgtable.h> +#include <asm/kvm_isa.h> +#include <asm/vector.h> + +#define KVM_ISA_EXT_ARR(ext) \ +[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext + +/* Mapping between KVM ISA Extension ID & guest ISA extension ID */ +static const unsigned long kvm_isa_ext_arr[] = { + /* Single letter extensions (alphabetically sorted) */ + [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, + [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, + [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, + [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, + [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, + [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, + [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, + [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, + /* Multi letter extensions (alphabetically sorted) */ + KVM_ISA_EXT_ARR(SMNPM), + KVM_ISA_EXT_ARR(SMSTATEEN), + KVM_ISA_EXT_ARR(SSAIA), + KVM_ISA_EXT_ARR(SSCOFPMF), + KVM_ISA_EXT_ARR(SSNPM), + KVM_ISA_EXT_ARR(SSTC), + KVM_ISA_EXT_ARR(SVADE), + KVM_ISA_EXT_ARR(SVADU), + KVM_ISA_EXT_ARR(SVINVAL), + KVM_ISA_EXT_ARR(SVNAPOT), + KVM_ISA_EXT_ARR(SVPBMT), + KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZAAMO), + KVM_ISA_EXT_ARR(ZABHA), + KVM_ISA_EXT_ARR(ZACAS), + KVM_ISA_EXT_ARR(ZALASR), + KVM_ISA_EXT_ARR(ZALRSC), + KVM_ISA_EXT_ARR(ZAWRS), + KVM_ISA_EXT_ARR(ZBA), + KVM_ISA_EXT_ARR(ZBB), + KVM_ISA_EXT_ARR(ZBC), + KVM_ISA_EXT_ARR(ZBKB), + KVM_ISA_EXT_ARR(ZBKC), + KVM_ISA_EXT_ARR(ZBKX), + KVM_ISA_EXT_ARR(ZBS), + KVM_ISA_EXT_ARR(ZCA), + KVM_ISA_EXT_ARR(ZCB), + KVM_ISA_EXT_ARR(ZCD), + KVM_ISA_EXT_ARR(ZCF), + KVM_ISA_EXT_ARR(ZCLSD), + KVM_ISA_EXT_ARR(ZCMOP), + KVM_ISA_EXT_ARR(ZFA), + KVM_ISA_EXT_ARR(ZFBFMIN), + KVM_ISA_EXT_ARR(ZFH), + KVM_ISA_EXT_ARR(ZFHMIN), + KVM_ISA_EXT_ARR(ZICBOM), + KVM_ISA_EXT_ARR(ZICBOP), + KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCRSE), + KVM_ISA_EXT_ARR(ZICNTR), + KVM_ISA_EXT_ARR(ZICOND), + KVM_ISA_EXT_ARR(ZICSR), + KVM_ISA_EXT_ARR(ZIFENCEI), + KVM_ISA_EXT_ARR(ZIHINTNTL), + KVM_ISA_EXT_ARR(ZIHINTPAUSE), + KVM_ISA_EXT_ARR(ZIHPM), + KVM_ISA_EXT_ARR(ZILSD), + KVM_ISA_EXT_ARR(ZIMOP), + KVM_ISA_EXT_ARR(ZKND), + KVM_ISA_EXT_ARR(ZKNE), + KVM_ISA_EXT_ARR(ZKNH), + KVM_ISA_EXT_ARR(ZKR), + KVM_ISA_EXT_ARR(ZKSED), + KVM_ISA_EXT_ARR(ZKSH), + KVM_ISA_EXT_ARR(ZKT), + KVM_ISA_EXT_ARR(ZTSO), + KVM_ISA_EXT_ARR(ZVBB), + KVM_ISA_EXT_ARR(ZVBC), + KVM_ISA_EXT_ARR(ZVFBFMIN), + KVM_ISA_EXT_ARR(ZVFBFWMA), + KVM_ISA_EXT_ARR(ZVFH), + KVM_ISA_EXT_ARR(ZVFHMIN), + KVM_ISA_EXT_ARR(ZVKB), + KVM_ISA_EXT_ARR(ZVKG), + KVM_ISA_EXT_ARR(ZVKNED), + KVM_ISA_EXT_ARR(ZVKNHA), + KVM_ISA_EXT_ARR(ZVKNHB), + KVM_ISA_EXT_ARR(ZVKSED), + KVM_ISA_EXT_ARR(ZVKSH), + KVM_ISA_EXT_ARR(ZVKT), +}; + +unsigned long kvm_riscv_base2isa_ext(unsigned long base_ext) +{ + unsigned long i; + + for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { + if (kvm_isa_ext_arr[i] == base_ext) + return i; + } + + return KVM_RISCV_ISA_EXT_MAX; +} + +int __kvm_riscv_isa_check_host(unsigned long kvm_ext, unsigned long *base_ext) +{ + unsigned long host_ext; + + if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX || + kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr)) + return -ENOENT; + + kvm_ext = array_index_nospec(kvm_ext, ARRAY_SIZE(kvm_isa_ext_arr)); + switch (kvm_isa_ext_arr[kvm_ext]) { + case RISCV_ISA_EXT_SMNPM: + /* + * Pointer masking effective in (H)S-mode is provided by the + * Smnpm extension, so that extension is reported to the guest, + * even though the CSR bits for configuring VS-mode pointer + * masking on the host side are part of the Ssnpm extension. + */ + host_ext = RISCV_ISA_EXT_SSNPM; + break; + default: + host_ext = kvm_isa_ext_arr[kvm_ext]; + break; + } + + if (!__riscv_isa_extension_available(NULL, host_ext)) + return -ENOENT; + + if (base_ext) + *base_ext = kvm_isa_ext_arr[kvm_ext]; + + return 0; +} + +bool kvm_riscv_isa_enable_allowed(unsigned long ext) +{ + switch (ext) { + case KVM_RISCV_ISA_EXT_H: + return false; + case KVM_RISCV_ISA_EXT_SSCOFPMF: + /* Sscofpmf depends on interrupt filtering defined in ssaia */ + return !kvm_riscv_isa_check_host(SSAIA); + case KVM_RISCV_ISA_EXT_SVADU: + /* + * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. + * Guest OS can use Svadu only when host OS enable Svadu. + */ + return arch_has_hw_pte_young(); + case KVM_RISCV_ISA_EXT_V: + return riscv_v_vstate_ctrl_user_allowed(); + default: + break; + } + + return true; +} + +bool kvm_riscv_isa_disable_allowed(unsigned long ext) +{ + switch (ext) { + /* Extensions which don't have any mechanism to disable */ + case KVM_RISCV_ISA_EXT_A: + case KVM_RISCV_ISA_EXT_C: + case KVM_RISCV_ISA_EXT_I: + case KVM_RISCV_ISA_EXT_M: + /* There is not architectural config bit to disable sscofpmf completely */ + case KVM_RISCV_ISA_EXT_SSCOFPMF: + case KVM_RISCV_ISA_EXT_SSNPM: + case KVM_RISCV_ISA_EXT_SSTC: + case KVM_RISCV_ISA_EXT_SVINVAL: + case KVM_RISCV_ISA_EXT_SVNAPOT: + case KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_RISCV_ISA_EXT_ZAAMO: + case KVM_RISCV_ISA_EXT_ZABHA: + case KVM_RISCV_ISA_EXT_ZACAS: + case KVM_RISCV_ISA_EXT_ZALASR: + case KVM_RISCV_ISA_EXT_ZALRSC: + case KVM_RISCV_ISA_EXT_ZAWRS: + case KVM_RISCV_ISA_EXT_ZBA: + case KVM_RISCV_ISA_EXT_ZBB: + case KVM_RISCV_ISA_EXT_ZBC: + case KVM_RISCV_ISA_EXT_ZBKB: + case KVM_RISCV_ISA_EXT_ZBKC: + case KVM_RISCV_ISA_EXT_ZBKX: + case KVM_RISCV_ISA_EXT_ZBS: + case KVM_RISCV_ISA_EXT_ZCA: + case KVM_RISCV_ISA_EXT_ZCB: + case KVM_RISCV_ISA_EXT_ZCD: + case KVM_RISCV_ISA_EXT_ZCF: + case KVM_RISCV_ISA_EXT_ZCMOP: + case KVM_RISCV_ISA_EXT_ZFA: + case KVM_RISCV_ISA_EXT_ZFBFMIN: + case KVM_RISCV_ISA_EXT_ZFH: + case KVM_RISCV_ISA_EXT_ZFHMIN: + case KVM_RISCV_ISA_EXT_ZICBOP: + case KVM_RISCV_ISA_EXT_ZICCRSE: + case KVM_RISCV_ISA_EXT_ZICNTR: + case KVM_RISCV_ISA_EXT_ZICOND: + case KVM_RISCV_ISA_EXT_ZICSR: + case KVM_RISCV_ISA_EXT_ZIFENCEI: + case KVM_RISCV_ISA_EXT_ZIHINTNTL: + case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: + case KVM_RISCV_ISA_EXT_ZIHPM: + case KVM_RISCV_ISA_EXT_ZIMOP: + case KVM_RISCV_ISA_EXT_ZKND: + case KVM_RISCV_ISA_EXT_ZKNE: + case KVM_RISCV_ISA_EXT_ZKNH: + case KVM_RISCV_ISA_EXT_ZKR: + case KVM_RISCV_ISA_EXT_ZKSED: + case KVM_RISCV_ISA_EXT_ZKSH: + case KVM_RISCV_ISA_EXT_ZKT: + case KVM_RISCV_ISA_EXT_ZTSO: + case KVM_RISCV_ISA_EXT_ZVBB: + case KVM_RISCV_ISA_EXT_ZVBC: + case KVM_RISCV_ISA_EXT_ZVFBFMIN: + case KVM_RISCV_ISA_EXT_ZVFBFWMA: + case KVM_RISCV_ISA_EXT_ZVFH: + case KVM_RISCV_ISA_EXT_ZVFHMIN: + case KVM_RISCV_ISA_EXT_ZVKB: + case KVM_RISCV_ISA_EXT_ZVKG: + case KVM_RISCV_ISA_EXT_ZVKNED: + case KVM_RISCV_ISA_EXT_ZVKNHA: + case KVM_RISCV_ISA_EXT_ZVKNHB: + case KVM_RISCV_ISA_EXT_ZVKSED: + case KVM_RISCV_ISA_EXT_ZVKSH: + case KVM_RISCV_ISA_EXT_ZVKT: + return false; + /* Extensions which can be disabled using Smstateen */ + case KVM_RISCV_ISA_EXT_SSAIA: + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN); + case KVM_RISCV_ISA_EXT_SVADE: + /* + * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. + * Svade can't be disabled unless we support Svadu. + */ + return arch_has_hw_pte_young(); + default: + break; + } + + return true; +} diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 1fa8be5ee509..cb8a65273c1f 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -11,9 +11,22 @@ #include <linux/module.h> #include <linux/kvm_host.h> #include <asm/cpufeature.h> +#include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> #include <asm/sbi.h> +DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); + +static void kvm_riscv_setup_vendor_features(void) +{ + /* Andes AX66: split two-stage TLBs */ + if (riscv_cached_mvendorid(0) == ANDES_VENDOR_ID && + (riscv_cached_marchid(0) & 0xFFFF) == 0x8A66) { + static_branch_enable(&kvm_riscv_vsstage_tlb_no_gpa); + kvm_info("VS-stage TLB does not cache guest physical address and VMID\n"); + } +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -28,8 +41,8 @@ int kvm_arch_enable_virtualization_cpu(void) if (rc) return rc; - csr_write(CSR_HEDELEG, KVM_HEDELEG_DEFAULT); - csr_write(CSR_HIDELEG, KVM_HIDELEG_DEFAULT); + csr_write(CSR_HEDELEG, 0); + csr_write(CSR_HIDELEG, 0); /* VS should access only the time counter directly. Everything else should trap */ csr_write(CSR_HCOUNTEREN, 0x02); @@ -92,6 +105,23 @@ static int __init riscv_kvm_init(void) return rc; kvm_riscv_gstage_mode_detect(); + switch (kvm_riscv_gstage_max_pgd_levels) { + case 2: + str = "Sv32x4"; + break; + case 3: + str = "Sv39x4"; + break; + case 4: + str = "Sv48x4"; + break; + case 5: + str = "Sv57x4"; + break; + default: + kvm_riscv_nacl_exit(); + return -ENODEV; + } kvm_riscv_gstage_vmid_detect(); @@ -134,23 +164,7 @@ static int __init riscv_kvm_init(void) (rc) ? slist : "no features"); } - switch (kvm_riscv_gstage_mode()) { - case HGATP_MODE_SV32X4: - str = "Sv32x4"; - break; - case HGATP_MODE_SV39X4: - str = "Sv39x4"; - break; - case HGATP_MODE_SV48X4: - str = "Sv48x4"; - break; - case HGATP_MODE_SV57X4: - str = "Sv57x4"; - break; - default: - return -ENODEV; - } - kvm_info("using %s G-stage page table format\n", str); + kvm_info("highest G-stage page table mode is %s\n", str); kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits()); @@ -158,7 +172,9 @@ static int __init riscv_kvm_init(void) kvm_info("AIA available with %d guest external interrupts\n", kvm_riscv_aia_nr_hgei); - kvm_register_perf_callbacks(NULL); + kvm_riscv_setup_vendor_features(); + + kvm_register_perf_callbacks(); rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (rc) { @@ -172,8 +188,8 @@ module_init(riscv_kvm_init); static void __exit riscv_kvm_exit(void) { - kvm_riscv_teardown(); - kvm_exit(); + + kvm_riscv_teardown(); } module_exit(riscv_kvm_exit); diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 1087ea74567b..2d3def024270 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -6,371 +6,67 @@ * Anup Patel <anup.patel@wdc.com> */ -#include <linux/bitops.h> #include <linux/errno.h> -#include <linux/err.h> #include <linux/hugetlb.h> #include <linux/module.h> #include <linux/uaccess.h> #include <linux/vmalloc.h> #include <linux/kvm_host.h> #include <linux/sched/signal.h> +#include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> -#include <asm/page.h> -#include <asm/pgtable.h> - -#ifdef CONFIG_64BIT -static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); -static unsigned long gstage_pgd_levels __ro_after_init = 3; -#define gstage_index_bits 9 -#else -static unsigned long gstage_mode __ro_after_init = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); -static unsigned long gstage_pgd_levels __ro_after_init = 2; -#define gstage_index_bits 10 -#endif - -#define gstage_pgd_xbits 2 -#define gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + gstage_pgd_xbits)) -#define gstage_gpa_bits (HGATP_PAGE_SHIFT + \ - (gstage_pgd_levels * gstage_index_bits) + \ - gstage_pgd_xbits) -#define gstage_gpa_size ((gpa_t)(1ULL << gstage_gpa_bits)) - -#define gstage_pte_leaf(__ptep) \ - (pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)) - -static inline unsigned long gstage_pte_index(gpa_t addr, u32 level) -{ - unsigned long mask; - unsigned long shift = HGATP_PAGE_SHIFT + (gstage_index_bits * level); - - if (level == (gstage_pgd_levels - 1)) - mask = (PTRS_PER_PTE * (1UL << gstage_pgd_xbits)) - 1; - else - mask = PTRS_PER_PTE - 1; - return (addr >> shift) & mask; -} - -static inline unsigned long gstage_pte_page_vaddr(pte_t pte) -{ - return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte))); -} - -static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) -{ - u32 i; - unsigned long psz = 1UL << 12; - - for (i = 0; i < gstage_pgd_levels; i++) { - if (page_size == (psz << (i * gstage_index_bits))) { - *out_level = i; - return 0; - } - } - - return -EINVAL; -} - -static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder) -{ - if (gstage_pgd_levels < level) - return -EINVAL; - - *out_pgorder = 12 + (level * gstage_index_bits); - return 0; -} - -static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize) -{ - int rc; - unsigned long page_order = PAGE_SHIFT; - - rc = gstage_level_to_page_order(level, &page_order); - if (rc) - return rc; - - *out_pgsize = BIT(page_order); - return 0; -} - -static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, - pte_t **ptepp, u32 *ptep_level) -{ - pte_t *ptep; - u32 current_level = gstage_pgd_levels - 1; - - *ptep_level = current_level; - ptep = (pte_t *)kvm->arch.pgd; - ptep = &ptep[gstage_pte_index(addr, current_level)]; - while (ptep && pte_val(ptep_get(ptep))) { - if (gstage_pte_leaf(ptep)) { - *ptep_level = current_level; - *ptepp = ptep; - return true; - } - - if (current_level) { - current_level--; - *ptep_level = current_level; - ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); - ptep = &ptep[gstage_pte_index(addr, current_level)]; - } else { - ptep = NULL; - } - } - - return false; -} - -static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr) -{ - unsigned long order = PAGE_SHIFT; - - if (gstage_level_to_page_order(level, &order)) - return; - addr &= ~(BIT(order) - 1); - - kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, addr, BIT(order), order); -} - -static int gstage_set_pte(struct kvm *kvm, u32 level, - struct kvm_mmu_memory_cache *pcache, - gpa_t addr, const pte_t *new_pte) -{ - u32 current_level = gstage_pgd_levels - 1; - pte_t *next_ptep = (pte_t *)kvm->arch.pgd; - pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)]; - - if (current_level < level) - return -EINVAL; - - while (current_level != level) { - if (gstage_pte_leaf(ptep)) - return -EEXIST; - - if (!pte_val(ptep_get(ptep))) { - if (!pcache) - return -ENOMEM; - next_ptep = kvm_mmu_memory_cache_alloc(pcache); - if (!next_ptep) - return -ENOMEM; - set_pte(ptep, pfn_pte(PFN_DOWN(__pa(next_ptep)), - __pgprot(_PAGE_TABLE))); - } else { - if (gstage_pte_leaf(ptep)) - return -EEXIST; - next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); - } - - current_level--; - ptep = &next_ptep[gstage_pte_index(addr, current_level)]; - } - - set_pte(ptep, *new_pte); - if (gstage_pte_leaf(ptep)) - gstage_remote_tlb_flush(kvm, current_level, addr); - - return 0; -} - -static int gstage_map_page(struct kvm *kvm, - struct kvm_mmu_memory_cache *pcache, - gpa_t gpa, phys_addr_t hpa, - unsigned long page_size, - bool page_rdonly, bool page_exec) -{ - int ret; - u32 level = 0; - pte_t new_pte; - pgprot_t prot; - - ret = gstage_page_size_to_level(page_size, &level); - if (ret) - return ret; - - /* - * A RISC-V implementation can choose to either: - * 1) Update 'A' and 'D' PTE bits in hardware - * 2) Generate page fault when 'A' and/or 'D' bits are not set - * PTE so that software can update these bits. - * - * We support both options mentioned above. To achieve this, we - * always set 'A' and 'D' PTE bits at time of creating G-stage - * mapping. To support KVM dirty page logging with both options - * mentioned above, we will write-protect G-stage PTEs to track - * dirty pages. - */ - - if (page_exec) { - if (page_rdonly) - prot = PAGE_READ_EXEC; - else - prot = PAGE_WRITE_EXEC; - } else { - if (page_rdonly) - prot = PAGE_READ; - else - prot = PAGE_WRITE; - } - new_pte = pfn_pte(PFN_DOWN(hpa), prot); - new_pte = pte_mkdirty(new_pte); - - return gstage_set_pte(kvm, level, pcache, gpa, &new_pte); -} - -enum gstage_op { - GSTAGE_OP_NOP = 0, /* Nothing */ - GSTAGE_OP_CLEAR, /* Clear/Unmap */ - GSTAGE_OP_WP, /* Write-protect */ -}; - -static void gstage_op_pte(struct kvm *kvm, gpa_t addr, - pte_t *ptep, u32 ptep_level, enum gstage_op op) -{ - int i, ret; - pte_t *next_ptep; - u32 next_ptep_level; - unsigned long next_page_size, page_size; - - ret = gstage_level_to_page_size(ptep_level, &page_size); - if (ret) - return; - - BUG_ON(addr & (page_size - 1)); - - if (!pte_val(ptep_get(ptep))) - return; - - if (ptep_level && !gstage_pte_leaf(ptep)) { - next_ptep = (pte_t *)gstage_pte_page_vaddr(ptep_get(ptep)); - next_ptep_level = ptep_level - 1; - ret = gstage_level_to_page_size(next_ptep_level, - &next_page_size); - if (ret) - return; - - if (op == GSTAGE_OP_CLEAR) - set_pte(ptep, __pte(0)); - for (i = 0; i < PTRS_PER_PTE; i++) - gstage_op_pte(kvm, addr + i * next_page_size, - &next_ptep[i], next_ptep_level, op); - if (op == GSTAGE_OP_CLEAR) - put_page(virt_to_page(next_ptep)); - } else { - if (op == GSTAGE_OP_CLEAR) - set_pte(ptep, __pte(0)); - else if (op == GSTAGE_OP_WP) - set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE)); - gstage_remote_tlb_flush(kvm, ptep_level, addr); - } -} - -static void gstage_unmap_range(struct kvm *kvm, gpa_t start, - gpa_t size, bool may_block) -{ - int ret; - pte_t *ptep; - u32 ptep_level; - bool found_leaf; - unsigned long page_size; - gpa_t addr = start, end = start + size; - - while (addr < end) { - found_leaf = gstage_get_leaf_entry(kvm, addr, - &ptep, &ptep_level); - ret = gstage_level_to_page_size(ptep_level, &page_size); - if (ret) - break; - - if (!found_leaf) - goto next; - - if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - gstage_op_pte(kvm, addr, ptep, - ptep_level, GSTAGE_OP_CLEAR); - -next: - addr += page_size; - - /* - * If the range is too large, release the kvm->mmu_lock - * to prevent starvation and lockup detector warnings. - */ - if (may_block && addr < end) - cond_resched_lock(&kvm->mmu_lock); - } -} - -static void gstage_wp_range(struct kvm *kvm, gpa_t start, gpa_t end) -{ - int ret; - pte_t *ptep; - u32 ptep_level; - bool found_leaf; - gpa_t addr = start; - unsigned long page_size; - - while (addr < end) { - found_leaf = gstage_get_leaf_entry(kvm, addr, - &ptep, &ptep_level); - ret = gstage_level_to_page_size(ptep_level, &page_size); - if (ret) - break; - - if (!found_leaf) - goto next; - - if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - gstage_op_pte(kvm, addr, ptep, - ptep_level, GSTAGE_OP_WP); - -next: - addr += page_size; - } -} - -static void gstage_wp_memory_region(struct kvm *kvm, int slot) +static void mmu_wp_memory_region(struct kvm *kvm, int slot) { struct kvm_memslots *slots = kvm_memslots(kvm); struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + struct kvm_gstage gstage; + + kvm_riscv_gstage_init(&gstage, kvm); spin_lock(&kvm->mmu_lock); - gstage_wp_range(kvm, start, end); + kvm_riscv_gstage_wp_range(&gstage, start, end); spin_unlock(&kvm->mmu_lock); - kvm_flush_remote_tlbs(kvm); + kvm_flush_remote_tlbs_memslot(kvm, memslot); } -int kvm_riscv_gstage_ioremap(struct kvm *kvm, gpa_t gpa, - phys_addr_t hpa, unsigned long size, - bool writable, bool in_atomic) +int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, + unsigned long size, bool writable, bool in_atomic) { - pte_t pte; int ret = 0; + pgprot_t prot; unsigned long pfn; phys_addr_t addr, end; struct kvm_mmu_memory_cache pcache = { .gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0, .gfp_zero = __GFP_ZERO, }; + struct kvm_gstage_mapping map; + struct kvm_gstage gstage; + + kvm_riscv_gstage_init(&gstage, kvm); end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK; pfn = __phys_to_pfn(hpa); + prot = pgprot_noncached(PAGE_WRITE); for (addr = gpa; addr < end; addr += PAGE_SIZE) { - pte = pfn_pte(pfn, PAGE_KERNEL_IO); + map.addr = addr; + map.pte = pfn_pte(pfn, prot); + map.pte = pte_mkdirty(map.pte); + map.level = 0; if (!writable) - pte = pte_wrprotect(pte); + map.pte = pte_wrprotect(map.pte); - ret = kvm_mmu_topup_memory_cache(&pcache, gstage_pgd_levels); + ret = kvm_mmu_topup_memory_cache(&pcache, kvm->arch.pgd_levels); if (ret) goto out; spin_lock(&kvm->mmu_lock); - ret = gstage_set_pte(kvm, 0, &pcache, addr, &pte); + ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map); spin_unlock(&kvm->mmu_lock); if (ret) goto out; @@ -383,10 +79,14 @@ out: return ret; } -void kvm_riscv_gstage_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) +void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size) { + struct kvm_gstage gstage; + + kvm_riscv_gstage_init(&gstage, kvm); + spin_lock(&kvm->mmu_lock); - gstage_unmap_range(kvm, gpa, size, false); + kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); spin_unlock(&kvm->mmu_lock); } @@ -398,8 +98,11 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, phys_addr_t base_gfn = slot->base_gfn + gfn_offset; phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + struct kvm_gstage gstage; + + kvm_riscv_gstage_init(&gstage, kvm); - gstage_wp_range(kvm, start, end); + kvm_riscv_gstage_wp_range(&gstage, start, end); } void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) @@ -416,7 +119,7 @@ void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - kvm_riscv_gstage_free_pgd(kvm); + kvm_riscv_mmu_free_pgd(kvm); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, @@ -424,9 +127,12 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, { gpa_t gpa = slot->base_gfn << PAGE_SHIFT; phys_addr_t size = slot->npages << PAGE_SHIFT; + struct kvm_gstage gstage; + + kvm_riscv_gstage_init(&gstage, kvm); spin_lock(&kvm->mmu_lock); - gstage_unmap_range(kvm, gpa, size, false); + kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false); spin_unlock(&kvm->mmu_lock); } @@ -440,8 +146,11 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * allocated dirty_bitmap[], dirty pages will be tracked while * the memory slot is write protected. */ - if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) - gstage_wp_memory_region(kvm, new->id); + if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) { + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + return; + mmu_wp_memory_region(kvm, new->id); + } } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -450,7 +159,6 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, enum kvm_mr_change change) { hva_t hva, reg_end, size; - gpa_t base_gpa; bool writable; int ret = 0; @@ -463,21 +171,19 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, * space addressable by the KVM guest GPA space. */ if ((new->base_gfn + new->npages) >= - (gstage_gpa_size >> PAGE_SHIFT)) + kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels) >> PAGE_SHIFT) return -EFAULT; hva = new->userspace_addr; size = new->npages << PAGE_SHIFT; reg_end = hva + size; - base_gpa = new->base_gfn << PAGE_SHIFT; writable = !(new->flags & KVM_MEM_READONLY); mmap_read_lock(current->mm); /* * A memory region could potentially cover multiple VMAs, and - * any holes between them, so iterate over all of them to find - * out if we can map any of them right now. + * any holes between them, so iterate over all of them. * * +--------------------------------------------+ * +---------------+----------------+ +----------------+ @@ -487,10 +193,11 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, * +--------------------------------------------+ */ do { - struct vm_area_struct *vma = find_vma(current->mm, hva); - hva_t vm_start, vm_end; + struct vm_area_struct *vma; + hva_t vm_end; - if (!vma || vma->vm_start >= reg_end) + vma = find_vma_intersection(current->mm, hva, reg_end); + if (!vma) break; /* @@ -503,37 +210,18 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, } /* Take the intersection of this VMA with the memory region */ - vm_start = max(hva, vma->vm_start); vm_end = min(reg_end, vma->vm_end); if (vma->vm_flags & VM_PFNMAP) { - gpa_t gpa = base_gpa + (vm_start - hva); - phys_addr_t pa; - - pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; - pa += vm_start - vma->vm_start; - /* IO region dirty page logging not allowed */ if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { ret = -EINVAL; goto out; } - - ret = kvm_riscv_gstage_ioremap(kvm, gpa, pa, - vm_end - vm_start, - writable, false); - if (ret) - break; } hva = vm_end; } while (hva < reg_end); - if (change == KVM_MR_FLAGS_ONLY) - goto out; - - if (ret) - kvm_riscv_gstage_iounmap(kvm, base_gpa, size); - out: mmap_read_unlock(current->mm); return ret; @@ -541,12 +229,19 @@ out: bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { + struct kvm_gstage gstage; + bool mmu_locked; + if (!kvm->arch.pgd) return false; - gstage_unmap_range(kvm, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + kvm_riscv_gstage_init(&gstage, kvm); + mmu_locked = spin_trylock(&kvm->mmu_lock); + kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT, + range->may_block); + if (mmu_locked) + spin_unlock(&kvm->mmu_lock); return false; } @@ -555,14 +250,16 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) pte_t *ptep; u32 ptep_level = 0; u64 size = (range->end - range->start) << PAGE_SHIFT; + struct kvm_gstage gstage; if (!kvm->arch.pgd) return false; WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); - if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT, - &ptep, &ptep_level)) + kvm_riscv_gstage_init(&gstage, kvm); + if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, + &ptep, &ptep_level)) return false; return ptep_test_and_clear_young(NULL, 0, ptep); @@ -573,22 +270,160 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) pte_t *ptep; u32 ptep_level = 0; u64 size = (range->end - range->start) << PAGE_SHIFT; + struct kvm_gstage gstage; if (!kvm->arch.pgd) return false; WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); - if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT, - &ptep, &ptep_level)) + kvm_riscv_gstage_init(&gstage, kvm); + if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT, + &ptep, &ptep_level)) return false; return pte_young(ptep_get(ptep)); } -int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, - struct kvm_memory_slot *memslot, - gpa_t gpa, unsigned long hva, bool is_write) +static bool fault_supports_gstage_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva) +{ + hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; + size_t size; + + size = memslot->npages * PAGE_SIZE; + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + gpa_start = memslot->base_gfn << PAGE_SHIFT; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD for userspace and GPA cannot be mapped with g-stage + * PMD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh vs-stage block | vs-stage block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SHIFT: + * +---+--------------------+--------------------+-----+ + * |abc|def g-stage block | g-stage block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those g-stage blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (PMD_SIZE - 1)) != (uaddr_start & (PMD_SIZE - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva >= ALIGN(uaddr_start, PMD_SIZE)) && (hva < ALIGN_DOWN(uaddr_end, PMD_SIZE)); +} + +static int get_hva_mapping_size(struct kvm *kvm, + unsigned long hva) +{ + int size = PAGE_SIZE; + unsigned long flags; + pgd_t pgd; + p4d_t p4d; + pud_t pud; + pmd_t pmd; + + /* + * Disable IRQs to prevent concurrent tear down of host page tables, + * e.g. if the primary MMU promotes a P*D to a huge page and then frees + * the original page table. + */ + local_irq_save(flags); + + /* + * Read each entry once. As above, a non-leaf entry can be promoted to + * a huge page _during_ this walk. Re-reading the entry could send the + * walk into the weeks, e.g. p*d_leaf() returns false (sees the old + * value) and then p*d_offset() walks into the target huge page instead + * of the old page table (sees the new value). + */ + pgd = pgdp_get(pgd_offset(kvm->mm, hva)); + if (pgd_none(pgd)) + goto out; + + p4d = p4dp_get(p4d_offset(&pgd, hva)); + if (p4d_none(p4d) || !p4d_present(p4d)) + goto out; + + pud = pudp_get(pud_offset(&p4d, hva)); + if (pud_none(pud) || !pud_present(pud)) + goto out; + + if (pud_leaf(pud)) { + size = PUD_SIZE; + goto out; + } + + pmd = pmdp_get(pmd_offset(&pud, hva)); + if (pmd_none(pmd) || !pmd_present(pmd)) + goto out; + + if (pmd_leaf(pmd)) + size = PMD_SIZE; + +out: + local_irq_restore(flags); + return size; +} + +static unsigned long transparent_hugepage_adjust(struct kvm *kvm, + struct kvm_memory_slot *memslot, + unsigned long hva, + kvm_pfn_t *hfnp, gpa_t *gpa) +{ + kvm_pfn_t hfn = *hfnp; + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and GPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (fault_supports_gstage_huge_mapping(memslot, hva)) { + int sz; + + sz = get_hva_mapping_size(kvm, hva); + if (sz < PMD_SIZE) + return sz; + + *gpa &= PMD_MASK; + hfn &= ~(PTRS_PER_PMD - 1); + *hfnp = hfn; + + return PMD_SIZE; + } + + return PAGE_SIZE; +} + +int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, + gpa_t gpa, unsigned long hva, bool is_write, + struct kvm_gstage_mapping *out_map) { int ret; kvm_pfn_t hfn; @@ -601,10 +436,16 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, bool logging = (memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY)) ? true : false; unsigned long vma_pagesize, mmu_seq; + struct kvm_gstage gstage; struct page *page; + kvm_riscv_gstage_init(&gstage, kvm); + + /* Setup initial state of output mapping */ + memset(out_map, 0, sizeof(*out_map)); + /* We need minimum second+third level pages */ - ret = kvm_mmu_topup_memory_cache(pcache, gstage_pgd_levels); + ret = kvm_mmu_topup_memory_cache(pcache, kvm->arch.pgd_levels); if (ret) { kvm_err("Failed to topup G-stage cache\n"); return ret; @@ -648,7 +489,8 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, return -EFAULT; } - hfn = kvm_faultin_pfn(vcpu, gfn, is_write, &writable, &page); + hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0, + &writable, &page); if (hfn == KVM_PFN_ERR_HWPOISON) { send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva, vma_pageshift, current); @@ -669,13 +511,17 @@ int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, if (mmu_invalidate_retry(kvm, mmu_seq)) goto out_unlock; + /* Check if we are backed by a THP and thus use block mapping if possible */ + if (!logging && (vma_pagesize == PAGE_SIZE)) + vma_pagesize = transparent_hugepage_adjust(kvm, memslot, hva, &hfn, &gpa); + if (writable) { - mark_page_dirty(kvm, gfn); - ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, - vma_pagesize, false, true); + mark_page_dirty_in_slot(kvm, memslot, gfn); + ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, + vma_pagesize, false, true, out_map); } else { - ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, - vma_pagesize, true, true); + ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT, + vma_pagesize, true, true, out_map); } if (ret) @@ -687,7 +533,7 @@ out_unlock: return ret; } -int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm) +int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) { struct page *pgd_page; @@ -697,76 +543,48 @@ int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm) } pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, - get_order(gstage_pgd_size)); + get_order(kvm_riscv_gstage_pgd_size)); if (!pgd_page) return -ENOMEM; kvm->arch.pgd = page_to_virt(pgd_page); kvm->arch.pgd_phys = page_to_phys(pgd_page); + kvm->arch.pgd_levels = kvm_riscv_gstage_max_pgd_levels; return 0; } -void kvm_riscv_gstage_free_pgd(struct kvm *kvm) +void kvm_riscv_mmu_free_pgd(struct kvm *kvm) { + struct kvm_gstage gstage; void *pgd = NULL; spin_lock(&kvm->mmu_lock); if (kvm->arch.pgd) { - gstage_unmap_range(kvm, 0UL, gstage_gpa_size, false); + kvm_riscv_gstage_init(&gstage, kvm); + kvm_riscv_gstage_unmap_range(&gstage, 0UL, + kvm_riscv_gstage_gpa_size(kvm->arch.pgd_levels), false); pgd = READ_ONCE(kvm->arch.pgd); kvm->arch.pgd = NULL; kvm->arch.pgd_phys = 0; + kvm->arch.pgd_levels = 0; } spin_unlock(&kvm->mmu_lock); if (pgd) - free_pages((unsigned long)pgd, get_order(gstage_pgd_size)); + free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size)); } -void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu) +void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) { - unsigned long hgatp = gstage_mode; - struct kvm_arch *k = &vcpu->kvm->arch; + struct kvm_arch *ka = &vcpu->kvm->arch; + unsigned long hgatp = kvm_riscv_gstage_mode(ka->pgd_levels) + << HGATP_MODE_SHIFT; - hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; - hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; + hgatp |= (READ_ONCE(ka->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; + hgatp |= (ka->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; ncsr_write(CSR_HGATP, hgatp); if (!kvm_riscv_gstage_vmid_bits()) kvm_riscv_local_hfence_gvma_all(); } - -void __init kvm_riscv_gstage_mode_detect(void) -{ -#ifdef CONFIG_64BIT - /* Try Sv57x4 G-stage mode */ - csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); - if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) { - gstage_mode = (HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); - gstage_pgd_levels = 5; - goto skip_sv48x4_test; - } - - /* Try Sv48x4 G-stage mode */ - csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); - if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { - gstage_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); - gstage_pgd_levels = 4; - } -skip_sv48x4_test: - - csr_write(CSR_HGATP, 0); - kvm_riscv_local_hfence_gvma_all(); -#endif -} - -unsigned long __init kvm_riscv_gstage_mode(void) -{ - return gstage_mode >> HGATP_MODE_SHIFT; -} - -int kvm_riscv_gstage_gpa_bits(void) -{ - return gstage_gpa_bits; -} diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 2f91ea5f8493..993b25ea94d6 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -15,6 +15,8 @@ #include <asm/cpufeature.h> #include <asm/insn-def.h> #include <asm/kvm_nacl.h> +#include <asm/kvm_tlb.h> +#include <asm/kvm_vmid.h> #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) @@ -177,6 +179,13 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); kvm_riscv_local_hfence_gvma_vmid_all(vmid); + + /* + * Flush VS-stage TLB entries for implementation where VS-stage + * TLB does not cache guest physical address and VMID. + */ + if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa)) + kvm_riscv_local_hfence_vvma_all(vmid); } void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) @@ -185,7 +194,7 @@ void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) local_flush_icache_all(); } -void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu) +void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu) { struct kvm_vmid *v = &vcpu->kvm->arch.vmid; unsigned long vmid = READ_ONCE(v->vmid); @@ -258,51 +267,58 @@ static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) { - unsigned long vmid; struct kvm_riscv_hfence d = { 0 }; - struct kvm_vmid *v = &vcpu->kvm->arch.vmid; while (vcpu_hfence_dequeue(vcpu, &d)) { switch (d.type) { case KVM_RISCV_HFENCE_UNKNOWN: break; case KVM_RISCV_HFENCE_GVMA_VMID_GPA: - vmid = READ_ONCE(v->vmid); if (kvm_riscv_nacl_available()) - nacl_hfence_gvma_vmid(nacl_shmem(), vmid, + nacl_hfence_gvma_vmid(nacl_shmem(), d.vmid, d.addr, d.size, d.order); else - kvm_riscv_local_hfence_gvma_vmid_gpa(vmid, d.addr, + kvm_riscv_local_hfence_gvma_vmid_gpa(d.vmid, d.addr, d.size, d.order); break; + case KVM_RISCV_HFENCE_GVMA_VMID_ALL: + if (kvm_riscv_nacl_available()) + nacl_hfence_gvma_vmid_all(nacl_shmem(), d.vmid); + else + kvm_riscv_local_hfence_gvma_vmid_all(d.vmid); + break; case KVM_RISCV_HFENCE_VVMA_ASID_GVA: kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); - vmid = READ_ONCE(v->vmid); if (kvm_riscv_nacl_available()) - nacl_hfence_vvma_asid(nacl_shmem(), vmid, d.asid, + nacl_hfence_vvma_asid(nacl_shmem(), d.vmid, d.asid, d.addr, d.size, d.order); else - kvm_riscv_local_hfence_vvma_asid_gva(vmid, d.asid, d.addr, + kvm_riscv_local_hfence_vvma_asid_gva(d.vmid, d.asid, d.addr, d.size, d.order); break; case KVM_RISCV_HFENCE_VVMA_ASID_ALL: kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); - vmid = READ_ONCE(v->vmid); if (kvm_riscv_nacl_available()) - nacl_hfence_vvma_asid_all(nacl_shmem(), vmid, d.asid); + nacl_hfence_vvma_asid_all(nacl_shmem(), d.vmid, d.asid); else - kvm_riscv_local_hfence_vvma_asid_all(vmid, d.asid); + kvm_riscv_local_hfence_vvma_asid_all(d.vmid, d.asid); break; case KVM_RISCV_HFENCE_VVMA_GVA: kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); - vmid = READ_ONCE(v->vmid); if (kvm_riscv_nacl_available()) - nacl_hfence_vvma(nacl_shmem(), vmid, + nacl_hfence_vvma(nacl_shmem(), d.vmid, d.addr, d.size, d.order); else - kvm_riscv_local_hfence_vvma_gva(vmid, d.addr, + kvm_riscv_local_hfence_vvma_gva(d.vmid, d.addr, d.size, d.order); break; + case KVM_RISCV_HFENCE_VVMA_ALL: + kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); + if (kvm_riscv_nacl_available()) + nacl_hfence_vvma_all(nacl_shmem(), d.vmid); + else + kvm_riscv_local_hfence_vvma_all(d.vmid); + break; default: break; } @@ -322,7 +338,8 @@ static void make_xfence_request(struct kvm *kvm, bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); kvm_for_each_vcpu(i, vcpu, kvm) { if (hbase != -1UL) { - if (vcpu->vcpu_id < hbase) + if (vcpu->vcpu_id < hbase || + vcpu->vcpu_id >= hbase + BITS_PER_LONG) continue; if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) continue; @@ -355,35 +372,43 @@ void kvm_riscv_fence_i(struct kvm *kvm, void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, unsigned long hbase, unsigned long hmask, gpa_t gpa, gpa_t gpsz, - unsigned long order) + unsigned long order, unsigned long vmid) { struct kvm_riscv_hfence data; data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; data.asid = 0; + data.vmid = vmid; data.addr = gpa; data.size = gpsz; data.order = order; make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, - KVM_REQ_HFENCE_GVMA_VMID_ALL, &data); + KVM_REQ_TLB_FLUSH, &data); } void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, - unsigned long hbase, unsigned long hmask) + unsigned long hbase, unsigned long hmask, + unsigned long vmid) { - make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL, - KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL); + struct kvm_riscv_hfence data = {0}; + + data.type = KVM_RISCV_HFENCE_GVMA_VMID_ALL; + data.vmid = vmid; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_TLB_FLUSH, &data); } void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, unsigned long hbase, unsigned long hmask, unsigned long gva, unsigned long gvsz, - unsigned long order, unsigned long asid) + unsigned long order, unsigned long asid, + unsigned long vmid) { struct kvm_riscv_hfence data; data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; data.asid = asid; + data.vmid = vmid; data.addr = gva; data.size = gvsz; data.order = order; @@ -393,13 +418,13 @@ void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, unsigned long hbase, unsigned long hmask, - unsigned long asid) + unsigned long asid, unsigned long vmid) { - struct kvm_riscv_hfence data; + struct kvm_riscv_hfence data = {0}; data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; data.asid = asid; - data.addr = data.size = data.order = 0; + data.vmid = vmid; make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, KVM_REQ_HFENCE_VVMA_ALL, &data); } @@ -407,12 +432,13 @@ void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, unsigned long hbase, unsigned long hmask, unsigned long gva, unsigned long gvsz, - unsigned long order) + unsigned long order, unsigned long vmid) { struct kvm_riscv_hfence data; data.type = KVM_RISCV_HFENCE_VVMA_GVA; data.asid = 0; + data.vmid = vmid; data.addr = gva; data.size = gvsz; data.order = order; @@ -421,8 +447,21 @@ void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, } void kvm_riscv_hfence_vvma_all(struct kvm *kvm, - unsigned long hbase, unsigned long hmask) + unsigned long hbase, unsigned long hmask, + unsigned long vmid) +{ + struct kvm_riscv_hfence data = {0}; + + data.type = KVM_RISCV_HFENCE_VVMA_ALL; + data.vmid = vmid; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_HFENCE_VVMA_ALL, &data); +} + +int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages) { - make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL, - KVM_REQ_HFENCE_VVMA_ALL, NULL); + kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, + gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT, + PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid)); + return 0; } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 60d684c76c58..a73690eda84b 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -7,7 +7,6 @@ */ #include <linux/bitops.h> -#include <linux/entry-kvm.h> #include <linux/errno.h> #include <linux/err.h> #include <linux/kdebug.h> @@ -18,13 +17,16 @@ #include <linux/fs.h> #include <linux/kvm_host.h> #include <asm/cacheflush.h> +#include <asm/kvm_mmu.h> #include <asm/kvm_nacl.h> #include <asm/kvm_vcpu_vector.h> #define CREATE_TRACE_POINTS #include "trace.h" -const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { +static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_former_vcpu); + +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), STATS_DESC_COUNTER(VCPU, ecall_exit_stat), STATS_DESC_COUNTER(VCPU, wfi_exit_stat), @@ -51,12 +53,33 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; -static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) +static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, + bool kvm_sbi_reset) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; - struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; + void *vector_datap = cntx->vector.datap; + + memset(cntx, 0, sizeof(*cntx)); + memset(csr, 0, sizeof(*csr)); + memset(&vcpu->arch.smstateen_csr, 0, sizeof(vcpu->arch.smstateen_csr)); + + /* Restore datap as it's not a part of the guest context. */ + cntx->vector.datap = vector_datap; + + if (kvm_sbi_reset) + kvm_riscv_vcpu_sbi_load_reset_state(vcpu); + + /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ + cntx->sstatus = SR_SPP | SR_SPIE; + + cntx->hstatus |= HSTATUS_VTW; + cntx->hstatus |= HSTATUS_SPVP; + cntx->hstatus |= HSTATUS_SPV; +} + +static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) +{ bool loaded; /** @@ -71,11 +94,7 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) vcpu->arch.last_exit_cpu = -1; - memcpy(csr, reset_csr, sizeof(*csr)); - - spin_lock(&vcpu->arch.reset_cntx_lock); - memcpy(cntx, reset_cntx, sizeof(*cntx)); - spin_unlock(&vcpu->arch.reset_cntx_lock); + kvm_riscv_vcpu_context_reset(vcpu, kvm_sbi_reset); kvm_riscv_vcpu_fp_reset(vcpu); @@ -94,7 +113,7 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) vcpu->arch.hfence_tail = 0; memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); - kvm_riscv_vcpu_sbi_sta_reset(vcpu); + kvm_riscv_vcpu_sbi_reset(vcpu); /* Reset the guest CSRs for hotplug usecase */ if (loaded) @@ -110,16 +129,18 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) { int rc; - struct kvm_cpu_context *cntx; - struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; spin_lock_init(&vcpu->arch.mp_state_lock); /* Mark this VCPU never ran */ vcpu->arch.ran_atleast_once = false; + vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); + /* Setup VCPU config */ + kvm_riscv_vcpu_config_init(vcpu); + /* Setup ISA features available to VCPU */ kvm_riscv_vcpu_setup_isa(vcpu); @@ -131,23 +152,11 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); - /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ - spin_lock_init(&vcpu->arch.reset_cntx_lock); + spin_lock_init(&vcpu->arch.reset_state.lock); - spin_lock(&vcpu->arch.reset_cntx_lock); - cntx = &vcpu->arch.guest_reset_context; - cntx->sstatus = SR_SPP | SR_SPIE; - cntx->hstatus = 0; - cntx->hstatus |= HSTATUS_VTW; - cntx->hstatus |= HSTATUS_SPVP; - cntx->hstatus |= HSTATUS_SPV; - spin_unlock(&vcpu->arch.reset_cntx_lock); - - if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) - return -ENOMEM; - - /* By default, make CY, TM, and IR counters accessible in VU mode */ - reset_csr->scounteren = 0x7; + rc = kvm_riscv_vcpu_alloc_vector_context(vcpu); + if (rc) + return rc; /* Setup VCPU timer */ kvm_riscv_vcpu_timer_init(vcpu); @@ -156,9 +165,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_pmu_init(vcpu); /* Setup VCPU AIA */ - rc = kvm_riscv_vcpu_aia_init(vcpu); - if (rc) - return rc; + kvm_riscv_vcpu_aia_init(vcpu); /* * Setup SBI extensions @@ -167,7 +174,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_riscv_vcpu_sbi_init(vcpu); /* Reset VCPU */ - kvm_riscv_reset_vcpu(vcpu); + kvm_riscv_reset_vcpu(vcpu, false); return 0; } @@ -185,6 +192,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { + kvm_riscv_vcpu_sbi_deinit(vcpu); + /* Cleanup VCPU AIA context */ kvm_riscv_vcpu_aia_deinit(vcpu); @@ -205,19 +214,9 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) return kvm_riscv_vcpu_timer_pending(vcpu); } -void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) -{ - kvm_riscv_aia_wakeon_hgei(vcpu, true); -} - -void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) -{ - kvm_riscv_aia_wakeon_hgei(vcpu, false); -} - int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { - return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && + return (kvm_riscv_vcpu_has_interrupts(vcpu, -1ULL) && !kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause); } @@ -243,8 +242,8 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -long kvm_arch_vcpu_async_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; void __user *argp = (void __user *)arg; @@ -516,6 +515,12 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, case KVM_MP_STATE_STOPPED: __kvm_riscv_vcpu_power_off(vcpu); break; + case KVM_MP_STATE_INIT_RECEIVED: + if (vcpu->kvm->arch.mp_state_reset) + kvm_riscv_reset_vcpu(vcpu, false); + else + ret = -EINVAL; + break; default: ret = -EINVAL; } @@ -528,58 +533,41 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) { - if (dbg->control & KVM_GUESTDBG_ENABLE) { + if (dbg->control & KVM_GUESTDBG_ENABLE) vcpu->guest_debug = dbg->control; - vcpu->arch.cfg.hedeleg &= ~BIT(EXC_BREAKPOINT); - } else { + else vcpu->guest_debug = 0; - vcpu->arch.cfg.hedeleg |= BIT(EXC_BREAKPOINT); - } return 0; } -static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) -{ - const unsigned long *isa = vcpu->arch.isa; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; - - if (riscv_isa_extension_available(isa, SVPBMT)) - cfg->henvcfg |= ENVCFG_PBMTE; - - if (riscv_isa_extension_available(isa, SSTC)) - cfg->henvcfg |= ENVCFG_STCE; - - if (riscv_isa_extension_available(isa, ZICBOM)) - cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); - - if (riscv_isa_extension_available(isa, ZICBOZ)) - cfg->henvcfg |= ENVCFG_CBZE; - - if (riscv_isa_extension_available(isa, SVADU) && - !riscv_isa_extension_available(isa, SVADE)) - cfg->henvcfg |= ENVCFG_ADUE; - - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; - if (riscv_isa_extension_available(isa, SSAIA)) - cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | - SMSTATEEN0_AIA | - SMSTATEEN0_AIA_ISEL; - if (riscv_isa_extension_available(isa, SMSTATEEN)) - cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; - } - - cfg->hedeleg = KVM_HEDELEG_DEFAULT; - if (vcpu->guest_debug) - cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); -} - void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { void *nsh; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + /* + * If VCPU is being reloaded on the same physical CPU and no + * other KVM VCPU has run on this CPU since it was last put, + * we can skip the expensive CSR and HGATP writes. + * + * Note: If a new CSR is added to this fast-path skip block, + * make sure that 'csr_dirty' is set to true in any + * ioctl (e.g., KVM_SET_ONE_REG) that modifies it. + */ + if (vcpu != __this_cpu_read(kvm_former_vcpu)) + __this_cpu_write(kvm_former_vcpu, vcpu); + else if (vcpu->arch.last_exit_cpu == cpu && !vcpu->arch.csr_dirty) + goto csr_restore_done; + + vcpu->arch.csr_dirty = false; + + /* + * Load VCPU config CSRs before other CSRs because + * the read/write behaviour of certain CSRs change + * based on VCPU config CSRs. + */ + kvm_riscv_vcpu_config_load(vcpu); if (kvm_riscv_nacl_sync_csr_available()) { nsh = nacl_shmem(); @@ -590,17 +578,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) nacl_csr_write(nsh, CSR_VSEPC, csr->vsepc); nacl_csr_write(nsh, CSR_VSCAUSE, csr->vscause); nacl_csr_write(nsh, CSR_VSTVAL, csr->vstval); - nacl_csr_write(nsh, CSR_HEDELEG, cfg->hedeleg); nacl_csr_write(nsh, CSR_HVIP, csr->hvip); nacl_csr_write(nsh, CSR_VSATP, csr->vsatp); - nacl_csr_write(nsh, CSR_HENVCFG, cfg->henvcfg); - if (IS_ENABLED(CONFIG_32BIT)) - nacl_csr_write(nsh, CSR_HENVCFGH, cfg->henvcfg >> 32); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - nacl_csr_write(nsh, CSR_HSTATEEN0, cfg->hstateen0); - if (IS_ENABLED(CONFIG_32BIT)) - nacl_csr_write(nsh, CSR_HSTATEEN0H, cfg->hstateen0 >> 32); - } } else { csr_write(CSR_VSSTATUS, csr->vsstatus); csr_write(CSR_VSIE, csr->vsie); @@ -609,21 +588,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) csr_write(CSR_VSEPC, csr->vsepc); csr_write(CSR_VSCAUSE, csr->vscause); csr_write(CSR_VSTVAL, csr->vstval); - csr_write(CSR_HEDELEG, cfg->hedeleg); csr_write(CSR_HVIP, csr->hvip); csr_write(CSR_VSATP, csr->vsatp); - csr_write(CSR_HENVCFG, cfg->henvcfg); - if (IS_ENABLED(CONFIG_32BIT)) - csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { - csr_write(CSR_HSTATEEN0, cfg->hstateen0); - if (IS_ENABLED(CONFIG_32BIT)) - csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); - } } - kvm_riscv_gstage_update_hgatp(vcpu); + kvm_riscv_mmu_update_hgatp(vcpu); + + kvm_riscv_vcpu_aia_load(vcpu, cpu); +csr_restore_done: kvm_riscv_vcpu_timer_restore(vcpu); kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); @@ -633,8 +606,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, vcpu->arch.isa); - kvm_riscv_vcpu_aia_load(vcpu, cpu); - kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); vcpu->cpu = cpu; @@ -682,7 +653,14 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) } } -static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) +/** + * kvm_riscv_check_vcpu_requests - check and handle pending vCPU requests + * @vcpu: the VCPU pointer + * + * Return: 1 if we should enter the guest + * 0 if we should exit to userspace + */ +static int kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) { struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); @@ -704,20 +682,16 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) } if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) - kvm_riscv_reset_vcpu(vcpu); + kvm_riscv_reset_vcpu(vcpu, true); if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) - kvm_riscv_gstage_update_hgatp(vcpu); + kvm_riscv_mmu_update_hgatp(vcpu); if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) kvm_riscv_fence_i_process(vcpu); - /* - * The generic KVM_REQ_TLB_FLUSH is same as - * KVM_REQ_HFENCE_GVMA_VMID_ALL - */ - if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) - kvm_riscv_hfence_gvma_vmid_all_process(vcpu); + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) + kvm_riscv_tlb_flush_process(vcpu); if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) kvm_riscv_hfence_vvma_all_process(vcpu); @@ -727,7 +701,12 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) kvm_riscv_vcpu_record_steal_time(vcpu); + + if (kvm_dirty_ring_check_request(vcpu)) + return 0; } + + return 1; } static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) @@ -742,28 +721,22 @@ static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu * { struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; vcpu->arch.host_scounteren = csr_swap(CSR_SCOUNTEREN, csr->scounteren); vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && - (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) - vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, - smcsr->sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, smcsr->sstateen0); } static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu) { struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; - struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; csr->scounteren = csr_swap(CSR_SCOUNTEREN, vcpu->arch.host_scounteren); csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg); - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && - (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) - smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, - vcpu->arch.host_sstateen0); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) + smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, vcpu->arch.host_sstateen0); } /* @@ -860,7 +833,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) struct kvm_run *run = vcpu->run; if (!vcpu->arch.ran_atleast_once) - kvm_riscv_vcpu_setup_config(vcpu); + kvm_riscv_vcpu_config_ran_once(vcpu); /* Mark this VCPU ran at least once */ vcpu->arch.ran_atleast_once = true; @@ -902,14 +875,16 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_UNKNOWN; while (ret > 0) { /* Check conditions before entering the guest */ - ret = xfer_to_guest_mode_handle_work(vcpu); + ret = kvm_xfer_to_guest_mode_handle_work(vcpu); if (ret) continue; ret = 1; kvm_riscv_gstage_vmid_update(vcpu); - kvm_riscv_check_vcpu_requests(vcpu); + ret = kvm_riscv_check_vcpu_requests(vcpu); + if (ret <= 0) + continue; preempt_disable(); @@ -953,7 +928,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } /* - * Cleanup stale TLB enteries + * Sanitize VMID mappings cached (TLB) on current CPU * * Note: This should be done after G-stage VMID has been * updated using kvm_riscv_gstage_vmid_ver_changed() diff --git a/arch/riscv/kvm/vcpu_config.c b/arch/riscv/kvm/vcpu_config.c new file mode 100644 index 000000000000..238418fed2b9 --- /dev/null +++ b/arch/riscv/kvm/vcpu_config.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Qualcomm Technologies, Inc. + */ + +#include <linux/kvm_host.h> +#include <asm/kvm_nacl.h> + +#define KVM_HEDELEG_DEFAULT (BIT(EXC_INST_MISALIGNED) | \ + BIT(EXC_INST_ILLEGAL) | \ + BIT(EXC_BREAKPOINT) | \ + BIT(EXC_SYSCALL) | \ + BIT(EXC_INST_PAGE_FAULT) | \ + BIT(EXC_LOAD_PAGE_FAULT) | \ + BIT(EXC_STORE_PAGE_FAULT)) + +#define KVM_HIDELEG_DEFAULT (BIT(IRQ_VS_SOFT) | \ + BIT(IRQ_VS_TIMER) | \ + BIT(IRQ_VS_EXT)) + +void kvm_riscv_vcpu_config_init(struct kvm_vcpu *vcpu) +{ + vcpu->arch.cfg.hedeleg = KVM_HEDELEG_DEFAULT; + vcpu->arch.cfg.hideleg = KVM_HIDELEG_DEFAULT; +} + +void kvm_riscv_vcpu_config_guest_debug(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (vcpu->guest_debug) + cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); + else + cfg->hedeleg |= BIT(EXC_BREAKPOINT); + + vcpu->arch.csr_dirty = true; +} + +void kvm_riscv_vcpu_config_ran_once(struct kvm_vcpu *vcpu) +{ + const unsigned long *isa = vcpu->arch.isa; + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (riscv_isa_extension_available(isa, SVPBMT)) + cfg->henvcfg |= ENVCFG_PBMTE; + + if (riscv_isa_extension_available(isa, SSTC)) + cfg->henvcfg |= ENVCFG_STCE; + + if (riscv_isa_extension_available(isa, ZICBOM)) + cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); + + if (riscv_isa_extension_available(isa, ZICBOZ)) + cfg->henvcfg |= ENVCFG_CBZE; + + if (riscv_isa_extension_available(isa, SVADU) && + !riscv_isa_extension_available(isa, SVADE)) + cfg->henvcfg |= ENVCFG_ADUE; + + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; + if (riscv_isa_extension_available(isa, SSAIA)) + cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | + SMSTATEEN0_AIA | + SMSTATEEN0_AIA_ISEL; + if (riscv_isa_extension_available(isa, SMSTATEEN)) + cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; + } + + if (vcpu->guest_debug) + cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); +} + +void kvm_riscv_vcpu_config_load(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + void *nsh; + + if (kvm_riscv_nacl_sync_csr_available()) { + nsh = nacl_shmem(); + nacl_csr_write(nsh, CSR_HEDELEG, cfg->hedeleg); + nacl_csr_write(nsh, CSR_HIDELEG, cfg->hideleg); + nacl_csr_write(nsh, CSR_HENVCFG, cfg->henvcfg); + if (IS_ENABLED(CONFIG_32BIT)) + nacl_csr_write(nsh, CSR_HENVCFGH, cfg->henvcfg >> 32); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + nacl_csr_write(nsh, CSR_HSTATEEN0, cfg->hstateen0); + if (IS_ENABLED(CONFIG_32BIT)) + nacl_csr_write(nsh, CSR_HSTATEEN0H, cfg->hstateen0 >> 32); + } + } else { + csr_write(CSR_HEDELEG, cfg->hedeleg); + csr_write(CSR_HIDELEG, cfg->hideleg); + csr_write(CSR_HENVCFG, cfg->henvcfg); + if (IS_ENABLED(CONFIG_32BIT)) + csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { + csr_write(CSR_HSTATEEN0, cfg->hstateen0); + if (IS_ENABLED(CONFIG_32BIT)) + csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); + } + } +} diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 6e0c18412795..0bb0c51e3c89 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -9,10 +9,13 @@ #include <linux/kvm_host.h> #include <asm/csr.h> #include <asm/insn-def.h> +#include <asm/kvm_mmu.h> +#include <asm/kvm_nacl.h> static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap) { + struct kvm_gstage_mapping host_map; struct kvm_memory_slot *memslot; unsigned long hva, fault_addr; bool writable; @@ -40,8 +43,9 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, }; } - ret = kvm_riscv_gstage_map(vcpu, memslot, fault_addr, hva, - (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false); + ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva, + (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false, + &host_map); if (ret < 0) return ret; @@ -135,7 +139,7 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap) { - unsigned long vsstatus = csr_read(CSR_VSSTATUS); + unsigned long vsstatus = ncsr_read(CSR_VSSTATUS); /* Change Guest SSTATUS.SPP bit */ vsstatus &= ~SR_SPP; @@ -151,15 +155,15 @@ void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, vsstatus &= ~SR_SIE; /* Update Guest SSTATUS */ - csr_write(CSR_VSSTATUS, vsstatus); + ncsr_write(CSR_VSSTATUS, vsstatus); /* Update Guest SCAUSE, STVAL, and SEPC */ - csr_write(CSR_VSCAUSE, trap->scause); - csr_write(CSR_VSTVAL, trap->stval); - csr_write(CSR_VSEPC, trap->sepc); + ncsr_write(CSR_VSCAUSE, trap->scause); + ncsr_write(CSR_VSTVAL, trap->stval); + ncsr_write(CSR_VSEPC, trap->sepc); /* Set Guest PC to Guest exception vector */ - vcpu->arch.guest_context.sepc = csr_read(CSR_VSTVEC); + vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC); /* Set Guest privilege mode to supervisor */ vcpu->arch.guest_context.sstatus |= SR_SPP; diff --git a/arch/riscv/kvm/vcpu_fp.c b/arch/riscv/kvm/vcpu_fp.c index 030904d82b58..6ad6df26a2fd 100644 --- a/arch/riscv/kvm/vcpu_fp.c +++ b/arch/riscv/kvm/vcpu_fp.c @@ -10,8 +10,10 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #ifdef CONFIG_FPU void kvm_riscv_vcpu_fp_reset(struct kvm_vcpu *vcpu) @@ -59,17 +61,17 @@ void kvm_riscv_vcpu_guest_fp_restore(struct kvm_cpu_context *cntx, void kvm_riscv_vcpu_host_fp_save(struct kvm_cpu_context *cntx) { /* No need to check host sstatus as it can be modified outside */ - if (riscv_isa_extension_available(NULL, d)) + if (!kvm_riscv_isa_check_host(D)) __kvm_riscv_fp_d_save(cntx); - else if (riscv_isa_extension_available(NULL, f)) + else if (!kvm_riscv_isa_check_host(F)) __kvm_riscv_fp_f_save(cntx); } void kvm_riscv_vcpu_host_fp_restore(struct kvm_cpu_context *cntx) { - if (riscv_isa_extension_available(NULL, d)) + if (!kvm_riscv_isa_check_host(D)) __kvm_riscv_fp_d_restore(cntx); - else if (riscv_isa_extension_available(NULL, f)) + else if (!kvm_riscv_isa_check_host(F)) __kvm_riscv_fp_f_restore(cntx); } #endif @@ -93,9 +95,11 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) reg_val = &cntx->fp.f.fcsr; else if ((KVM_REG_RISCV_FP_F_REG(f[0]) <= reg_num) && - reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) + reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) { + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.f.f)); reg_val = &cntx->fp.f.f[reg_num]; - else + } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && riscv_isa_extension_available(vcpu->arch.isa, d)) { @@ -107,6 +111,8 @@ int kvm_riscv_vcpu_get_reg_fp(struct kvm_vcpu *vcpu, reg_num <= KVM_REG_RISCV_FP_D_REG(f[31])) { if (KVM_REG_SIZE(reg->id) != sizeof(u64)) return -EINVAL; + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.d.f)); reg_val = &cntx->fp.d.f[reg_num]; } else return -ENOENT; @@ -138,9 +144,11 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, if (reg_num == KVM_REG_RISCV_FP_F_REG(fcsr)) reg_val = &cntx->fp.f.fcsr; else if ((KVM_REG_RISCV_FP_F_REG(f[0]) <= reg_num) && - reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) + reg_num <= KVM_REG_RISCV_FP_F_REG(f[31])) { + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.f.f)); reg_val = &cntx->fp.f.f[reg_num]; - else + } else return -ENOENT; } else if ((rtype == KVM_REG_RISCV_FP_D) && riscv_isa_extension_available(vcpu->arch.isa, d)) { @@ -152,6 +160,8 @@ int kvm_riscv_vcpu_set_reg_fp(struct kvm_vcpu *vcpu, reg_num <= KVM_REG_RISCV_FP_D_REG(f[31])) { if (KVM_REG_SIZE(reg->id) != sizeof(u64)) return -EINVAL; + reg_num = array_index_nospec(reg_num, + ARRAY_SIZE(cntx->fp.d.f)); reg_val = &cntx->fp.d.f[reg_num]; } else return -ENOENT; diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 97dec18e6989..f09f9251d1f0 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -8,133 +8,7 @@ #include <linux/kvm_host.h> #include <asm/cpufeature.h> - -#define INSN_OPCODE_MASK 0x007c -#define INSN_OPCODE_SHIFT 2 -#define INSN_OPCODE_SYSTEM 28 - -#define INSN_MASK_WFI 0xffffffff -#define INSN_MATCH_WFI 0x10500073 - -#define INSN_MASK_WRS 0xffffffff -#define INSN_MATCH_WRS 0x00d00073 - -#define INSN_MATCH_CSRRW 0x1073 -#define INSN_MASK_CSRRW 0x707f -#define INSN_MATCH_CSRRS 0x2073 -#define INSN_MASK_CSRRS 0x707f -#define INSN_MATCH_CSRRC 0x3073 -#define INSN_MASK_CSRRC 0x707f -#define INSN_MATCH_CSRRWI 0x5073 -#define INSN_MASK_CSRRWI 0x707f -#define INSN_MATCH_CSRRSI 0x6073 -#define INSN_MASK_CSRRSI 0x707f -#define INSN_MATCH_CSRRCI 0x7073 -#define INSN_MASK_CSRRCI 0x707f - -#define INSN_MATCH_LB 0x3 -#define INSN_MASK_LB 0x707f -#define INSN_MATCH_LH 0x1003 -#define INSN_MASK_LH 0x707f -#define INSN_MATCH_LW 0x2003 -#define INSN_MASK_LW 0x707f -#define INSN_MATCH_LD 0x3003 -#define INSN_MASK_LD 0x707f -#define INSN_MATCH_LBU 0x4003 -#define INSN_MASK_LBU 0x707f -#define INSN_MATCH_LHU 0x5003 -#define INSN_MASK_LHU 0x707f -#define INSN_MATCH_LWU 0x6003 -#define INSN_MASK_LWU 0x707f -#define INSN_MATCH_SB 0x23 -#define INSN_MASK_SB 0x707f -#define INSN_MATCH_SH 0x1023 -#define INSN_MASK_SH 0x707f -#define INSN_MATCH_SW 0x2023 -#define INSN_MASK_SW 0x707f -#define INSN_MATCH_SD 0x3023 -#define INSN_MASK_SD 0x707f - -#define INSN_MATCH_C_LD 0x6000 -#define INSN_MASK_C_LD 0xe003 -#define INSN_MATCH_C_SD 0xe000 -#define INSN_MASK_C_SD 0xe003 -#define INSN_MATCH_C_LW 0x4000 -#define INSN_MASK_C_LW 0xe003 -#define INSN_MATCH_C_SW 0xc000 -#define INSN_MASK_C_SW 0xe003 -#define INSN_MATCH_C_LDSP 0x6002 -#define INSN_MASK_C_LDSP 0xe003 -#define INSN_MATCH_C_SDSP 0xe002 -#define INSN_MASK_C_SDSP 0xe003 -#define INSN_MATCH_C_LWSP 0x4002 -#define INSN_MASK_C_LWSP 0xe003 -#define INSN_MATCH_C_SWSP 0xc002 -#define INSN_MASK_C_SWSP 0xe003 - -#define INSN_16BIT_MASK 0x3 - -#define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK) - -#define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4) - -#ifdef CONFIG_64BIT -#define LOG_REGBYTES 3 -#else -#define LOG_REGBYTES 2 -#endif -#define REGBYTES (1 << LOG_REGBYTES) - -#define SH_RD 7 -#define SH_RS1 15 -#define SH_RS2 20 -#define SH_RS2C 2 -#define MASK_RX 0x1f - -#define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) -#define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ - (RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 1) << 6)) -#define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 5, 2) << 6)) -#define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 2) << 6)) -#define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ - (RV_X(x, 12, 1) << 5) | \ - (RV_X(x, 2, 3) << 6)) -#define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ - (RV_X(x, 7, 2) << 6)) -#define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ - (RV_X(x, 7, 3) << 6)) -#define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) -#define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) -#define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) - -#define SHIFT_RIGHT(x, y) \ - ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) - -#define REG_MASK \ - ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) - -#define REG_OFFSET(insn, pos) \ - (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) - -#define REG_PTR(insn, pos, regs) \ - ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))) - -#define GET_FUNCT3(insn) (((insn) >> 12) & 7) - -#define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) -#define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) -#define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) -#define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) -#define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) -#define GET_SP(regs) (*REG_PTR(2, 0, regs)) -#define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) -#define IMM_I(insn) ((s32)(insn) >> 20) -#define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ - (s32)(((insn) >> 7) & 0x1f)) +#include <asm/insn.h> struct insn_func { unsigned long mask; @@ -424,6 +298,22 @@ static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, return (rc <= 0) ? rc : 1; } +static bool is_load_guest_page_fault(unsigned long scause) +{ + /** + * If a g-stage page fault occurs, the direct approach + * is to let the g-stage page fault handler handle it + * naturally, however, calling the g-stage page fault + * handler here seems rather strange. + * Considering this is a corner case, we can directly + * return to the guest and re-execute the same PC, this + * will trigger a g-stage page fault again and then the + * regular g-stage page fault handler will populate + * g-stage page table. + */ + return (scause == EXC_LOAD_GUEST_PAGE_FAULT); +} + /** * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap * @@ -449,6 +339,8 @@ int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); return 1; @@ -504,6 +396,8 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; /* Redirect trap if we failed to read instruction */ utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); @@ -521,7 +415,6 @@ int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run, shift = 8 * (sizeof(ulong) - len); } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) { len = 1; - shift = 8 * (sizeof(ulong) - len); #ifdef CONFIG_64BIT } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { len = 8; @@ -630,6 +523,8 @@ int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run, insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc, &utrap); if (utrap.scause) { + if (is_load_guest_page_fault(utrap.scause)) + return 1; /* Redirect trap if we failed to read instruction */ utrap.sepc = ct->sepc; kvm_riscv_vcpu_trap_redirect(vcpu, &utrap); @@ -753,22 +648,22 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) case 1: data8 = *((u8 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data8 << shift >> shift); + (long)((ulong)data8 << shift) >> shift); break; case 2: data16 = *((u16 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data16 << shift >> shift); + (long)((ulong)data16 << shift) >> shift); break; case 4: data32 = *((u32 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data32 << shift >> shift); + (long)((ulong)data32 << shift) >> shift); break; case 8: data64 = *((u64 *)run->mmio.data); SET_RD(insn, &vcpu->arch.guest_context, - (ulong)data64 << shift >> shift); + (long)((ulong)data64 << shift) >> shift); break; default: return -EOPNOTSUPP; diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index f6d27b59c641..bb920e8923c9 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -10,218 +10,25 @@ #include <linux/bitops.h> #include <linux/errno.h> #include <linux/err.h> +#include <linux/nospec.h> #include <linux/uaccess.h> #include <linux/kvm_host.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_vector.h> -#include <asm/pgtable.h> -#include <asm/vector.h> #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) -#define KVM_ISA_EXT_ARR(ext) \ -[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext - -/* Mapping between KVM ISA Extension ID & Host ISA extension ID */ -static const unsigned long kvm_isa_ext_arr[] = { - /* Single letter extensions (alphabetically sorted) */ - [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, - [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, - [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, - [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, - [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, - [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, - [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, - [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, - /* Multi letter extensions (alphabetically sorted) */ - [KVM_RISCV_ISA_EXT_SMNPM] = RISCV_ISA_EXT_SSNPM, - KVM_ISA_EXT_ARR(SMSTATEEN), - KVM_ISA_EXT_ARR(SSAIA), - KVM_ISA_EXT_ARR(SSCOFPMF), - KVM_ISA_EXT_ARR(SSNPM), - KVM_ISA_EXT_ARR(SSTC), - KVM_ISA_EXT_ARR(SVADE), - KVM_ISA_EXT_ARR(SVADU), - KVM_ISA_EXT_ARR(SVINVAL), - KVM_ISA_EXT_ARR(SVNAPOT), - KVM_ISA_EXT_ARR(SVPBMT), - KVM_ISA_EXT_ARR(SVVPTC), - KVM_ISA_EXT_ARR(ZABHA), - KVM_ISA_EXT_ARR(ZACAS), - KVM_ISA_EXT_ARR(ZAWRS), - KVM_ISA_EXT_ARR(ZBA), - KVM_ISA_EXT_ARR(ZBB), - KVM_ISA_EXT_ARR(ZBC), - KVM_ISA_EXT_ARR(ZBKB), - KVM_ISA_EXT_ARR(ZBKC), - KVM_ISA_EXT_ARR(ZBKX), - KVM_ISA_EXT_ARR(ZBS), - KVM_ISA_EXT_ARR(ZCA), - KVM_ISA_EXT_ARR(ZCB), - KVM_ISA_EXT_ARR(ZCD), - KVM_ISA_EXT_ARR(ZCF), - KVM_ISA_EXT_ARR(ZCMOP), - KVM_ISA_EXT_ARR(ZFA), - KVM_ISA_EXT_ARR(ZFH), - KVM_ISA_EXT_ARR(ZFHMIN), - KVM_ISA_EXT_ARR(ZICBOM), - KVM_ISA_EXT_ARR(ZICBOZ), - KVM_ISA_EXT_ARR(ZICCRSE), - KVM_ISA_EXT_ARR(ZICNTR), - KVM_ISA_EXT_ARR(ZICOND), - KVM_ISA_EXT_ARR(ZICSR), - KVM_ISA_EXT_ARR(ZIFENCEI), - KVM_ISA_EXT_ARR(ZIHINTNTL), - KVM_ISA_EXT_ARR(ZIHINTPAUSE), - KVM_ISA_EXT_ARR(ZIHPM), - KVM_ISA_EXT_ARR(ZIMOP), - KVM_ISA_EXT_ARR(ZKND), - KVM_ISA_EXT_ARR(ZKNE), - KVM_ISA_EXT_ARR(ZKNH), - KVM_ISA_EXT_ARR(ZKR), - KVM_ISA_EXT_ARR(ZKSED), - KVM_ISA_EXT_ARR(ZKSH), - KVM_ISA_EXT_ARR(ZKT), - KVM_ISA_EXT_ARR(ZTSO), - KVM_ISA_EXT_ARR(ZVBB), - KVM_ISA_EXT_ARR(ZVBC), - KVM_ISA_EXT_ARR(ZVFH), - KVM_ISA_EXT_ARR(ZVFHMIN), - KVM_ISA_EXT_ARR(ZVKB), - KVM_ISA_EXT_ARR(ZVKG), - KVM_ISA_EXT_ARR(ZVKNED), - KVM_ISA_EXT_ARR(ZVKNHA), - KVM_ISA_EXT_ARR(ZVKNHB), - KVM_ISA_EXT_ARR(ZVKSED), - KVM_ISA_EXT_ARR(ZVKSH), - KVM_ISA_EXT_ARR(ZVKT), -}; - -static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) -{ - unsigned long i; - - for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { - if (kvm_isa_ext_arr[i] == base_ext) - return i; - } - - return KVM_RISCV_ISA_EXT_MAX; -} - -static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) -{ - switch (ext) { - case KVM_RISCV_ISA_EXT_H: - return false; - case KVM_RISCV_ISA_EXT_SSCOFPMF: - /* Sscofpmf depends on interrupt filtering defined in ssaia */ - return __riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSAIA); - case KVM_RISCV_ISA_EXT_SVADU: - /* - * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. - * Guest OS can use Svadu only when host OS enable Svadu. - */ - return arch_has_hw_pte_young(); - case KVM_RISCV_ISA_EXT_V: - return riscv_v_vstate_ctrl_user_allowed(); - default: - break; - } - - return true; -} - -static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) -{ - switch (ext) { - /* Extensions which don't have any mechanism to disable */ - case KVM_RISCV_ISA_EXT_A: - case KVM_RISCV_ISA_EXT_C: - case KVM_RISCV_ISA_EXT_I: - case KVM_RISCV_ISA_EXT_M: - case KVM_RISCV_ISA_EXT_SMNPM: - /* There is not architectural config bit to disable sscofpmf completely */ - case KVM_RISCV_ISA_EXT_SSCOFPMF: - case KVM_RISCV_ISA_EXT_SSNPM: - case KVM_RISCV_ISA_EXT_SSTC: - case KVM_RISCV_ISA_EXT_SVINVAL: - case KVM_RISCV_ISA_EXT_SVNAPOT: - case KVM_RISCV_ISA_EXT_SVVPTC: - case KVM_RISCV_ISA_EXT_ZABHA: - case KVM_RISCV_ISA_EXT_ZACAS: - case KVM_RISCV_ISA_EXT_ZAWRS: - case KVM_RISCV_ISA_EXT_ZBA: - case KVM_RISCV_ISA_EXT_ZBB: - case KVM_RISCV_ISA_EXT_ZBC: - case KVM_RISCV_ISA_EXT_ZBKB: - case KVM_RISCV_ISA_EXT_ZBKC: - case KVM_RISCV_ISA_EXT_ZBKX: - case KVM_RISCV_ISA_EXT_ZBS: - case KVM_RISCV_ISA_EXT_ZCA: - case KVM_RISCV_ISA_EXT_ZCB: - case KVM_RISCV_ISA_EXT_ZCD: - case KVM_RISCV_ISA_EXT_ZCF: - case KVM_RISCV_ISA_EXT_ZCMOP: - case KVM_RISCV_ISA_EXT_ZFA: - case KVM_RISCV_ISA_EXT_ZFH: - case KVM_RISCV_ISA_EXT_ZFHMIN: - case KVM_RISCV_ISA_EXT_ZICCRSE: - case KVM_RISCV_ISA_EXT_ZICNTR: - case KVM_RISCV_ISA_EXT_ZICOND: - case KVM_RISCV_ISA_EXT_ZICSR: - case KVM_RISCV_ISA_EXT_ZIFENCEI: - case KVM_RISCV_ISA_EXT_ZIHINTNTL: - case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: - case KVM_RISCV_ISA_EXT_ZIHPM: - case KVM_RISCV_ISA_EXT_ZIMOP: - case KVM_RISCV_ISA_EXT_ZKND: - case KVM_RISCV_ISA_EXT_ZKNE: - case KVM_RISCV_ISA_EXT_ZKNH: - case KVM_RISCV_ISA_EXT_ZKR: - case KVM_RISCV_ISA_EXT_ZKSED: - case KVM_RISCV_ISA_EXT_ZKSH: - case KVM_RISCV_ISA_EXT_ZKT: - case KVM_RISCV_ISA_EXT_ZTSO: - case KVM_RISCV_ISA_EXT_ZVBB: - case KVM_RISCV_ISA_EXT_ZVBC: - case KVM_RISCV_ISA_EXT_ZVFH: - case KVM_RISCV_ISA_EXT_ZVFHMIN: - case KVM_RISCV_ISA_EXT_ZVKB: - case KVM_RISCV_ISA_EXT_ZVKG: - case KVM_RISCV_ISA_EXT_ZVKNED: - case KVM_RISCV_ISA_EXT_ZVKNHA: - case KVM_RISCV_ISA_EXT_ZVKNHB: - case KVM_RISCV_ISA_EXT_ZVKSED: - case KVM_RISCV_ISA_EXT_ZVKSH: - case KVM_RISCV_ISA_EXT_ZVKT: - return false; - /* Extensions which can be disabled using Smstateen */ - case KVM_RISCV_ISA_EXT_SSAIA: - return riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN); - case KVM_RISCV_ISA_EXT_SVADE: - /* - * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero. - * Svade is not allowed to disable when the platform use Svade. - */ - return arch_has_hw_pte_young(); - default: - break; - } - - return true; -} - void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) { - unsigned long host_isa, i; + unsigned long guest_ext, i; - for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) { - host_isa = kvm_isa_ext_arr[i]; - if (__riscv_isa_extension_available(NULL, host_isa) && - kvm_riscv_vcpu_isa_enable_allowed(i)) - set_bit(host_isa, vcpu->arch.isa); + for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { + if (__kvm_riscv_isa_check_host(i, &guest_ext)) + continue; + if (kvm_riscv_isa_enable_allowed(i)) + set_bit(guest_ext, vcpu->arch.isa); } } @@ -243,15 +50,20 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) + if (kvm_riscv_isa_check_host(ZICBOM)) return -ENOENT; reg_val = riscv_cbom_block_size; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) + if (kvm_riscv_isa_check_host(ZICBOZ)) return -ENOENT; reg_val = riscv_cboz_block_size; break; + case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): + if (kvm_riscv_isa_check_host(ZICBOP)) + return -ENOENT; + reg_val = riscv_cbop_block_size; + break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): reg_val = vcpu->arch.mvendorid; break; @@ -309,15 +121,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, if (!vcpu->arch.ran_atleast_once) { /* Ignore the enable/disable request for certain extensions */ for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { - isa_ext = kvm_riscv_vcpu_base2isa_ext(i); + isa_ext = kvm_riscv_base2isa_ext(i); if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { reg_val &= ~BIT(i); continue; } - if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext)) + if (!kvm_riscv_isa_enable_allowed(isa_ext)) if (reg_val & BIT(i)) reg_val &= ~BIT(i); - if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext)) + if (!kvm_riscv_isa_disable_allowed(isa_ext)) if (!(reg_val & BIT(i))) reg_val |= BIT(i); } @@ -332,17 +144,23 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, } break; case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) + if (kvm_riscv_isa_check_host(ZICBOM)) return -ENOENT; if (reg_val != riscv_cbom_block_size) return -EINVAL; break; case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): - if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) + if (kvm_riscv_isa_check_host(ZICBOZ)) return -ENOENT; if (reg_val != riscv_cboz_block_size) return -EINVAL; break; + case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): + if (kvm_riscv_isa_check_host(ZICBOP)) + return -ENOENT; + if (reg_val != riscv_cbop_block_size) + return -EINVAL; + break; case KVM_REG_RISCV_CONFIG_REG(mvendorid): if (reg_val == vcpu->arch.mvendorid) break; @@ -387,13 +205,16 @@ static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_CORE); + unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); unsigned long reg_val; if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) return -EINVAL; - if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) reg_val = cntx->sepc; else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && @@ -420,13 +241,16 @@ static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_CORE); + unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); unsigned long reg_val; if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) return -EINVAL; - if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) return -EFAULT; @@ -451,10 +275,13 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { kvm_riscv_vcpu_flush_interrupts(vcpu); *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; @@ -470,10 +297,13 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, unsigned long reg_val) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) + if (reg_num >= regs_max) return -ENOENT; + reg_num = array_index_nospec(reg_num, regs_max); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { reg_val &= VSIP_VALID_MASK; reg_val <<= VSIP_TO_HVIP_SHIFT; @@ -492,10 +322,15 @@ static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu, unsigned long reg_val) { struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / + sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) / - sizeof(unsigned long)) - return -EINVAL; + if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); ((unsigned long *)csr)[reg_num] = reg_val; return 0; @@ -506,10 +341,15 @@ static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu, unsigned long *out_val) { struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; + unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / + sizeof(unsigned long); - if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) / - sizeof(unsigned long)) - return -EINVAL; + if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) + return -ENOENT; + if (reg_num >= regs_max) + return -ENOENT; + + reg_num = array_index_nospec(reg_num, regs_max); *out_val = ((unsigned long *)csr)[reg_num]; return 0; @@ -539,10 +379,7 @@ static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); break; case KVM_REG_RISCV_CSR_SMSTATEEN: - rc = -EINVAL; - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) - rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, - ®_val); + rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, ®_val); break; default: rc = -ENOENT; @@ -584,10 +421,7 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); break; case KVM_REG_RISCV_CSR_SMSTATEEN: - rc = -EINVAL; - if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) - rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, - reg_val); + rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); break; default: rc = -ENOENT; @@ -596,6 +430,8 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, if (rc) return rc; + vcpu->arch.csr_dirty = true; + return 0; } @@ -603,18 +439,15 @@ static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, unsigned long reg_num, unsigned long *reg_val) { - unsigned long host_isa_ext; - - if (reg_num >= KVM_RISCV_ISA_EXT_MAX || - reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) - return -ENOENT; + unsigned long guest_ext; + int ret; - host_isa_ext = kvm_isa_ext_arr[reg_num]; - if (!__riscv_isa_extension_available(NULL, host_isa_ext)) - return -ENOENT; + ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); + if (ret) + return ret; *reg_val = 0; - if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext)) + if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext)) *reg_val = 1; /* Mark the given extension as available */ return 0; @@ -624,17 +457,14 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, unsigned long reg_num, unsigned long reg_val) { - unsigned long host_isa_ext; - - if (reg_num >= KVM_RISCV_ISA_EXT_MAX || - reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) - return -ENOENT; + unsigned long guest_ext; + int ret; - host_isa_ext = kvm_isa_ext_arr[reg_num]; - if (!__riscv_isa_extension_available(NULL, host_isa_ext)) - return -ENOENT; + ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); + if (ret) + return ret; - if (reg_val == test_bit(host_isa_ext, vcpu->arch.isa)) + if (reg_val == test_bit(guest_ext, vcpu->arch.isa)) return 0; if (!vcpu->arch.ran_atleast_once) { @@ -643,11 +473,11 @@ static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, * extension can be disabled */ if (reg_val == 1 && - kvm_riscv_vcpu_isa_enable_allowed(reg_num)) - set_bit(host_isa_ext, vcpu->arch.isa); + kvm_riscv_isa_enable_allowed(reg_num)) + set_bit(guest_ext, vcpu->arch.isa); else if (!reg_val && - kvm_riscv_vcpu_isa_disable_allowed(reg_num)) - clear_bit(host_isa_ext, vcpu->arch.isa); + kvm_riscv_isa_disable_allowed(reg_num)) + clear_bit(guest_ext, vcpu->arch.isa); else return -EINVAL; kvm_riscv_vcpu_fp_reset(vcpu); @@ -789,10 +619,13 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, * was not available. */ if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && - !riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) + kvm_riscv_isa_check_host(ZICBOM)) continue; else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && - !riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) + kvm_riscv_isa_check_host(ZICBOZ)) + continue; + else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) && + kvm_riscv_isa_check_host(ZICBOP)) continue; size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; @@ -1005,16 +838,15 @@ static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu, static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, u64 __user *uindices) { + unsigned long guest_ext; unsigned int n = 0; - unsigned long isa_ext; for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; - isa_ext = kvm_isa_ext_arr[i]; - if (!__riscv_isa_extension_available(NULL, isa_ext)) + if (__kvm_riscv_isa_check_host(i, &guest_ext)) continue; if (uindices) { @@ -1034,66 +866,14 @@ static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) return copy_isa_ext_reg_indices(vcpu, NULL); } -static int copy_sbi_ext_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - unsigned int n = 0; - - for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { - u64 size = IS_ENABLED(CONFIG_32BIT) ? - KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; - u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | - KVM_REG_RISCV_SBI_SINGLE | i; - - if (!riscv_vcpu_supports_sbi_ext(vcpu, i)) - continue; - - if (uindices) { - if (put_user(reg, uindices)) - return -EFAULT; - uindices++; - } - - n++; - } - - return n; -} - static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) { - return copy_sbi_ext_reg_indices(vcpu, NULL); -} - -static int copy_sbi_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; - int total = 0; - - if (scontext->ext_status[KVM_RISCV_SBI_EXT_STA] == KVM_RISCV_SBI_EXT_STATUS_ENABLED) { - u64 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; - int n = sizeof(struct kvm_riscv_sbi_sta) / sizeof(unsigned long); - - for (int i = 0; i < n; i++) { - u64 reg = KVM_REG_RISCV | size | - KVM_REG_RISCV_SBI_STATE | - KVM_REG_RISCV_SBI_STA | i; - - if (uindices) { - if (put_user(reg, uindices)) - return -EFAULT; - uindices++; - } - } - - total += n; - } - - return total; + return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL); } static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) { - return copy_sbi_reg_indices(vcpu, NULL); + return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL); } static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) @@ -1216,12 +996,12 @@ int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, return ret; uindices += ret; - ret = copy_sbi_ext_reg_indices(vcpu, uindices); + ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices); if (ret < 0) return ret; uindices += ret; - ret = copy_sbi_reg_indices(vcpu, uindices); + ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices); if (ret < 0) return ret; uindices += ret; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 2707a51b082c..bb46dcbfb24d 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -7,15 +7,17 @@ */ #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt +#include <linux/bitops.h> #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/nospec.h> #include <linux/perf/riscv_pmu.h> #include <asm/csr.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_sbi.h> #include <asm/kvm_vcpu_pmu.h> #include <asm/sbi.h> -#include <linux/bitops.h> #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs) #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16) @@ -60,6 +62,7 @@ static u32 kvm_pmu_get_perf_event_type(unsigned long eidx) type = PERF_TYPE_HW_CACHE; break; case SBI_PMU_EVENT_TYPE_RAW: + case SBI_PMU_EVENT_TYPE_RAW_V2: case SBI_PMU_EVENT_TYPE_FW: type = PERF_TYPE_RAW; break; @@ -86,7 +89,8 @@ static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code) { - return hw_event_perf_map[sbi_event_code]; + return hw_event_perf_map[array_index_nospec(sbi_event_code, + SBI_PMU_HW_GENERAL_MAX)]; } static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code) @@ -128,6 +132,9 @@ static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data) case SBI_PMU_EVENT_TYPE_RAW: config = evt_data & RISCV_PMU_RAW_EVENT_MASK; break; + case SBI_PMU_EVENT_TYPE_RAW_V2: + config = evt_data & RISCV_PMU_RAW_EVENT_V2_MASK; + break; case SBI_PMU_EVENT_TYPE_FW: if (ecode < SBI_PMU_FW_MAX) config = (1ULL << 63) | ecode; @@ -214,12 +221,20 @@ static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[cidx]; if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW) return -EINVAL; + if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) + return -EINVAL; + fevent_code = get_event_code(pmc->event_idx); + if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, + "Invalid firmware event code: %d\n", fevent_code)) + return -EINVAL; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; *out_val = pmc->counter_val >> 32; @@ -240,10 +255,18 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[cidx]; if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { + if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID) + return -EINVAL; + fevent_code = get_event_code(pmc->event_idx); + if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX, + "Invalid firmware event code: %d\n", fevent_code)) + return -EINVAL; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; } else if (pmc->perf_event) { pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); @@ -258,8 +281,10 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base, unsigned long ctr_mask) { - /* Make sure the we have a valid counter mask requested from the caller */ - if (!ctr_mask || (ctr_base + __fls(ctr_mask) >= kvm_pmu_num_counters(kvpmu))) + unsigned long num_ctrs = kvm_pmu_num_counters(kvpmu); + + /* Make sure we have a valid counter mask requested from the caller */ + if (!ctr_mask || ctr_base >= num_ctrs || (ctr_base + __fls(ctr_mask) >= num_ctrs)) return -EINVAL; return 0; @@ -405,8 +430,6 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data); int sbiret = 0; gpa_t saddr; - unsigned long hva; - bool writable; if (!kvpmu || flags) { sbiret = SBI_ERR_INVALID_PARAM; @@ -421,26 +444,25 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s saddr = saddr_low; if (saddr_high != 0) { - if (IS_ENABLED(CONFIG_32BIT)) + if (IS_ENABLED(CONFIG_32BIT)) { saddr |= ((gpa_t)saddr_high << 32); - else + } else { sbiret = SBI_ERR_INVALID_ADDRESS; - goto out; + goto out; + } } - hva = kvm_vcpu_gfn_to_hva_prot(vcpu, saddr >> PAGE_SHIFT, &writable); - if (kvm_is_error_hva(hva) || !writable) { - sbiret = SBI_ERR_INVALID_ADDRESS; + kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC); + if (!kvpmu->sdata) { + sbiret = SBI_ERR_FAILURE; goto out; } - kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC); - if (!kvpmu->sdata) - return -ENOMEM; - + /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) { kfree(kvpmu->sdata); - sbiret = SBI_ERR_FAILURE; + kvpmu->sdata = NULL; + sbiret = SBI_ERR_INVALID_ADDRESS; goto out; } @@ -452,6 +474,64 @@ out: return 0; } +int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low, + unsigned long saddr_high, unsigned long num_events, + unsigned long flags, struct kvm_vcpu_sbi_return *retdata) +{ + struct riscv_pmu_event_info *einfo = NULL; + int shmem_size = num_events * sizeof(*einfo); + gpa_t shmem; + u32 eidx, etype; + u64 econfig; + int ret; + + if (flags != 0 || (saddr_low & (SZ_16 - 1) || num_events == 0)) { + ret = SBI_ERR_INVALID_PARAM; + goto out; + } + + shmem = saddr_low; + if (saddr_high != 0) { + if (IS_ENABLED(CONFIG_32BIT)) { + shmem |= ((gpa_t)saddr_high << 32); + } else { + ret = SBI_ERR_INVALID_ADDRESS; + goto out; + } + } + + einfo = kzalloc(shmem_size, GFP_KERNEL); + if (!einfo) { + ret = SBI_ERR_FAILURE; + goto out; + } + + ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size); + if (ret) { + ret = SBI_ERR_FAILURE; + goto free_mem; + } + + for (int i = 0; i < num_events; i++) { + eidx = einfo[i].event_idx; + etype = kvm_pmu_get_perf_event_type(eidx); + econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data); + ret = riscv_pmu_get_event_info(etype, econfig, NULL); + einfo[i].output = (ret > 0) ? 1 : 0; + } + + ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size); + if (ret) + ret = SBI_ERR_INVALID_ADDRESS; + +free_mem: + kfree(einfo); +out: + retdata->err_val = ret; + + return 0; +} + int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu, struct kvm_vcpu_sbi_return *retdata) { @@ -467,11 +547,12 @@ int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); - if (cidx > RISCV_KVM_MAX_COUNTERS || cidx == 1) { + if (cidx >= RISCV_KVM_MAX_COUNTERS || cidx == 1) { retdata->err_val = SBI_ERR_INVALID_PARAM; return 0; } + cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS); retdata->out_val = kvpmu->pmc[cidx].cinfo.value; return 0; @@ -506,7 +587,8 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, } /* Start the counters that have been configured and requested by the guest */ for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { - pmc_index = i + ctr_base; + pmc_index = array_index_nospec(i + ctr_base, + RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) continue; /* The guest started the counter again. Reset the overflow status */ @@ -577,7 +659,8 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, /* Stop the counters that have been configured and requested by the guest */ for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { - pmc_index = i + ctr_base; + pmc_index = array_index_nospec(i + ctr_base, + RISCV_KVM_MAX_COUNTERS); if (!test_bit(pmc_index, kvpmu->pmc_in_use)) continue; pmc = &kvpmu->pmc[pmc_index]; @@ -615,7 +698,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); /* - * The counter and overflow indicies in the snapshot region are w.r.to + * The counter and overflow indices in the snapshot region are w.r.to * cbase. Modify the set bit in the counter mask instead of the pmc_index * which indicates the absolute counter index. */ @@ -666,6 +749,7 @@ int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_ba .type = etype, .size = sizeof(struct perf_event_attr), .pinned = true, + .disabled = true, /* * It should never reach here if the platform doesn't support the sscofpmf * extension as mode filtering won't work without it. @@ -707,6 +791,7 @@ int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_ba } } + ctr_idx = array_index_nospec(ctr_idx, RISCV_KVM_MAX_COUNTERS); pmc = &kvpmu->pmc[ctr_idx]; pmc->idx = ctr_idx; @@ -765,7 +850,7 @@ void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu) * filtering is available in the host. Otherwise, guest will always count * events while the execution is in hypervisor mode. */ - if (!riscv_isa_extension_available(NULL, SSCOFPMF)) + if (kvm_riscv_isa_check_host(SSCOFPMF)) return; ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs); diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index d1c83a77735e..46ab7b989432 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -79,6 +79,14 @@ static const struct kvm_riscv_sbi_extension_entry sbi_ext[] = { .ext_ptr = &vcpu_sbi_ext_sta, }, { + .ext_idx = KVM_RISCV_SBI_EXT_FWFT, + .ext_ptr = &vcpu_sbi_ext_fwft, + }, + { + .ext_idx = KVM_RISCV_SBI_EXT_MPXY, + .ext_ptr = &vcpu_sbi_ext_mpxy, + }, + { .ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL, .ext_ptr = &vcpu_sbi_ext_experimental, }, @@ -106,7 +114,7 @@ riscv_vcpu_get_sbi_ext(struct kvm_vcpu *vcpu, unsigned long idx) return sext; } -bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) +static bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) { struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; const struct kvm_riscv_sbi_extension_entry *sext; @@ -116,7 +124,9 @@ bool riscv_vcpu_supports_sbi_ext(struct kvm_vcpu *vcpu, int idx) return sext && scontext->ext_status[sext->ext_idx] != KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE; } -void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_riscv_vcpu_sbi_forward_handler(struct kvm_vcpu *vcpu, + struct kvm_run *run, + struct kvm_vcpu_sbi_return *retdata) { struct kvm_cpu_context *cp = &vcpu->arch.guest_context; @@ -133,6 +143,8 @@ void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run) run->riscv_sbi.args[5] = cp->a5; run->riscv_sbi.ret[0] = SBI_ERR_NOT_SUPPORTED; run->riscv_sbi.ret[1] = 0; + retdata->uexit = true; + return 0; } void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, @@ -143,9 +155,9 @@ void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, struct kvm_vcpu *tmp; kvm_for_each_vcpu(i, tmp, vcpu->kvm) { - spin_lock(&vcpu->arch.mp_state_lock); + spin_lock(&tmp->arch.mp_state_lock); WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); - spin_unlock(&vcpu->arch.mp_state_lock); + spin_unlock(&tmp->arch.mp_state_lock); } kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); @@ -156,6 +168,34 @@ void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, run->exit_reason = KVM_EXIT_SYSTEM_EVENT; } +void kvm_riscv_vcpu_sbi_request_reset(struct kvm_vcpu *vcpu, + unsigned long pc, unsigned long a1) +{ + spin_lock(&vcpu->arch.reset_state.lock); + vcpu->arch.reset_state.pc = pc; + vcpu->arch.reset_state.a1 = a1; + spin_unlock(&vcpu->arch.reset_state.lock); + + kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); +} + +void kvm_riscv_vcpu_sbi_load_reset_state(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; + struct kvm_vcpu_reset_state *reset_state = &vcpu->arch.reset_state; + + cntx->a0 = vcpu->vcpu_id; + + spin_lock(&vcpu->arch.reset_state.lock); + cntx->sepc = reset_state->pc; + cntx->a1 = reset_state->a1; + spin_unlock(&vcpu->arch.reset_state.lock); + + cntx->sstatus &= ~SR_SIE; + csr->vsatp = 0; +} + int kvm_riscv_vcpu_sbi_return(struct kvm_vcpu *vcpu, struct kvm_run *run) { struct kvm_cpu_context *cp = &vcpu->arch.guest_context; @@ -256,6 +296,31 @@ static int riscv_vcpu_get_sbi_ext_multi(struct kvm_vcpu *vcpu, return 0; } +int kvm_riscv_vcpu_reg_indices_sbi_ext(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + unsigned int n = 0; + + for (int i = 0; i < KVM_RISCV_SBI_EXT_MAX; i++) { + u64 size = IS_ENABLED(CONFIG_32BIT) ? + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; + u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | + KVM_REG_RISCV_SBI_SINGLE | i; + + if (!riscv_vcpu_supports_sbi_ext(vcpu, i)) + continue; + + if (uindices) { + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + + n++; + } + + return n; +} + int kvm_riscv_vcpu_set_reg_sbi_ext(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { @@ -332,64 +397,163 @@ int kvm_riscv_vcpu_get_reg_sbi_ext(struct kvm_vcpu *vcpu, return 0; } -int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, - const struct kvm_one_reg *reg) +int kvm_riscv_vcpu_reg_indices_sbi(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + unsigned long state_reg_count; + int i, j, rc, count = 0; + u64 reg; + + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + + if (!ext->get_state_reg_count || + scontext->ext_status[entry->ext_idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED) + continue; + + state_reg_count = ext->get_state_reg_count(vcpu); + if (!uindices) + goto skip_put_user; + + for (j = 0; j < state_reg_count; j++) { + if (ext->get_state_reg_id) { + rc = ext->get_state_reg_id(vcpu, j, ®); + if (rc) + return rc; + } else { + reg = KVM_REG_RISCV | + (IS_ENABLED(CONFIG_32BIT) ? + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64) | + KVM_REG_RISCV_SBI_STATE | + ext->state_reg_subtype | j; + } + + if (put_user(reg, uindices)) + return -EFAULT; + uindices++; + } + +skip_put_user: + count += state_reg_count; + } + + return count; +} + +static const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext_withstate(struct kvm_vcpu *vcpu, + unsigned long subtype) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + int i; + + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + + if (ext->get_state_reg_count && + ext->state_reg_subtype == subtype && + scontext->ext_status[entry->ext_idx] == KVM_RISCV_SBI_EXT_STATUS_ENABLED) + return ext; + } + + return NULL; +} + +int kvm_riscv_vcpu_set_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { unsigned long __user *uaddr = (unsigned long __user *)(unsigned long)reg->addr; unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_SBI_STATE); - unsigned long reg_subtype, reg_val; - - if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + const struct kvm_vcpu_sbi_extension *ext; + unsigned long reg_subtype; + void *reg_val; + u64 data64; + u32 data32; + u16 data16; + u8 data8; + + switch (KVM_REG_SIZE(reg->id)) { + case 1: + reg_val = &data8; + break; + case 2: + reg_val = &data16; + break; + case 4: + reg_val = &data32; + break; + case 8: + reg_val = &data64; + break; + default: return -EINVAL; + } - if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) + if (copy_from_user(reg_val, uaddr, KVM_REG_SIZE(reg->id))) return -EFAULT; reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; - switch (reg_subtype) { - case KVM_REG_RISCV_SBI_STA: - return kvm_riscv_vcpu_set_reg_sbi_sta(vcpu, reg_num, reg_val); - default: + ext = kvm_vcpu_sbi_find_ext_withstate(vcpu, reg_subtype); + if (!ext || !ext->set_state_reg) return -EINVAL; - } - return 0; + return ext->set_state_reg(vcpu, reg_num, KVM_REG_SIZE(reg->id), reg_val); } -int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, - const struct kvm_one_reg *reg) +int kvm_riscv_vcpu_get_reg_sbi(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { unsigned long __user *uaddr = (unsigned long __user *)(unsigned long)reg->addr; unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_RISCV_SBI_STATE); - unsigned long reg_subtype, reg_val; + const struct kvm_vcpu_sbi_extension *ext; + unsigned long reg_subtype; + void *reg_val; + u64 data64; + u32 data32; + u16 data16; + u8 data8; int ret; - if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + switch (KVM_REG_SIZE(reg->id)) { + case 1: + reg_val = &data8; + break; + case 2: + reg_val = &data16; + break; + case 4: + reg_val = &data32; + break; + case 8: + reg_val = &data64; + break; + default: return -EINVAL; + } reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; - switch (reg_subtype) { - case KVM_REG_RISCV_SBI_STA: - ret = kvm_riscv_vcpu_get_reg_sbi_sta(vcpu, reg_num, ®_val); - break; - default: + ext = kvm_vcpu_sbi_find_ext_withstate(vcpu, reg_subtype); + if (!ext || !ext->get_state_reg) return -EINVAL; - } + ret = ext->get_state_reg(vcpu, reg_num, KVM_REG_SIZE(reg->id), reg_val); if (ret) return ret; - if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) + if (copy_to_user(uaddr, reg_val, KVM_REG_SIZE(reg->id))) return -EFAULT; return 0; @@ -508,5 +672,54 @@ void kvm_riscv_vcpu_sbi_init(struct kvm_vcpu *vcpu) scontext->ext_status[idx] = ext->default_disabled ? KVM_RISCV_SBI_EXT_STATUS_DISABLED : KVM_RISCV_SBI_EXT_STATUS_ENABLED; + + if (ext->init && ext->init(vcpu) != 0) + scontext->ext_status[idx] = KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE; + } +} + +void kvm_riscv_vcpu_sbi_deinit(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + int idx, i; + + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + idx = entry->ext_idx; + + if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status)) + continue; + + if (scontext->ext_status[idx] == KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE || + !ext->deinit) + continue; + + ext->deinit(vcpu); + } +} + +void kvm_riscv_vcpu_sbi_reset(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_sbi_context *scontext = &vcpu->arch.sbi_context; + const struct kvm_riscv_sbi_extension_entry *entry; + const struct kvm_vcpu_sbi_extension *ext; + int idx, i; + + for (i = 0; i < ARRAY_SIZE(sbi_ext); i++) { + entry = &sbi_ext[i]; + ext = entry->ext_ptr; + idx = entry->ext_idx; + + if (idx < 0 || idx >= ARRAY_SIZE(scontext->ext_status)) + continue; + + if (scontext->ext_status[idx] != KVM_RISCV_SBI_EXT_STATUS_ENABLED || + !ext->reset) + continue; + + ext->reset(vcpu); } } diff --git a/arch/riscv/kvm/vcpu_sbi_base.c b/arch/riscv/kvm/vcpu_sbi_base.c index 5bc570b984f4..06fdd5f69364 100644 --- a/arch/riscv/kvm/vcpu_sbi_base.c +++ b/arch/riscv/kvm/vcpu_sbi_base.c @@ -41,8 +41,7 @@ static int kvm_sbi_ext_base_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, * For experimental/vendor extensions * forward it to the userspace */ - kvm_riscv_vcpu_sbi_forward(vcpu, run); - retdata->uexit = true; + return kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata); } else { sbi_ext = kvm_vcpu_sbi_find_ext(vcpu, cp->a0); *out_val = sbi_ext && sbi_ext->probe ? @@ -71,28 +70,3 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_base = { .extid_end = SBI_EXT_BASE, .handler = kvm_sbi_ext_base_handler, }; - -static int kvm_sbi_ext_forward_handler(struct kvm_vcpu *vcpu, - struct kvm_run *run, - struct kvm_vcpu_sbi_return *retdata) -{ - /* - * Both SBI experimental and vendor extensions are - * unconditionally forwarded to userspace. - */ - kvm_riscv_vcpu_sbi_forward(vcpu, run); - retdata->uexit = true; - return 0; -} - -const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental = { - .extid_start = SBI_EXT_EXPERIMENTAL_START, - .extid_end = SBI_EXT_EXPERIMENTAL_END, - .handler = kvm_sbi_ext_forward_handler, -}; - -const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor = { - .extid_start = SBI_EXT_VENDOR_START, - .extid_end = SBI_EXT_VENDOR_END, - .handler = kvm_sbi_ext_forward_handler, -}; diff --git a/arch/riscv/kvm/vcpu_sbi_forward.c b/arch/riscv/kvm/vcpu_sbi_forward.c new file mode 100644 index 000000000000..5a3c75eb23c5 --- /dev/null +++ b/arch/riscv/kvm/vcpu_sbi_forward.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025 Ventana Micro Systems Inc. + */ + +#include <linux/kvm_host.h> +#include <asm/kvm_vcpu_sbi.h> +#include <asm/sbi.h> + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental = { + .extid_start = SBI_EXT_EXPERIMENTAL_START, + .extid_end = SBI_EXT_EXPERIMENTAL_END, + .handler = kvm_riscv_vcpu_sbi_forward_handler, +}; + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor = { + .extid_start = SBI_EXT_VENDOR_START, + .extid_end = SBI_EXT_VENDOR_END, + .handler = kvm_riscv_vcpu_sbi_forward_handler, +}; + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn = { + .extid_start = SBI_EXT_DBCN, + .extid_end = SBI_EXT_DBCN, + .default_disabled = true, + .handler = kvm_riscv_vcpu_sbi_forward_handler, +}; + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_mpxy = { + .extid_start = SBI_EXT_MPXY, + .extid_end = SBI_EXT_MPXY, + .default_disabled = true, + .handler = kvm_riscv_vcpu_sbi_forward_handler, +}; diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c new file mode 100644 index 000000000000..2eab15339694 --- /dev/null +++ b/arch/riscv/kvm/vcpu_sbi_fwft.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025 Rivos Inc. + * + * Authors: + * Clément Léger <cleger@rivosinc.com> + */ + +#include <linux/errno.h> +#include <linux/err.h> +#include <linux/kvm_host.h> +#include <asm/cpufeature.h> +#include <asm/sbi.h> +#include <asm/kvm_vcpu_sbi.h> +#include <asm/kvm_vcpu_sbi_fwft.h> + +#define MIS_DELEG (BIT_ULL(EXC_LOAD_MISALIGNED) | BIT_ULL(EXC_STORE_MISALIGNED)) + +struct kvm_sbi_fwft_feature { + /** + * @id: Feature ID + */ + enum sbi_fwft_feature_t id; + + /** + * @first_reg_num: ONE_REG index of the first ONE_REG register + */ + unsigned long first_reg_num; + + /** + * @supported: Check if the feature is supported on the vcpu + * + * This callback is optional, if not provided the feature is assumed to + * be supported + */ + bool (*supported)(struct kvm_vcpu *vcpu); + + /** + * @reset: Reset the feature value irrespective whether feature is supported or not + * + * This callback is mandatory + */ + void (*reset)(struct kvm_vcpu *vcpu); + + /** + * @set: Set the feature value + * + * Return SBI_SUCCESS on success or an SBI error (SBI_ERR_*) + * + * This callback is mandatory + */ + long (*set)(struct kvm_vcpu *vcpu, struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value); + + /** + * @get: Get the feature current value + * + * Return SBI_SUCCESS on success or an SBI error (SBI_ERR_*) + * + * This callback is mandatory + */ + long (*get)(struct kvm_vcpu *vcpu, struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value); +}; + +static const enum sbi_fwft_feature_t kvm_fwft_defined_features[] = { + SBI_FWFT_MISALIGNED_EXC_DELEG, + SBI_FWFT_LANDING_PAD, + SBI_FWFT_SHADOW_STACK, + SBI_FWFT_DOUBLE_TRAP, + SBI_FWFT_PTE_AD_HW_UPDATING, + SBI_FWFT_POINTER_MASKING_PMLEN, +}; + +static bool kvm_fwft_is_defined_feature(enum sbi_fwft_feature_t feature) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kvm_fwft_defined_features); i++) { + if (kvm_fwft_defined_features[i] == feature) + return true; + } + + return false; +} + +static bool kvm_sbi_fwft_misaligned_delegation_supported(struct kvm_vcpu *vcpu) +{ + return misaligned_traps_can_delegate(); +} + +static void kvm_sbi_fwft_reset_misaligned_delegation(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + cfg->hedeleg &= ~MIS_DELEG; +} + +static long kvm_sbi_fwft_set_misaligned_delegation(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + if (value == 1) { + cfg->hedeleg |= MIS_DELEG; + if (!one_reg_access) + csr_set(CSR_HEDELEG, MIS_DELEG); + } else if (value == 0) { + cfg->hedeleg &= ~MIS_DELEG; + if (!one_reg_access) + csr_clear(CSR_HEDELEG, MIS_DELEG); + } else { + return SBI_ERR_INVALID_PARAM; + } + + return SBI_SUCCESS; +} + +static long kvm_sbi_fwft_get_misaligned_delegation(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value) +{ + struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; + + *value = (cfg->hedeleg & MIS_DELEG) == MIS_DELEG; + return SBI_SUCCESS; +} + +#ifndef CONFIG_32BIT + +static bool try_to_set_pmm(unsigned long value) +{ + csr_set(CSR_HENVCFG, value); + return (csr_read_clear(CSR_HENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value; +} + +static bool kvm_sbi_fwft_pointer_masking_pmlen_supported(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + + if (!riscv_isa_extension_available(vcpu->arch.isa, SMNPM)) + return false; + + fwft->have_vs_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7); + fwft->have_vs_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16); + + return fwft->have_vs_pmlen_7 || fwft->have_vs_pmlen_16; +} + +static void kvm_sbi_fwft_reset_pointer_masking_pmlen(struct kvm_vcpu *vcpu) +{ + vcpu->arch.cfg.henvcfg &= ~ENVCFG_PMM; +} + +static long kvm_sbi_fwft_set_pointer_masking_pmlen(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long value) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + unsigned long pmm; + + switch (value) { + case 0: + pmm = ENVCFG_PMM_PMLEN_0; + break; + case 7: + if (!fwft->have_vs_pmlen_7) + return SBI_ERR_INVALID_PARAM; + pmm = ENVCFG_PMM_PMLEN_7; + break; + case 16: + if (!fwft->have_vs_pmlen_16) + return SBI_ERR_INVALID_PARAM; + pmm = ENVCFG_PMM_PMLEN_16; + break; + default: + return SBI_ERR_INVALID_PARAM; + } + + vcpu->arch.cfg.henvcfg &= ~ENVCFG_PMM; + vcpu->arch.cfg.henvcfg |= pmm; + + /* + * Instead of waiting for vcpu_load/put() to update HENVCFG CSR, + * update here so that VCPU see's pointer masking mode change + * immediately. + */ + if (!one_reg_access) + csr_write(CSR_HENVCFG, vcpu->arch.cfg.henvcfg); + + return SBI_SUCCESS; +} + +static long kvm_sbi_fwft_get_pointer_masking_pmlen(struct kvm_vcpu *vcpu, + struct kvm_sbi_fwft_config *conf, + bool one_reg_access, unsigned long *value) +{ + switch (vcpu->arch.cfg.henvcfg & ENVCFG_PMM) { + case ENVCFG_PMM_PMLEN_0: + *value = 0; + break; + case ENVCFG_PMM_PMLEN_7: + *value = 7; + break; + case ENVCFG_PMM_PMLEN_16: + *value = 16; + break; + default: + return SBI_ERR_FAILURE; + } + + return SBI_SUCCESS; +} + +#endif + +static const struct kvm_sbi_fwft_feature features[] = { + { + .id = SBI_FWFT_MISALIGNED_EXC_DELEG, + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, misaligned_deleg.enable) / + sizeof(unsigned long), + .supported = kvm_sbi_fwft_misaligned_delegation_supported, + .reset = kvm_sbi_fwft_reset_misaligned_delegation, + .set = kvm_sbi_fwft_set_misaligned_delegation, + .get = kvm_sbi_fwft_get_misaligned_delegation, + }, +#ifndef CONFIG_32BIT + { + .id = SBI_FWFT_POINTER_MASKING_PMLEN, + .first_reg_num = offsetof(struct kvm_riscv_sbi_fwft, pointer_masking.enable) / + sizeof(unsigned long), + .supported = kvm_sbi_fwft_pointer_masking_pmlen_supported, + .reset = kvm_sbi_fwft_reset_pointer_masking_pmlen, + .set = kvm_sbi_fwft_set_pointer_masking_pmlen, + .get = kvm_sbi_fwft_get_pointer_masking_pmlen, + }, +#endif +}; + +static const struct kvm_sbi_fwft_feature *kvm_sbi_fwft_regnum_to_feature(unsigned long reg_num) +{ + const struct kvm_sbi_fwft_feature *feature; + int i; + + for (i = 0; i < ARRAY_SIZE(features); i++) { + feature = &features[i]; + if (feature->first_reg_num <= reg_num && reg_num < (feature->first_reg_num + 3)) + return feature; + } + + return NULL; +} + +static struct kvm_sbi_fwft_config * +kvm_sbi_fwft_get_config(struct kvm_vcpu *vcpu, enum sbi_fwft_feature_t feature) +{ + int i; + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + + for (i = 0; i < ARRAY_SIZE(features); i++) { + if (fwft->configs[i].feature->id == feature) + return &fwft->configs[i]; + } + + return NULL; +} + +static int kvm_fwft_get_feature(struct kvm_vcpu *vcpu, u32 feature, + struct kvm_sbi_fwft_config **conf) +{ + struct kvm_sbi_fwft_config *tconf; + + tconf = kvm_sbi_fwft_get_config(vcpu, feature); + if (!tconf) { + if (kvm_fwft_is_defined_feature(feature)) + return SBI_ERR_NOT_SUPPORTED; + + return SBI_ERR_DENIED; + } + + if (!tconf->supported || !tconf->enabled) + return SBI_ERR_NOT_SUPPORTED; + + *conf = tconf; + + return SBI_SUCCESS; +} + +static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature, + unsigned long value, unsigned long flags) +{ + int ret; + struct kvm_sbi_fwft_config *conf; + + ret = kvm_fwft_get_feature(vcpu, feature, &conf); + if (ret) + return ret; + + if ((flags & ~SBI_FWFT_SET_FLAG_LOCK) != 0) + return SBI_ERR_INVALID_PARAM; + + if (conf->flags & SBI_FWFT_SET_FLAG_LOCK) + return SBI_ERR_DENIED_LOCKED; + + conf->flags = flags; + + return conf->feature->set(vcpu, conf, false, value); +} + +static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature, + unsigned long *value) +{ + int ret; + struct kvm_sbi_fwft_config *conf; + + ret = kvm_fwft_get_feature(vcpu, feature, &conf); + if (ret) + return ret; + + return conf->feature->get(vcpu, conf, false, value); +} + +static int kvm_sbi_ext_fwft_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct kvm_vcpu_sbi_return *retdata) +{ + int ret; + struct kvm_cpu_context *cp = &vcpu->arch.guest_context; + unsigned long funcid = cp->a6; + + switch (funcid) { + case SBI_EXT_FWFT_SET: + ret = kvm_sbi_fwft_set(vcpu, cp->a0, cp->a1, cp->a2); + break; + case SBI_EXT_FWFT_GET: + ret = kvm_sbi_fwft_get(vcpu, cp->a0, &retdata->out_val); + break; + default: + ret = SBI_ERR_NOT_SUPPORTED; + break; + } + + retdata->err_val = ret; + + return 0; +} + +static int kvm_sbi_ext_fwft_init(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + int i; + + fwft->configs = kzalloc_objs(struct kvm_sbi_fwft_config, + ARRAY_SIZE(features)); + if (!fwft->configs) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(features); i++) { + feature = &features[i]; + conf = &fwft->configs[i]; + if (feature->supported) + conf->supported = feature->supported(vcpu); + else + conf->supported = true; + + conf->enabled = conf->supported; + conf->feature = feature; + } + + return 0; +} + +static void kvm_sbi_ext_fwft_deinit(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + + kfree(fwft->configs); +} + +static void kvm_sbi_ext_fwft_reset(struct kvm_vcpu *vcpu) +{ + struct kvm_sbi_fwft *fwft = vcpu_to_fwft(vcpu); + const struct kvm_sbi_fwft_feature *feature; + int i; + + for (i = 0; i < ARRAY_SIZE(features); i++) { + fwft->configs[i].flags = 0; + feature = &features[i]; + if (feature->reset) + feature->reset(vcpu); + } +} + +static unsigned long kvm_sbi_ext_fwft_get_reg_count(struct kvm_vcpu *vcpu) +{ + unsigned long max_reg_count = sizeof(struct kvm_riscv_sbi_fwft) / sizeof(unsigned long); + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + unsigned long reg, ret = 0; + + for (reg = 0; reg < max_reg_count; reg++) { + feature = kvm_sbi_fwft_regnum_to_feature(reg); + if (!feature) + continue; + + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); + if (!conf || !conf->supported) + continue; + + ret++; + } + + return ret; +} + +static int kvm_sbi_ext_fwft_get_reg_id(struct kvm_vcpu *vcpu, int index, u64 *reg_id) +{ + int reg, max_reg_count = sizeof(struct kvm_riscv_sbi_fwft) / sizeof(unsigned long); + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + int idx = 0; + + for (reg = 0; reg < max_reg_count; reg++) { + feature = kvm_sbi_fwft_regnum_to_feature(reg); + if (!feature) + continue; + + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); + if (!conf || !conf->supported) + continue; + + if (index == idx) { + *reg_id = KVM_REG_RISCV | + (IS_ENABLED(CONFIG_32BIT) ? + KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64) | + KVM_REG_RISCV_SBI_STATE | + KVM_REG_RISCV_SBI_FWFT | reg; + return 0; + } + + idx++; + } + + return -ENOENT; +} + +static int kvm_sbi_ext_fwft_get_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, void *reg_val) +{ + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + unsigned long *value; + int ret = 0; + + if (reg_size != sizeof(unsigned long)) + return -EINVAL; + value = reg_val; + + feature = kvm_sbi_fwft_regnum_to_feature(reg_num); + if (!feature) + return -ENOENT; + + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); + if (!conf || !conf->supported) + return -ENOENT; + + switch (reg_num - feature->first_reg_num) { + case 0: + *value = conf->enabled; + break; + case 1: + *value = conf->flags; + break; + case 2: + ret = conf->feature->get(vcpu, conf, true, value); + break; + default: + return -ENOENT; + } + + return sbi_err_map_linux_errno(ret); +} + +static int kvm_sbi_ext_fwft_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, const void *reg_val) +{ + const struct kvm_sbi_fwft_feature *feature; + struct kvm_sbi_fwft_config *conf; + unsigned long value; + int ret = 0; + + if (reg_size != sizeof(unsigned long)) + return -EINVAL; + value = *(const unsigned long *)reg_val; + + feature = kvm_sbi_fwft_regnum_to_feature(reg_num); + if (!feature) + return -ENOENT; + + conf = kvm_sbi_fwft_get_config(vcpu, feature->id); + if (!conf || !conf->supported) + return -ENOENT; + + switch (reg_num - feature->first_reg_num) { + case 0: + switch (value) { + case 0: + conf->enabled = false; + break; + case 1: + conf->enabled = true; + break; + default: + return -EINVAL; + } + break; + case 1: + conf->flags = value & SBI_FWFT_SET_FLAG_LOCK; + break; + case 2: + ret = conf->feature->set(vcpu, conf, true, value); + break; + default: + return -ENOENT; + } + + return sbi_err_map_linux_errno(ret); +} + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft = { + .extid_start = SBI_EXT_FWFT, + .extid_end = SBI_EXT_FWFT, + .handler = kvm_sbi_ext_fwft_handler, + .init = kvm_sbi_ext_fwft_init, + .deinit = kvm_sbi_ext_fwft_deinit, + .reset = kvm_sbi_ext_fwft_reset, + .state_reg_subtype = KVM_REG_RISCV_SBI_FWFT, + .get_state_reg_count = kvm_sbi_ext_fwft_get_reg_count, + .get_state_reg_id = kvm_sbi_ext_fwft_get_reg_id, + .get_state_reg = kvm_sbi_ext_fwft_get_reg, + .set_state_reg = kvm_sbi_ext_fwft_set_reg, +}; diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index dce667f4b6ab..f26207f84bab 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -9,12 +9,12 @@ #include <linux/errno.h> #include <linux/err.h> #include <linux/kvm_host.h> +#include <linux/wordpart.h> #include <asm/sbi.h> #include <asm/kvm_vcpu_sbi.h> static int kvm_sbi_hsm_vcpu_start(struct kvm_vcpu *vcpu) { - struct kvm_cpu_context *reset_cntx; struct kvm_cpu_context *cp = &vcpu->arch.guest_context; struct kvm_vcpu *target_vcpu; unsigned long target_vcpuid = cp->a0; @@ -31,17 +31,7 @@ static int kvm_sbi_hsm_vcpu_start(struct kvm_vcpu *vcpu) goto out; } - spin_lock(&target_vcpu->arch.reset_cntx_lock); - reset_cntx = &target_vcpu->arch.guest_reset_context; - /* start address */ - reset_cntx->sepc = cp->a1; - /* target vcpu id to start */ - reset_cntx->a0 = target_vcpuid; - /* private data passed from kernel */ - reset_cntx->a1 = cp->a2; - spin_unlock(&target_vcpu->arch.reset_cntx_lock); - - kvm_make_request(KVM_REQ_VCPU_RESET, target_vcpu); + kvm_riscv_vcpu_sbi_request_reset(target_vcpu, cp->a1, cp->a2); __kvm_riscv_vcpu_power_on(target_vcpu); @@ -79,12 +69,12 @@ static int kvm_sbi_hsm_vcpu_get_status(struct kvm_vcpu *vcpu) target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, target_vcpuid); if (!target_vcpu) return SBI_ERR_INVALID_PARAM; - if (!kvm_riscv_vcpu_stopped(target_vcpu)) - return SBI_HSM_STATE_STARTED; - else if (vcpu->stat.generic.blocking) + if (kvm_riscv_vcpu_stopped(target_vcpu)) + return SBI_HSM_STATE_STOPPED; + else if (target_vcpu->stat.generic.blocking) return SBI_HSM_STATE_SUSPENDED; else - return SBI_HSM_STATE_STOPPED; + return SBI_HSM_STATE_STARTED; } static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, @@ -109,7 +99,7 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, } return 0; case SBI_EXT_HSM_HART_SUSPEND: - switch (cp->a0) { + switch (lower_32_bits(cp->a0)) { case SBI_HSM_SUSPEND_RET_DEFAULT: kvm_riscv_vcpu_wfi(vcpu); break; diff --git a/arch/riscv/kvm/vcpu_sbi_pmu.c b/arch/riscv/kvm/vcpu_sbi_pmu.c index e4be34e03e83..a020d979d179 100644 --- a/arch/riscv/kvm/vcpu_sbi_pmu.c +++ b/arch/riscv/kvm/vcpu_sbi_pmu.c @@ -73,6 +73,9 @@ static int kvm_sbi_ext_pmu_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, case SBI_EXT_PMU_SNAPSHOT_SET_SHMEM: ret = kvm_riscv_vcpu_pmu_snapshot_set_shmem(vcpu, cp->a0, cp->a1, cp->a2, retdata); break; + case SBI_EXT_PMU_EVENT_GET_INFO: + ret = kvm_riscv_vcpu_pmu_event_info(vcpu, cp->a0, cp->a1, cp->a2, cp->a3, retdata); + break; default: retdata->err_val = SBI_ERR_NOT_SUPPORTED; } diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 9c2ab3dfa93a..506a510b6bff 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -21,7 +21,7 @@ static int kvm_sbi_ext_time_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, u64 next_cycle; if (cp->a6 != SBI_EXT_TIME_SET_TIMER) { - retdata->err_val = SBI_ERR_INVALID_PARAM; + retdata->err_val = SBI_ERR_NOT_SUPPORTED; return 0; } @@ -51,9 +51,10 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_context *cp = &vcpu->arch.guest_context; unsigned long hmask = cp->a0; unsigned long hbase = cp->a1; + unsigned long hart_bit = 0, sentmask = 0; if (cp->a6 != SBI_EXT_IPI_SEND_IPI) { - retdata->err_val = SBI_ERR_INVALID_PARAM; + retdata->err_val = SBI_ERR_NOT_SUPPORTED; return 0; } @@ -62,15 +63,23 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (hbase != -1UL) { if (tmp->vcpu_id < hbase) continue; - if (!(hmask & (1UL << (tmp->vcpu_id - hbase)))) + hart_bit = tmp->vcpu_id - hbase; + if (hart_bit >= __riscv_xlen) + goto done; + if (!(hmask & (1UL << hart_bit))) continue; } ret = kvm_riscv_vcpu_set_interrupt(tmp, IRQ_VS_SOFT); if (ret < 0) break; + sentmask |= 1UL << hart_bit; kvm_riscv_vcpu_pmu_incr_fw(tmp, SBI_PMU_FW_IPI_RCVD); } +done: + if (hbase != -1UL && (hmask ^ sentmask)) + retdata->err_val = SBI_ERR_INVALID_PARAM; + return ret; } @@ -87,6 +96,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run unsigned long hmask = cp->a0; unsigned long hbase = cp->a1; unsigned long funcid = cp->a6; + unsigned long vmid; switch (funcid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: @@ -94,22 +104,22 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: - if (cp->a2 == 0 && cp->a3 == 0) - kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask); + vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) + kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid); else kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, - cp->a2, cp->a3, PAGE_SHIFT); + cp->a2, cp->a3, PAGE_SHIFT, vmid); kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: - if (cp->a2 == 0 && cp->a3 == 0) - kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, - hbase, hmask, cp->a4); + vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) + kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask, + cp->a4, vmid); else - kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, - hbase, hmask, - cp->a2, cp->a3, - PAGE_SHIFT, cp->a4); + kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, hbase, hmask, cp->a2, + cp->a3, PAGE_SHIFT, cp->a4, vmid); kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_SENT); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: @@ -118,9 +128,9 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: /* * Until nested virtualization is implemented, the - * SBI HFENCE calls should be treated as NOPs + * SBI HFENCE calls should return not supported + * hence fallthrough. */ - break; default: retdata->err_val = SBI_ERR_NOT_SUPPORTED; } @@ -175,35 +185,3 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_srst = { .extid_end = SBI_EXT_SRST, .handler = kvm_sbi_ext_srst_handler, }; - -static int kvm_sbi_ext_dbcn_handler(struct kvm_vcpu *vcpu, - struct kvm_run *run, - struct kvm_vcpu_sbi_return *retdata) -{ - struct kvm_cpu_context *cp = &vcpu->arch.guest_context; - unsigned long funcid = cp->a6; - - switch (funcid) { - case SBI_EXT_DBCN_CONSOLE_WRITE: - case SBI_EXT_DBCN_CONSOLE_READ: - case SBI_EXT_DBCN_CONSOLE_WRITE_BYTE: - /* - * The SBI debug console functions are unconditionally - * forwarded to the userspace. - */ - kvm_riscv_vcpu_sbi_forward(vcpu, run); - retdata->uexit = true; - break; - default: - retdata->err_val = SBI_ERR_NOT_SUPPORTED; - } - - return 0; -} - -const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn = { - .extid_start = SBI_EXT_DBCN, - .extid_end = SBI_EXT_DBCN, - .default_disabled = true, - .handler = kvm_sbi_ext_dbcn_handler, -}; diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index 5f35427114c1..60e50296a008 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -16,7 +16,7 @@ #include <asm/sbi.h> #include <asm/uaccess.h> -void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu) +static void kvm_riscv_vcpu_sbi_sta_reset(struct kvm_vcpu *vcpu) { vcpu->arch.sta.shmem = INVALID_GPA; vcpu->arch.sta.last_steal = 0; @@ -46,7 +46,7 @@ void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) gfn = shmem >> PAGE_SHIFT; hva = kvm_vcpu_gfn_to_hva(vcpu, gfn); - if (WARN_ON(kvm_is_error_hva(hva))) { + if (kvm_is_error_hva(hva)) { vcpu->arch.sta.shmem = INVALID_GPA; return; } @@ -85,8 +85,6 @@ static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) unsigned long shmem_phys_hi = cp->a1; u32 flags = cp->a2; struct sbi_sta_struct zero_sta = {0}; - unsigned long hva; - bool writable; gpa_t shmem; int ret; @@ -111,13 +109,10 @@ static int kvm_sbi_sta_steal_time_set_shmem(struct kvm_vcpu *vcpu) return SBI_ERR_INVALID_ADDRESS; } - hva = kvm_vcpu_gfn_to_hva_prot(vcpu, shmem >> PAGE_SHIFT, &writable); - if (kvm_is_error_hva(hva) || !writable) - return SBI_ERR_INVALID_ADDRESS; - + /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */ ret = kvm_vcpu_write_guest(vcpu, shmem, &zero_sta, sizeof(zero_sta)); if (ret) - return SBI_ERR_FAILURE; + return SBI_ERR_INVALID_ADDRESS; vcpu->arch.sta.shmem = shmem; vcpu->arch.sta.last_steal = current->sched_info.run_delay; @@ -151,62 +146,88 @@ static unsigned long kvm_sbi_ext_sta_probe(struct kvm_vcpu *vcpu) return !!sched_info_on(); } -const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { - .extid_start = SBI_EXT_STA, - .extid_end = SBI_EXT_STA, - .handler = kvm_sbi_ext_sta_handler, - .probe = kvm_sbi_ext_sta_probe, -}; +static unsigned long kvm_sbi_ext_sta_get_state_reg_count(struct kvm_vcpu *vcpu) +{ + return sizeof(struct kvm_riscv_sbi_sta) / sizeof(unsigned long); +} -int kvm_riscv_vcpu_get_reg_sbi_sta(struct kvm_vcpu *vcpu, - unsigned long reg_num, - unsigned long *reg_val) +static int kvm_sbi_ext_sta_get_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, void *reg_val) { + unsigned long *value; + + if (reg_size != sizeof(unsigned long)) + return -EINVAL; + value = reg_val; + switch (reg_num) { case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): - *reg_val = (unsigned long)vcpu->arch.sta.shmem; + *value = (unsigned long)vcpu->arch.sta.shmem; break; case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): if (IS_ENABLED(CONFIG_32BIT)) - *reg_val = upper_32_bits(vcpu->arch.sta.shmem); + *value = upper_32_bits(vcpu->arch.sta.shmem); else - *reg_val = 0; + *value = 0; break; default: - return -EINVAL; + return -ENOENT; } return 0; } -int kvm_riscv_vcpu_set_reg_sbi_sta(struct kvm_vcpu *vcpu, - unsigned long reg_num, - unsigned long reg_val) +static int kvm_sbi_ext_sta_set_reg(struct kvm_vcpu *vcpu, unsigned long reg_num, + unsigned long reg_size, const void *reg_val) { + unsigned long value; + gpa_t new_shmem = INVALID_GPA; + + if (reg_size != sizeof(unsigned long)) + return -EINVAL; + value = *(const unsigned long *)reg_val; + switch (reg_num) { case KVM_REG_RISCV_SBI_STA_REG(shmem_lo): if (IS_ENABLED(CONFIG_32BIT)) { gpa_t hi = upper_32_bits(vcpu->arch.sta.shmem); - vcpu->arch.sta.shmem = reg_val; - vcpu->arch.sta.shmem |= hi << 32; + new_shmem = value; + new_shmem |= hi << 32; } else { - vcpu->arch.sta.shmem = reg_val; + new_shmem = value; } break; case KVM_REG_RISCV_SBI_STA_REG(shmem_hi): if (IS_ENABLED(CONFIG_32BIT)) { gpa_t lo = lower_32_bits(vcpu->arch.sta.shmem); - vcpu->arch.sta.shmem = ((gpa_t)reg_val << 32); - vcpu->arch.sta.shmem |= lo; - } else if (reg_val != 0) { + new_shmem = ((gpa_t)value << 32); + new_shmem |= lo; + } else if (value != 0) { return -EINVAL; } break; default: - return -EINVAL; + return -ENOENT; } + if (new_shmem != INVALID_GPA && !IS_ALIGNED(new_shmem, 64)) + return -EINVAL; + + vcpu->arch.sta.shmem = new_shmem; + return 0; } + +const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta = { + .extid_start = SBI_EXT_STA, + .extid_end = SBI_EXT_STA, + .handler = kvm_sbi_ext_sta_handler, + .probe = kvm_sbi_ext_sta_probe, + .reset = kvm_riscv_vcpu_sbi_sta_reset, + .state_reg_subtype = KVM_REG_RISCV_SBI_STA, + .get_state_reg_count = kvm_sbi_ext_sta_get_state_reg_count, + .get_state_reg = kvm_sbi_ext_sta_get_reg, + .set_state_reg = kvm_sbi_ext_sta_set_reg, +}; diff --git a/arch/riscv/kvm/vcpu_sbi_system.c b/arch/riscv/kvm/vcpu_sbi_system.c index 5d55e08791fa..c6f7e609ac79 100644 --- a/arch/riscv/kvm/vcpu_sbi_system.c +++ b/arch/riscv/kvm/vcpu_sbi_system.c @@ -4,6 +4,7 @@ */ #include <linux/kvm_host.h> +#include <linux/wordpart.h> #include <asm/kvm_vcpu_sbi.h> #include <asm/sbi.h> @@ -12,14 +13,13 @@ static int kvm_sbi_ext_susp_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_vcpu_sbi_return *retdata) { struct kvm_cpu_context *cp = &vcpu->arch.guest_context; - struct kvm_cpu_context *reset_cntx; unsigned long funcid = cp->a6; unsigned long hva, i; struct kvm_vcpu *tmp; switch (funcid) { case SBI_EXT_SUSP_SYSTEM_SUSPEND: - if (cp->a0 != SBI_SUSP_SLEEP_TYPE_SUSPEND_TO_RAM) { + if (lower_32_bits(cp->a0) != SBI_SUSP_SLEEP_TYPE_SUSPEND_TO_RAM) { retdata->err_val = SBI_ERR_INVALID_PARAM; return 0; } @@ -44,19 +44,10 @@ static int kvm_sbi_ext_susp_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, } } - spin_lock(&vcpu->arch.reset_cntx_lock); - reset_cntx = &vcpu->arch.guest_reset_context; - reset_cntx->sepc = cp->a1; - reset_cntx->a0 = vcpu->vcpu_id; - reset_cntx->a1 = cp->a2; - spin_unlock(&vcpu->arch.reset_cntx_lock); - - kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); + kvm_riscv_vcpu_sbi_request_reset(vcpu, cp->a1, cp->a2); /* userspace provides the suspend implementation */ - kvm_riscv_vcpu_sbi_forward(vcpu, run); - retdata->uexit = true; - break; + return kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata); default: retdata->err_val = SBI_ERR_NOT_SUPPORTED; break; diff --git a/arch/riscv/kvm/vcpu_sbi_v01.c b/arch/riscv/kvm/vcpu_sbi_v01.c index 8f4c4fa16227..c9c323d4577a 100644 --- a/arch/riscv/kvm/vcpu_sbi_v01.c +++ b/arch/riscv/kvm/vcpu_sbi_v01.c @@ -23,6 +23,7 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm *kvm = vcpu->kvm; struct kvm_cpu_context *cp = &vcpu->arch.guest_context; struct kvm_cpu_trap *utrap = retdata->utrap; + unsigned long vmid; switch (cp->a7) { case SBI_EXT_0_1_CONSOLE_GETCHAR: @@ -31,8 +32,7 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, * The CONSOLE_GETCHAR/CONSOLE_PUTCHAR SBI calls cannot be * handled in kernel so we forward these to user-space */ - kvm_riscv_vcpu_sbi_forward(vcpu, run); - retdata->uexit = true; + ret = kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata); break; case SBI_EXT_0_1_SET_TIMER: #if __riscv_xlen == 32 @@ -55,6 +55,8 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, for_each_set_bit(i, &hmask, BITS_PER_LONG) { rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i); + if (!rvcpu) + continue; ret = kvm_riscv_vcpu_set_interrupt(rvcpu, IRQ_VS_SOFT); if (ret < 0) break; @@ -78,25 +80,21 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I) kvm_riscv_fence_i(vcpu->kvm, 0, hmask); else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) { + vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); if (cp->a1 == 0 && cp->a2 == 0) - kvm_riscv_hfence_vvma_all(vcpu->kvm, - 0, hmask); + kvm_riscv_hfence_vvma_all(vcpu->kvm, 0, hmask, vmid); else - kvm_riscv_hfence_vvma_gva(vcpu->kvm, - 0, hmask, - cp->a1, cp->a2, - PAGE_SHIFT); + kvm_riscv_hfence_vvma_gva(vcpu->kvm, 0, hmask, cp->a1, + cp->a2, PAGE_SHIFT, vmid); } else { + vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); if (cp->a1 == 0 && cp->a2 == 0) - kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, - 0, hmask, - cp->a3); + kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, 0, hmask, + cp->a3, vmid); else - kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, - 0, hmask, - cp->a1, cp->a2, - PAGE_SHIFT, - cp->a3); + kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, 0, hmask, + cp->a1, cp->a2, PAGE_SHIFT, + cp->a3, vmid); } break; default: diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index 96e7a4e463f7..9817ff802821 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -12,6 +12,7 @@ #include <linux/uaccess.h> #include <clocksource/timer-riscv.h> #include <asm/delay.h> +#include <asm/kvm_isa.h> #include <asm/kvm_nacl.h> #include <asm/kvm_vcpu_timer.h> @@ -72,8 +73,9 @@ static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) static int kvm_riscv_vcpu_update_vstimecmp(struct kvm_vcpu *vcpu, u64 ncycles) { #if defined(CONFIG_32BIT) - ncsr_write(CSR_VSTIMECMP, ncycles & 0xFFFFFFFF); + ncsr_write(CSR_VSTIMECMP, ULONG_MAX); ncsr_write(CSR_VSTIMECMPH, ncycles >> 32); + ncsr_write(CSR_VSTIMECMP, (u32)ncycles); #else ncsr_write(CSR_VSTIMECMP, ncycles); #endif @@ -248,18 +250,19 @@ int kvm_riscv_vcpu_timer_init(struct kvm_vcpu *vcpu) if (t->init_done) return -EINVAL; - hrtimer_init(&t->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_REL); t->init_done = true; t->next_set = false; /* Enable sstc for every vcpu if available in hardware */ - if (riscv_isa_extension_available(NULL, SSTC)) { + if (!kvm_riscv_isa_check_host(SSTC)) { t->sstc_enabled = true; - t->hrt.function = kvm_riscv_vcpu_vstimer_expired; + hrtimer_setup(&t->hrt, kvm_riscv_vcpu_vstimer_expired, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); t->timer_next_event = kvm_riscv_vcpu_update_vstimecmp; } else { t->sstc_enabled = false; - t->hrt.function = kvm_riscv_vcpu_hrtimer_expired; + hrtimer_setup(&t->hrt, kvm_riscv_vcpu_hrtimer_expired, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); t->timer_next_event = kvm_riscv_vcpu_update_hrtimer; } @@ -306,8 +309,9 @@ void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu) return; #if defined(CONFIG_32BIT) - ncsr_write(CSR_VSTIMECMP, (u32)t->next_cycles); + ncsr_write(CSR_VSTIMECMP, ULONG_MAX); ncsr_write(CSR_VSTIMECMPH, (u32)(t->next_cycles >> 32)); + ncsr_write(CSR_VSTIMECMP, (u32)(t->next_cycles)); #else ncsr_write(CSR_VSTIMECMP, t->next_cycles); #endif @@ -344,8 +348,24 @@ void kvm_riscv_vcpu_timer_save(struct kvm_vcpu *vcpu) /* * The vstimecmp CSRs are saved by kvm_riscv_vcpu_timer_sync() * upon every VM exit so no need to save here. + * + * If VS-timer expires when no VCPU running on a host CPU then + * WFI executed by such host CPU will be effective NOP resulting + * in no power savings. This is because as-per RISC-V Privileged + * specificaiton: "WFI is also required to resume execution for + * locally enabled interrupts pending at any privilege level, + * regardless of the global interrupt enable at each privilege + * level." + * + * To address the above issue, vstimecmp CSR must be set to -1UL + * over here when VCPU is scheduled-out or exits to user space. */ + csr_write(CSR_VSTIMECMP, -1UL); +#if defined(CONFIG_32BIT) + csr_write(CSR_VSTIMECMPH, -1UL); +#endif + /* timer should be enabled for the remaining operations */ if (unlikely(!t->init_done)) return; diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index d92d1348045c..62d2fb77bb9b 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -12,6 +12,7 @@ #include <linux/kvm_host.h> #include <linux/uaccess.h> #include <asm/cpufeature.h> +#include <asm/kvm_isa.h> #include <asm/kvm_vcpu_vector.h> #include <asm/vector.h> @@ -22,6 +23,9 @@ void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; cntx->sstatus &= ~SR_VS; + + cntx->vector.vlenb = riscv_v_vsize / 32; + if (riscv_isa_extension_available(isa, v)) { cntx->sstatus |= SR_VS_INITIAL; WARN_ON(!cntx->vector.datap); @@ -60,34 +64,35 @@ void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) { /* No need to check host sstatus as it can be modified outside */ - if (riscv_isa_extension_available(NULL, v)) + if (!kvm_riscv_isa_check_host(V)) __kvm_riscv_vector_save(cntx); } void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) { - if (riscv_isa_extension_available(NULL, v)) + if (!kvm_riscv_isa_check_host(V)) __kvm_riscv_vector_restore(cntx); } -int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu, - struct kvm_cpu_context *cntx) +int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) { - cntx->vector.datap = kmalloc(riscv_v_vsize, GFP_KERNEL); - if (!cntx->vector.datap) + vcpu->arch.guest_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); + if (!vcpu->arch.guest_context.vector.datap) return -ENOMEM; - cntx->vector.vlenb = riscv_v_vsize / 32; vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); - if (!vcpu->arch.host_context.vector.datap) + if (!vcpu->arch.host_context.vector.datap) { + kfree(vcpu->arch.guest_context.vector.datap); + vcpu->arch.guest_context.vector.datap = NULL; return -ENOMEM; + } return 0; } void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) { - kfree(vcpu->arch.guest_reset_context.vector.datap); + kfree(vcpu->arch.guest_context.vector.datap); kfree(vcpu->arch.host_context.vector.datap); } #endif @@ -126,6 +131,7 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { if (reg_size != vlenb) return -EINVAL; + WARN_ON(!cntx->vector.datap); *reg_addr = cntx->vector.datap + (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb; } else { @@ -181,6 +187,8 @@ int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; unsigned long reg_val; + if (reg_size != sizeof(reg_val)) + return -EINVAL; if (copy_from_user(®_val, uaddr, reg_size)) return -EFAULT; if (reg_val != cntx->vector.vlenb) diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 7396b8654f45..a9f083feeb76 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -11,8 +11,9 @@ #include <linux/module.h> #include <linux/uaccess.h> #include <linux/kvm_host.h> +#include <asm/kvm_mmu.h> -const struct _kvm_stats_desc kvm_vm_stats_desc[] = { +const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS() }; static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == @@ -31,13 +32,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int r; - r = kvm_riscv_gstage_alloc_pgd(kvm); + r = kvm_riscv_mmu_alloc_pgd(kvm); if (r) return r; r = kvm_riscv_gstage_vmid_init(kvm); if (r) { - kvm_riscv_gstage_free_pgd(kvm); + kvm_riscv_mmu_free_pgd(kvm); return r; } @@ -94,7 +95,7 @@ int kvm_riscv_setup_default_irq_routing(struct kvm *kvm, u32 lines) struct kvm_irq_routing_entry *ents; int i, rc; - ents = kcalloc(lines, sizeof(*ents), GFP_KERNEL); + ents = kzalloc_objs(*ents, lines); if (!ents) return -ENOMEM; @@ -180,7 +181,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_IOEVENTFD: case KVM_CAP_USER_MEMORY: - case KVM_CAP_SYNC_MMU: case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_ONE_REG: case KVM_CAP_READONLY_MEM: @@ -199,7 +199,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_USER_MEM_SLOTS; break; case KVM_CAP_VM_GPA_BITS: - r = kvm_riscv_gstage_gpa_bits(); + if (!kvm) + r = kvm_riscv_gstage_gpa_bits(kvm_riscv_gstage_max_pgd_levels); + else + r = kvm_riscv_gstage_gpa_bits(kvm->arch.pgd_levels); break; default: r = 0; @@ -209,6 +212,59 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) return r; } +int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) +{ + if (cap->flags) + return -EINVAL; + + switch (cap->cap) { + case KVM_CAP_RISCV_MP_STATE_RESET: + kvm->arch.mp_state_reset = true; + return 0; + case KVM_CAP_VM_GPA_BITS: { + unsigned long gpa_bits = cap->args[0]; + unsigned long new_levels; + int r = 0; + + /* Decide target pgd levels from requested gpa_bits */ +#ifdef CONFIG_64BIT + if (gpa_bits <= 41) + new_levels = 3; /* Sv39x4 */ + else if (gpa_bits <= 50) + new_levels = 4; /* Sv48x4 */ + else if (gpa_bits <= 59) + new_levels = 5; /* Sv57x4 */ + else + return -EINVAL; +#else + /* 32-bit: only Sv32x4*/ + if (gpa_bits <= 34) + new_levels = 2; + else + return -EINVAL; +#endif + if (new_levels > kvm_riscv_gstage_max_pgd_levels) + return -EINVAL; + + /* Follow KVM's lock ordering: kvm->lock -> kvm->slots_lock. */ + mutex_lock(&kvm->lock); + mutex_lock(&kvm->slots_lock); + + if (kvm->created_vcpus || !kvm_are_all_memslots_empty(kvm)) + r = -EBUSY; + else + kvm->arch.pgd_levels = new_levels; + + mutex_unlock(&kvm->slots_lock); + mutex_unlock(&kvm->lock); + + return r; + } + default: + return -EINVAL; + } +} + int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { return -EINVAL; diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index ddc98714ce8e..c15bdb1dd8be 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -14,6 +14,9 @@ #include <linux/smp.h> #include <linux/kvm_host.h> #include <asm/csr.h> +#include <asm/kvm_mmu.h> +#include <asm/kvm_tlb.h> +#include <asm/kvm_vmid.h> static unsigned long vmid_version = 1; static unsigned long vmid_next; @@ -22,15 +25,13 @@ static DEFINE_SPINLOCK(vmid_lock); void __init kvm_riscv_gstage_vmid_detect(void) { - unsigned long old; - /* Figure-out number of VMID bits in HW */ - old = csr_read(CSR_HGATP); - csr_write(CSR_HGATP, old | HGATP_VMID); + csr_write(CSR_HGATP, (kvm_riscv_gstage_mode(kvm_riscv_gstage_max_pgd_levels) << + HGATP_MODE_SHIFT) | HGATP_VMID); vmid_bits = csr_read(CSR_HGATP); vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT; vmid_bits = fls_long(vmid_bits); - csr_write(CSR_HGATP, old); + csr_write(CSR_HGATP, 0); /* We polluted local TLB so flush all guest TLB */ kvm_riscv_local_hfence_gvma_all(); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 79368a895fee..6f767b2a349d 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -7,6 +7,9 @@ ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) lib-y += strcmp.o lib-y += strlen.o lib-y += strncmp.o +lib-y += strnlen.o +lib-y += strchr.o +lib-y += strrchr.o endif lib-y += csum.o ifeq ($(CONFIG_MMU), y) @@ -15,7 +18,5 @@ endif lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o -obj-$(CONFIG_CRC32_ARCH) += crc32-riscv.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o -lib-$(CONFIG_RISCV_ISA_V) += xor.o lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o diff --git a/arch/riscv/lib/crc32-riscv.c b/arch/riscv/lib/crc32-riscv.c deleted file mode 100644 index 53d56ab422c7..000000000000 --- a/arch/riscv/lib/crc32-riscv.c +++ /dev/null @@ -1,311 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated CRC32 implementation with Zbc extension. - * - * Copyright (C) 2024 Intel Corporation - */ - -#include <asm/hwcap.h> -#include <asm/alternative-macros.h> -#include <asm/byteorder.h> - -#include <linux/types.h> -#include <linux/minmax.h> -#include <linux/crc32poly.h> -#include <linux/crc32.h> -#include <linux/byteorder/generic.h> -#include <linux/module.h> - -/* - * Refer to https://www.corsix.org/content/barrett-reduction-polynomials for - * better understanding of how this math works. - * - * let "+" denotes polynomial add (XOR) - * let "-" denotes polynomial sub (XOR) - * let "*" denotes polynomial multiplication - * let "/" denotes polynomial floor division - * let "S" denotes source data, XLEN bit wide - * let "P" denotes CRC32 polynomial - * let "T" denotes 2^(XLEN+32) - * let "QT" denotes quotient of T/P, with the bit for 2^XLEN being implicit - * - * crc32(S, P) - * => S * (2^32) - S * (2^32) / P * P - * => lowest 32 bits of: S * (2^32) / P * P - * => lowest 32 bits of: S * (2^32) * (T / P) / T * P - * => lowest 32 bits of: S * (2^32) * quotient / T * P - * => lowest 32 bits of: S * quotient / 2^XLEN * P - * => lowest 32 bits of: (clmul_high_part(S, QT) + S) * P - * => clmul_low_part(clmul_high_part(S, QT) + S, P) - * - * In terms of below implementations, the BE case is more intuitive, since the - * higher order bit sits at more significant position. - */ - -#if __riscv_xlen == 64 -/* Slide by XLEN bits per iteration */ -# define STEP_ORDER 3 - -/* Each below polynomial quotient has an implicit bit for 2^XLEN */ - -/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in LE format */ -# define CRC32_POLY_QT_LE 0x5a72d812fb808b20 - -/* Polynomial quotient of (2^(XLEN+32))/CRC32C_POLY, in LE format */ -# define CRC32C_POLY_QT_LE 0xa434f61c6f5389f8 - -/* Polynomial quotient of (2^(XLEN+32))/CRC32_POLY, in BE format, it should be - * the same as the bit-reversed version of CRC32_POLY_QT_LE - */ -# define CRC32_POLY_QT_BE 0x04d101df481b4e5a - -static inline u64 crc32_le_prep(u32 crc, unsigned long const *ptr) -{ - return (u64)crc ^ (__force u64)__cpu_to_le64(*ptr); -} - -static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) -{ - u32 crc; - - /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ - asm volatile (".option push\n" - ".option arch,+zbc\n" - "clmul %0, %1, %2\n" - "slli %0, %0, 1\n" - "xor %0, %0, %1\n" - "clmulr %0, %0, %3\n" - "srli %0, %0, 32\n" - ".option pop\n" - : "=&r" (crc) - : "r" (s), - "r" (poly_qt), - "r" ((u64)poly << 32) - :); - return crc; -} - -static inline u64 crc32_be_prep(u32 crc, unsigned long const *ptr) -{ - return ((u64)crc << 32) ^ (__force u64)__cpu_to_be64(*ptr); -} - -#elif __riscv_xlen == 32 -# define STEP_ORDER 2 -/* Each quotient should match the upper half of its analog in RV64 */ -# define CRC32_POLY_QT_LE 0xfb808b20 -# define CRC32C_POLY_QT_LE 0x6f5389f8 -# define CRC32_POLY_QT_BE 0x04d101df - -static inline u32 crc32_le_prep(u32 crc, unsigned long const *ptr) -{ - return crc ^ (__force u32)__cpu_to_le32(*ptr); -} - -static inline u32 crc32_le_zbc(unsigned long s, u32 poly, unsigned long poly_qt) -{ - u32 crc; - - /* We don't have a "clmulrh" insn, so use clmul + slli instead. */ - asm volatile (".option push\n" - ".option arch,+zbc\n" - "clmul %0, %1, %2\n" - "slli %0, %0, 1\n" - "xor %0, %0, %1\n" - "clmulr %0, %0, %3\n" - ".option pop\n" - : "=&r" (crc) - : "r" (s), - "r" (poly_qt), - "r" (poly) - :); - return crc; -} - -static inline u32 crc32_be_prep(u32 crc, unsigned long const *ptr) -{ - return crc ^ (__force u32)__cpu_to_be32(*ptr); -} - -#else -# error "Unexpected __riscv_xlen" -#endif - -static inline u32 crc32_be_zbc(unsigned long s) -{ - u32 crc; - - asm volatile (".option push\n" - ".option arch,+zbc\n" - "clmulh %0, %1, %2\n" - "xor %0, %0, %1\n" - "clmul %0, %0, %3\n" - ".option pop\n" - : "=&r" (crc) - : "r" (s), - "r" (CRC32_POLY_QT_BE), - "r" (CRC32_POLY_BE) - :); - return crc; -} - -#define STEP (1 << STEP_ORDER) -#define OFFSET_MASK (STEP - 1) - -typedef u32 (*fallback)(u32 crc, unsigned char const *p, size_t len); - -static inline u32 crc32_le_unaligned(u32 crc, unsigned char const *p, - size_t len, u32 poly, - unsigned long poly_qt) -{ - size_t bits = len * 8; - unsigned long s = 0; - u32 crc_low = 0; - - for (int i = 0; i < len; i++) - s = ((unsigned long)*p++ << (__riscv_xlen - 8)) | (s >> 8); - - s ^= (unsigned long)crc << (__riscv_xlen - bits); - if (__riscv_xlen == 32 || len < sizeof(u32)) - crc_low = crc >> bits; - - crc = crc32_le_zbc(s, poly, poly_qt); - crc ^= crc_low; - - return crc; -} - -static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, - size_t len, u32 poly, - unsigned long poly_qt, - fallback crc_fb) -{ - size_t offset, head_len, tail_len; - unsigned long const *p_ul; - unsigned long s; - - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBC, 1) - : : : : legacy); - - /* Handle the unaligned head. */ - offset = (unsigned long)p & OFFSET_MASK; - if (offset && len) { - head_len = min(STEP - offset, len); - crc = crc32_le_unaligned(crc, p, head_len, poly, poly_qt); - p += head_len; - len -= head_len; - } - - tail_len = len & OFFSET_MASK; - len = len >> STEP_ORDER; - p_ul = (unsigned long const *)p; - - for (int i = 0; i < len; i++) { - s = crc32_le_prep(crc, p_ul); - crc = crc32_le_zbc(s, poly, poly_qt); - p_ul++; - } - - /* Handle the tail bytes. */ - p = (unsigned char const *)p_ul; - if (tail_len) - crc = crc32_le_unaligned(crc, p, tail_len, poly, poly_qt); - - return crc; - -legacy: - return crc_fb(crc, p, len); -} - -u32 __pure crc32_le_arch(u32 crc, const u8 *p, size_t len) -{ - return crc32_le_generic(crc, p, len, CRC32_POLY_LE, CRC32_POLY_QT_LE, - crc32_le_base); -} -EXPORT_SYMBOL(crc32_le_arch); - -u32 __pure crc32c_le_arch(u32 crc, const u8 *p, size_t len) -{ - return crc32_le_generic(crc, p, len, CRC32C_POLY_LE, - CRC32C_POLY_QT_LE, crc32c_le_base); -} -EXPORT_SYMBOL(crc32c_le_arch); - -static inline u32 crc32_be_unaligned(u32 crc, unsigned char const *p, - size_t len) -{ - size_t bits = len * 8; - unsigned long s = 0; - u32 crc_low = 0; - - s = 0; - for (int i = 0; i < len; i++) - s = *p++ | (s << 8); - - if (__riscv_xlen == 32 || len < sizeof(u32)) { - s ^= crc >> (32 - bits); - crc_low = crc << bits; - } else { - s ^= (unsigned long)crc << (bits - 32); - } - - crc = crc32_be_zbc(s); - crc ^= crc_low; - - return crc; -} - -u32 __pure crc32_be_arch(u32 crc, const u8 *p, size_t len) -{ - size_t offset, head_len, tail_len; - unsigned long const *p_ul; - unsigned long s; - - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBC, 1) - : : : : legacy); - - /* Handle the unaligned head. */ - offset = (unsigned long)p & OFFSET_MASK; - if (offset && len) { - head_len = min(STEP - offset, len); - crc = crc32_be_unaligned(crc, p, head_len); - p += head_len; - len -= head_len; - } - - tail_len = len & OFFSET_MASK; - len = len >> STEP_ORDER; - p_ul = (unsigned long const *)p; - - for (int i = 0; i < len; i++) { - s = crc32_be_prep(crc, p_ul); - crc = crc32_be_zbc(s); - p_ul++; - } - - /* Handle the tail bytes. */ - p = (unsigned char const *)p_ul; - if (tail_len) - crc = crc32_be_unaligned(crc, p, tail_len); - - return crc; - -legacy: - return crc32_be_base(crc, p, len); -} -EXPORT_SYMBOL(crc32_be_arch); - -u32 crc32_optimizations(void) -{ - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) - return CRC32_LE_OPTIMIZATION | - CRC32_BE_OPTIMIZATION | - CRC32C_OPTIMIZATION; - return 0; -} -EXPORT_SYMBOL(crc32_optimizations); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Accelerated CRC32 implementation with Zbc extension"); diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 7fb12c59e571..5d4379768e5d 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -40,25 +40,11 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, uproto = (__force unsigned int)htonl(proto); sum += uproto; - /* - * Zbb support saves 4 instructions, so not worth checking without - * alternatives if supported - */ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); asm(".option push \n\ .option arch,+zbb \n\ rori %[fold_temp], %[sum], 32 \n\ @@ -71,7 +57,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, : [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp)); return (__force __sum16)(sum >> 16); } -no_zbb: + sum += ror64(sum, 32); sum >>= 32; return csum_fold((__force __wsum)sum); @@ -157,26 +143,11 @@ do_csum_with_alignment(const unsigned char *buff, int len) csum = do_csum_common(ptr, end, data); #ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT - /* - * Zbb support saves 6 instructions, so not worth checking without - * alternatives if supported - */ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm_goto_output(".option push \n\ .option arch,+zbb \n\ @@ -214,7 +185,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) end: return csum >> 16; } -no_zbb: + #endif /* CC_HAS_ASM_GOTO_TIED_OUTPUT */ #ifndef CONFIG_32BIT csum += ror64(csum, 32); @@ -244,26 +215,11 @@ do_csum_no_alignment(const unsigned char *buff, int len) end = (const unsigned long *)(buff + len); csum = do_csum_common(ptr, end, data); - /* - * Zbb support saves 6 instructions, so not worth checking without - * alternatives if supported - */ if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && - IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm (".option push \n\ .option arch,+zbb \n\ @@ -289,7 +245,7 @@ do_csum_no_alignment(const unsigned char *buff, int len) #endif /* !CONFIG_32BIT */ return csum >> 16; } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; @@ -313,7 +269,7 @@ unsigned int do_csum(const unsigned char *buff, int len) * on machines with fast misaligned accesses. * * There is some duplicate code between the "with_alignment" and - * "no_alignment" implmentations, but the overlap is too awkward to be + * "no_alignment" implementations, but the overlap is too awkward to be * able to fit in one function without introducing multiple static * branches. The largest chunk of overlap was delegated into the * do_csum_common function. diff --git a/arch/riscv/lib/memmove.S b/arch/riscv/lib/memmove.S index cb3e2e7ef0ba..410e598d4677 100644 --- a/arch/riscv/lib/memmove.S +++ b/arch/riscv/lib/memmove.S @@ -40,8 +40,8 @@ SYM_FUNC_START(__memmove) * Both Copy Modes: t1 - Temporary for load-store * Both Copy Modes: t2 - Temporary for load-store * Both Copy Modes: a5 - dest to src alignment offset - * Both Copy Modes: a6 - Shift ammount - * Both Copy Modes: a7 - Inverse Shift ammount + * Both Copy Modes: a6 - Shift amount + * Both Copy Modes: a7 - Inverse Shift amount * Both Copy Modes: a2 - Alternate breakpoint for unrolled loops */ diff --git a/arch/riscv/lib/riscv_v_helpers.c b/arch/riscv/lib/riscv_v_helpers.c index be38a93cedae..7bbdfc6d4552 100644 --- a/arch/riscv/lib/riscv_v_helpers.c +++ b/arch/riscv/lib/riscv_v_helpers.c @@ -16,8 +16,11 @@ #ifdef CONFIG_MMU size_t riscv_v_usercopy_threshold = CONFIG_RISCV_ISA_V_UCOPY_THRESHOLD; int __asm_vector_usercopy(void *dst, void *src, size_t n); +int __asm_vector_usercopy_sum_enabled(void *dst, void *src, size_t n); int fallback_scalar_usercopy(void *dst, void *src, size_t n); -asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n) +int fallback_scalar_usercopy_sum_enabled(void *dst, void *src, size_t n); +asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n, + bool enable_sum) { size_t remain, copied; @@ -26,7 +29,8 @@ asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n) goto fallback; kernel_vector_begin(); - remain = __asm_vector_usercopy(dst, src, n); + remain = enable_sum ? __asm_vector_usercopy(dst, src, n) : + __asm_vector_usercopy_sum_enabled(dst, src, n); kernel_vector_end(); if (remain) { @@ -40,6 +44,7 @@ asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n) return remain; fallback: - return fallback_scalar_usercopy(dst, src, n); + return enable_sum ? fallback_scalar_usercopy(dst, src, n) : + fallback_scalar_usercopy_sum_enabled(dst, src, n); } #endif diff --git a/arch/riscv/lib/strchr.S b/arch/riscv/lib/strchr.S new file mode 100644 index 000000000000..48c3a9da53e3 --- /dev/null +++ b/arch/riscv/lib/strchr.S @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* char *strchr(const char *s, int c) */ +SYM_FUNC_START(strchr) + /* + * Parameters + * a0 - The string to be searched + * a1 - The character to search for + * + * Returns + * a0 - Address of first occurrence of 'c' or 0 + * + * Clobbers + * t0 + */ + andi a1, a1, 0xff +1: + lbu t0, 0(a0) + beq t0, a1, 2f + addi a0, a0, 1 + bnez t0, 1b + li a0, 0 +2: + ret +SYM_FUNC_END(strchr) + +SYM_FUNC_ALIAS_WEAK(__pi_strchr, strchr) +EXPORT_SYMBOL(strchr) diff --git a/arch/riscv/lib/strcmp.S b/arch/riscv/lib/strcmp.S index 57a5c0066231..65027e742af1 100644 --- a/arch/riscv/lib/strcmp.S +++ b/arch/riscv/lib/strcmp.S @@ -8,7 +8,8 @@ /* int strcmp(const char *cs, const char *ct) */ SYM_FUNC_START(strcmp) - ALTERNATIVE("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strcmp_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -43,7 +44,7 @@ SYM_FUNC_START(strcmp) * The code was published as part of the bitmanip manual * in Appendix A. */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strcmp_zbb: .option push diff --git a/arch/riscv/lib/strlen.S b/arch/riscv/lib/strlen.S index 962983b73251..e7736ccda514 100644 --- a/arch/riscv/lib/strlen.S +++ b/arch/riscv/lib/strlen.S @@ -8,7 +8,8 @@ /* int strlen(const char *s) */ SYM_FUNC_START(strlen) - ALTERNATIVE("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strlen_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -20,20 +21,18 @@ SYM_FUNC_START(strlen) * Clobbers: * t0, t1 */ - mv t1, a0 + addi t1, a0, -1 1: - lbu t0, 0(t1) - beqz t0, 2f addi t1, t1, 1 - j 1b -2: + lbu t0, 0(t1) + bnez t0, 1b sub a0, t1, a0 ret /* * Variant of strlen using the ZBB extension if available */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strlen_zbb: #ifdef CONFIG_CPU_BIG_ENDIAN diff --git a/arch/riscv/lib/strncmp.S b/arch/riscv/lib/strncmp.S index 7b2d0ff9ed6c..062000c468c8 100644 --- a/arch/riscv/lib/strncmp.S +++ b/arch/riscv/lib/strncmp.S @@ -8,7 +8,8 @@ /* int strncmp(const char *cs, const char *ct, size_t count) */ SYM_FUNC_START(strncmp) - ALTERNATIVE("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, CONFIG_RISCV_ISA_ZBB) + __ALTERNATIVE_CFG("nop", "j strncmp_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) /* * Returns @@ -46,7 +47,7 @@ SYM_FUNC_START(strncmp) /* * Variant of strncmp using the ZBB extension if available */ -#ifdef CONFIG_RISCV_ISA_ZBB +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) strncmp_zbb: .option push diff --git a/arch/riscv/lib/strnlen.S b/arch/riscv/lib/strnlen.S new file mode 100644 index 000000000000..53afa7b5b314 --- /dev/null +++ b/arch/riscv/lib/strnlen.S @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Base on arch/riscv/lib/strlen.S + * + * Copyright (C) Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> +#include <asm/alternative-macros.h> +#include <asm/hwcap.h> + +/* size_t strnlen(const char *s, size_t count) */ +SYM_FUNC_START(strnlen) + + __ALTERNATIVE_CFG("nop", "j strnlen_zbb", 0, RISCV_ISA_EXT_ZBB, + IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) + + + /* + * Returns + * a0 - String length + * + * Parameters + * a0 - String to measure + * a1 - Max length of string + * + * Clobbers + * t0, t1, t2 + */ + addi t1, a0, -1 + add t2, a0, a1 +1: + addi t1, t1, 1 + beq t1, t2, 2f + lbu t0, 0(t1) + bnez t0, 1b +2: + sub a0, t1, a0 + ret + + +/* + * Variant of strnlen using the ZBB extension if available + */ +#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) +strnlen_zbb: + +#ifdef CONFIG_CPU_BIG_ENDIAN +# define CZ clz +# define SHIFT sll +#else +# define CZ ctz +# define SHIFT srl +#endif + +.option push +.option arch,+zbb + + /* + * Returns + * a0 - String length + * + * Parameters + * a0 - String to measure + * a1 - Max length of string + * + * Clobbers + * t0, t1, t2, t3, t4 + */ + + /* If maxlen is 0, return 0. */ + beqz a1, 3f + + /* Number of irrelevant bytes in the first word. */ + andi t2, a0, SZREG-1 + + /* Align pointer. */ + andi t0, a0, -SZREG + + li t3, SZREG + sub t3, t3, t2 + slli t2, t2, 3 + + /* Aligned boundary. */ + add t4, a0, a1 + andi t4, t4, -SZREG + + /* Get the first word. */ + REG_L t1, 0(t0) + + /* + * Shift away the partial data we loaded to remove the irrelevant bytes + * preceding the string with the effect of adding NUL bytes at the + * end of the string's first word. + */ + SHIFT t1, t1, t2 + + /* Convert non-NUL into 0xff and NUL into 0x00. */ + orc.b t1, t1 + + /* Convert non-NUL into 0x00 and NUL into 0xff. */ + not t1, t1 + + /* + * Search for the first set bit (corresponding to a NUL byte in the + * original chunk). + */ + CZ t1, t1 + + /* + * The first chunk is special: compare against the number + * of valid bytes in this chunk. + */ + srli a0, t1, 3 + + /* Limit the result by maxlen. */ + minu a0, a0, a1 + + bgtu t3, a0, 2f + + /* Prepare for the word comparison loop. */ + addi t2, t0, SZREG + li t3, -1 + + /* + * Our critical loop is 4 instructions and processes data in + * 4 byte or 8 byte chunks. + */ + .p2align 3 +1: + REG_L t1, SZREG(t0) + addi t0, t0, SZREG + orc.b t1, t1 + bgeu t0, t4, 4f + beq t1, t3, 1b +4: + not t1, t1 + CZ t1, t1 + srli t1, t1, 3 + + /* Get number of processed bytes. */ + sub t2, t0, t2 + + /* Add number of characters in the first word. */ + add a0, a0, t2 + + /* Add number of characters in the last word. */ + add a0, a0, t1 + + /* Ensure the final result does not exceed maxlen. */ + minu a0, a0, a1 +2: + ret +3: + mv a0, a1 + ret + +.option pop +#endif +SYM_FUNC_END(strnlen) +SYM_FUNC_ALIAS(__pi_strnlen, strnlen) +EXPORT_SYMBOL(strnlen) diff --git a/arch/riscv/lib/strrchr.S b/arch/riscv/lib/strrchr.S new file mode 100644 index 000000000000..ac58b20ca21d --- /dev/null +++ b/arch/riscv/lib/strrchr.S @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/* + * Copyright (C) 2025 Feng Jiang <jiangfeng@kylinos.cn> + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* char *strrchr(const char *s, int c) */ +SYM_FUNC_START(strrchr) + /* + * Parameters + * a0 - The string to be searched + * a1 - The character to seaerch for + * + * Returns + * a0 - Address of last occurrence of 'c' or 0 + * + * Clobbers + * t0, t1 + */ + andi a1, a1, 0xff + mv t1, a0 + li a0, 0 +1: + lbu t0, 0(t1) + bne t0, a1, 2f + mv a0, t1 +2: + addi t1, t1, 1 + bnez t0, 1b + ret +SYM_FUNC_END(strrchr) + +SYM_FUNC_ALIAS_WEAK(__pi_strrchr, strrchr) +EXPORT_SYMBOL(strrchr) diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 6a9f116bb545..4efea1b3326c 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -17,14 +17,43 @@ SYM_FUNC_START(__asm_copy_to_user) ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) REG_L t0, riscv_v_usercopy_threshold bltu a2, t0, fallback_scalar_usercopy - tail enter_vector_usercopy + li a3, 1 + tail enter_vector_usercopy #endif -SYM_FUNC_START(fallback_scalar_usercopy) +SYM_FUNC_END(__asm_copy_to_user) +EXPORT_SYMBOL(__asm_copy_to_user) +SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) +EXPORT_SYMBOL(__asm_copy_from_user) +SYM_FUNC_START(fallback_scalar_usercopy) /* Enable access to user memory */ - li t6, SR_SUM - csrs CSR_STATUS, t6 + li t6, SR_SUM + csrs CSR_STATUS, t6 + mv t6, ra + call fallback_scalar_usercopy_sum_enabled + + /* Disable access to user memory */ + mv ra, t6 + li t6, SR_SUM + csrc CSR_STATUS, t6 + ret +SYM_FUNC_END(fallback_scalar_usercopy) + +SYM_FUNC_START(__asm_copy_to_user_sum_enabled) +#ifdef CONFIG_RISCV_ISA_V + ALTERNATIVE("j fallback_scalar_usercopy_sum_enabled", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) + REG_L t0, riscv_v_usercopy_threshold + bltu a2, t0, fallback_scalar_usercopy_sum_enabled + li a3, 0 + tail enter_vector_usercopy +#endif +SYM_FUNC_END(__asm_copy_to_user_sum_enabled) +SYM_FUNC_ALIAS(__asm_copy_from_user_sum_enabled, __asm_copy_to_user_sum_enabled) +EXPORT_SYMBOL(__asm_copy_from_user_sum_enabled) +EXPORT_SYMBOL(__asm_copy_to_user_sum_enabled) + +SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) /* * Save the terminal address which will be used to compute the number * of bytes copied in case of a fixup exception. @@ -178,23 +207,12 @@ SYM_FUNC_START(fallback_scalar_usercopy) bltu a0, t0, 4b /* t0 - end of dst */ .Lout_copy_user: - /* Disable access to user memory */ - csrc CSR_STATUS, t6 li a0, 0 ret - - /* Exception fixup code */ 10: - /* Disable access to user memory */ - csrc CSR_STATUS, t6 sub a0, t5, a0 ret -SYM_FUNC_END(__asm_copy_to_user) -SYM_FUNC_END(fallback_scalar_usercopy) -EXPORT_SYMBOL(__asm_copy_to_user) -SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) -EXPORT_SYMBOL(__asm_copy_from_user) - +SYM_FUNC_END(fallback_scalar_usercopy_sum_enabled) SYM_FUNC_START(__clear_user) diff --git a/arch/riscv/lib/uaccess_vector.S b/arch/riscv/lib/uaccess_vector.S index 7c45f26de4f7..03b5560609a2 100644 --- a/arch/riscv/lib/uaccess_vector.S +++ b/arch/riscv/lib/uaccess_vector.S @@ -24,7 +24,18 @@ SYM_FUNC_START(__asm_vector_usercopy) /* Enable access to user memory */ li t6, SR_SUM csrs CSR_STATUS, t6 + mv t6, ra + call __asm_vector_usercopy_sum_enabled + + /* Disable access to user memory */ + mv ra, t6 + li t6, SR_SUM + csrc CSR_STATUS, t6 + ret +SYM_FUNC_END(__asm_vector_usercopy) + +SYM_FUNC_START(__asm_vector_usercopy_sum_enabled) loop: vsetvli iVL, iNum, e8, ELEM_LMUL_SETTING, ta, ma fixup vle8.v vData, (pSrc), 10f @@ -36,8 +47,6 @@ loop: /* Exception fixup for vector load is shared with normal exit */ 10: - /* Disable access to user memory */ - csrc CSR_STATUS, t6 mv a0, iNum ret @@ -49,4 +58,4 @@ loop: csrr t2, CSR_VSTART sub iNum, iNum, t2 j 10b -SYM_FUNC_END(__asm_vector_usercopy) +SYM_FUNC_END(__asm_vector_usercopy_sum_enabled) diff --git a/arch/riscv/lib/xor.S b/arch/riscv/lib/xor.S deleted file mode 100644 index b28f2430e52f..000000000000 --- a/arch/riscv/lib/xor.S +++ /dev/null @@ -1,81 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2021 SiFive - */ -#include <linux/linkage.h> -#include <linux/export.h> -#include <asm/asm.h> - -SYM_FUNC_START(xor_regs_2_) - vsetvli a3, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a3 - vxor.vv v16, v0, v8 - add a2, a2, a3 - vse8.v v16, (a1) - add a1, a1, a3 - bnez a0, xor_regs_2_ - ret -SYM_FUNC_END(xor_regs_2_) -EXPORT_SYMBOL(xor_regs_2_) - -SYM_FUNC_START(xor_regs_3_) - vsetvli a4, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a4 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a4 - vxor.vv v16, v0, v16 - add a3, a3, a4 - vse8.v v16, (a1) - add a1, a1, a4 - bnez a0, xor_regs_3_ - ret -SYM_FUNC_END(xor_regs_3_) -EXPORT_SYMBOL(xor_regs_3_) - -SYM_FUNC_START(xor_regs_4_) - vsetvli a5, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a5 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a5 - vxor.vv v0, v0, v16 - vle8.v v24, (a4) - add a3, a3, a5 - vxor.vv v16, v0, v24 - add a4, a4, a5 - vse8.v v16, (a1) - add a1, a1, a5 - bnez a0, xor_regs_4_ - ret -SYM_FUNC_END(xor_regs_4_) -EXPORT_SYMBOL(xor_regs_4_) - -SYM_FUNC_START(xor_regs_5_) - vsetvli a6, a0, e8, m8, ta, ma - vle8.v v0, (a1) - vle8.v v8, (a2) - sub a0, a0, a6 - vxor.vv v0, v0, v8 - vle8.v v16, (a3) - add a2, a2, a6 - vxor.vv v0, v0, v16 - vle8.v v24, (a4) - add a3, a3, a6 - vxor.vv v0, v0, v24 - vle8.v v8, (a5) - add a4, a4, a6 - vxor.vv v16, v0, v8 - add a5, a5, a6 - vse8.v v16, (a1) - add a1, a1, a6 - bnez a0, xor_regs_5_ - ret -SYM_FUNC_END(xor_regs_5_) -EXPORT_SYMBOL(xor_regs_5_) diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index cbe4d775ef56..b916a68d324a 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -19,7 +19,7 @@ obj-y += context.o obj-y += pmem.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_PTDUMP_CORE) += ptdump.o +obj-$(CONFIG_PTDUMP) += ptdump.o obj-$(CONFIG_KASAN) += kasan_init.o ifdef CONFIG_KASAN diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index b81672729887..f8ead7cb7c7d 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -24,7 +24,20 @@ void flush_icache_all(void) if (num_online_cpus() < 2) return; - else if (riscv_use_sbi_for_rfence()) + + /* + * Make sure all previous writes to the D$ are ordered before making + * the IPI. The RISC-V spec states that a hart must execute a data fence + * before triggering a remote fence.i in order to make the modification + * visible for remote harts. + * + * IPIs on RISC-V are triggered by MMIO writes to either CLINT or + * S-IMSIC, so the fence ensures previous data writes "happen before" + * the MMIO. + */ + RISCV_FENCE(w, o); + + if (riscv_use_sbi_for_rfence()) sbi_remote_fence_i(NULL); else on_each_cpu(ipi_remote_fence_i, NULL, 1); @@ -88,9 +101,9 @@ void flush_icache_pte(struct mm_struct *mm, pte_t pte) { struct folio *folio = page_folio(pte_page(pte)); - if (!test_bit(PG_dcache_clean, &folio->flags)) { + if (!test_bit(PG_dcache_clean, &folio->flags.f)) { flush_icache_mm(mm, false); - set_bit(PG_dcache_clean, &folio->flags); + set_bit(PG_dcache_clean, &folio->flags.f); } } #endif /* CONFIG_MMU */ @@ -101,6 +114,9 @@ EXPORT_SYMBOL_GPL(riscv_cbom_block_size); unsigned int riscv_cboz_block_size; EXPORT_SYMBOL_GPL(riscv_cboz_block_size); +unsigned int riscv_cbop_block_size; +EXPORT_SYMBOL_GPL(riscv_cbop_block_size); + static void __init cbo_get_block_size(struct device_node *node, const char *name, u32 *block_size, unsigned long *first_hartid) @@ -125,8 +141,8 @@ static void __init cbo_get_block_size(struct device_node *node, void __init riscv_init_cbo_blocksizes(void) { - unsigned long cbom_hartid, cboz_hartid; - u32 cbom_block_size = 0, cboz_block_size = 0; + unsigned long cbom_hartid, cboz_hartid, cbop_hartid; + u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0; struct device_node *node; struct acpi_table_header *rhct; acpi_status status; @@ -138,13 +154,15 @@ void __init riscv_init_cbo_blocksizes(void) &cbom_block_size, &cbom_hartid); cbo_get_block_size(node, "riscv,cboz-block-size", &cboz_block_size, &cboz_hartid); + cbo_get_block_size(node, "riscv,cbop-block-size", + &cbop_block_size, &cbop_hartid); } } else { status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); if (ACPI_FAILURE(status)) return; - acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); + acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size); acpi_put_table((struct acpi_table_header *)rhct); } @@ -153,6 +171,9 @@ void __init riscv_init_cbo_blocksizes(void) if (cboz_block_size) riscv_cboz_block_size = cboz_block_size; + + if (cbop_block_size) + riscv_cbop_block_size = cbop_block_size; } #ifdef CONFIG_SMP @@ -172,7 +193,7 @@ static void set_icache_stale_mask(void) stale_cpu = cpumask_test_cpu(cpu, mask); cpumask_setall(mask); - cpumask_assign_cpu(cpu, mask, stale_cpu); + __assign_cpu(cpu, mask, stale_cpu); put_cpu(); } #endif diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 4abe3de23225..55c20ad1f744 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -158,7 +158,7 @@ static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) * * - We get a zero back from the cmpxchg and end up waiting on the * lock. Taking the lock synchronises with the rollover and so - * we are forced to see the updated verion. + * we are forced to see the updated version. * * - We get a valid context back from the cmpxchg then we continue * using old ASID because __flush_context() would have marked ASID diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 0194324a0c50..04ed6f8acae4 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -20,6 +20,9 @@ #include <asm/ptrace.h> #include <asm/tlbflush.h> +#define CREATE_TRACE_POINTS +#include <trace/events/exceptions.h> + #include "../kernel/head.h" static void show_pte(unsigned long addr) @@ -291,6 +294,11 @@ void handle_page_fault(struct pt_regs *regs) if (kprobe_page_fault(regs, cause)) return; + if (user_mode(regs)) + trace_page_fault_user(addr, regs, cause); + else + trace_page_fault_kernel(addr, regs, cause); + /* * Fault-in kernel-space virtual memory on-demand. * The 'reference' page table is init_mm.pgd. diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index 42314f093922..a6d217112cf4 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -148,22 +148,25 @@ unsigned long hugetlb_mask_last_page(struct hstate *h) static pte_t get_clear_contig(struct mm_struct *mm, unsigned long addr, pte_t *ptep, - unsigned long pte_num) + unsigned long ncontig) { - pte_t orig_pte = ptep_get(ptep); - unsigned long i; - - for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) { - pte_t pte = ptep_get_and_clear(mm, addr, ptep); - - if (pte_dirty(pte)) - orig_pte = pte_mkdirty(orig_pte); - - if (pte_young(pte)) - orig_pte = pte_mkyoung(orig_pte); + pte_t pte, tmp_pte; + bool present; + + pte = ptep_get_and_clear(mm, addr, ptep); + present = pte_present(pte); + while (--ncontig) { + ptep++; + addr += PAGE_SIZE; + tmp_pte = ptep_get_and_clear(mm, addr, ptep); + if (present) { + if (pte_dirty(tmp_pte)) + pte = pte_mkdirty(pte); + if (pte_young(tmp_pte)) + pte = pte_mkyoung(pte); + } } - - return orig_pte; + return pte; } static pte_t get_clear_contig_flush(struct mm_struct *mm, @@ -212,6 +215,26 @@ static void clear_flush(struct mm_struct *mm, flush_tlb_range(&vma, saddr, addr); } +static int num_contig_ptes_from_size(unsigned long sz, size_t *pgsize) +{ + unsigned long hugepage_shift; + + if (sz >= PGDIR_SIZE) + hugepage_shift = PGDIR_SHIFT; + else if (sz >= P4D_SIZE) + hugepage_shift = P4D_SHIFT; + else if (sz >= PUD_SIZE) + hugepage_shift = PUD_SHIFT; + else if (sz >= PMD_SIZE) + hugepage_shift = PMD_SHIFT; + else + hugepage_shift = PAGE_SHIFT; + + *pgsize = 1 << hugepage_shift; + + return sz >> hugepage_shift; +} + /* * When dealing with NAPOT mappings, the privileged specification indicates that * "if an update needs to be made, the OS generally should first mark all of the @@ -226,22 +249,10 @@ void set_huge_pte_at(struct mm_struct *mm, pte_t pte, unsigned long sz) { - unsigned long hugepage_shift, pgsize; + size_t pgsize; int i, pte_num; - if (sz >= PGDIR_SIZE) - hugepage_shift = PGDIR_SHIFT; - else if (sz >= P4D_SIZE) - hugepage_shift = P4D_SHIFT; - else if (sz >= PUD_SIZE) - hugepage_shift = PUD_SHIFT; - else if (sz >= PMD_SIZE) - hugepage_shift = PMD_SHIFT; - else - hugepage_shift = PAGE_SHIFT; - - pte_num = sz >> hugepage_shift; - pgsize = 1 << hugepage_shift; + pte_num = num_contig_ptes_from_size(sz, &pgsize); if (!pte_present(pte)) { for (i = 0; i < pte_num; i++, ptep++, addr += pgsize) @@ -293,15 +304,16 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) + pte_t *ptep, unsigned long sz) { + size_t pgsize; pte_t orig_pte = ptep_get(ptep); int pte_num; if (!pte_napot(orig_pte)) return ptep_get_and_clear(mm, addr, ptep); - pte_num = napot_pte_num(napot_cont_order(orig_pte)); + pte_num = num_contig_ptes_from_size(sz, &pgsize); return get_clear_contig(mm, addr, ptep, pte_num); } @@ -351,6 +363,7 @@ void huge_pte_clear(struct mm_struct *mm, pte_t *ptep, unsigned long sz) { + size_t pgsize; pte_t pte = ptep_get(ptep); int i, pte_num; @@ -359,8 +372,9 @@ void huge_pte_clear(struct mm_struct *mm, return; } - pte_num = napot_pte_num(napot_cont_order(pte)); - for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) + pte_num = num_contig_ptes_from_size(sz, &pgsize); + + for (i = 0; i < pte_num; i++, addr += pgsize, ptep++) pte_clear(mm, addr, ptep); } @@ -433,3 +447,11 @@ static __init int gigantic_pages_init(void) } arch_initcall(gigantic_pages_init); #endif + +unsigned int __init arch_hugetlb_cma_order(void) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return PUD_SHIFT - PAGE_SHIFT; + + return 0; +} diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 15b2eda4c364..fa8d2f6f554b 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -20,15 +20,14 @@ #include <linux/dma-map-ops.h> #include <linux/crash_dump.h> #include <linux/hugetlb.h> -#ifdef CONFIG_RELOCATABLE -#include <linux/elf.h> -#endif #include <linux/kfence.h> #include <linux/execmem.h> +#include <asm/alternative.h> #include <asm/fixmap.h> #include <asm/io.h> #include <asm/kasan.h> +#include <asm/module.h> #include <asm/numa.h> #include <asm/pgtable.h> #include <asm/sections.h> @@ -42,20 +41,17 @@ u64 new_vmalloc[NR_CPUS / sizeof(u64) + 1]; struct kernel_mapping kernel_map __ro_after_init; EXPORT_SYMBOL(kernel_map); -#ifdef CONFIG_XIP_KERNEL -#define kernel_map (*(struct kernel_mapping *)XIP_FIXUP(&kernel_map)) -#endif #ifdef CONFIG_64BIT -u64 satp_mode __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL) ? SATP_MODE_57 : SATP_MODE_39; +u64 satp_mode __ro_after_init = SATP_MODE_57; #else u64 satp_mode __ro_after_init = SATP_MODE_32; #endif EXPORT_SYMBOL(satp_mode); #ifdef CONFIG_64BIT -bool pgtable_l4_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL); -bool pgtable_l5_enabled __ro_after_init = !IS_ENABLED(CONFIG_XIP_KERNEL); +bool pgtable_l4_enabled __ro_after_init = true; +bool pgtable_l5_enabled __ro_after_init = true; EXPORT_SYMBOL(pgtable_l4_enabled); EXPORT_SYMBOL(pgtable_l5_enabled); #endif @@ -64,32 +60,25 @@ phys_addr_t phys_ram_base __ro_after_init; EXPORT_SYMBOL(phys_ram_base); #ifdef CONFIG_SPARSEMEM_VMEMMAP -#define VMEMMAP_ADDR_ALIGN (1ULL << SECTION_SIZE_BITS) +#define VMEMMAP_ADDR_ALIGN max(1ULL << SECTION_SIZE_BITS, \ + MAX_FOLIO_VMEMMAP_ALIGN) unsigned long vmemmap_start_pfn __ro_after_init; EXPORT_SYMBOL(vmemmap_start_pfn); #endif -unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] - __page_aligned_bss; -EXPORT_SYMBOL(empty_zero_page); - extern char _start[]; void *_dtb_early_va __initdata; uintptr_t _dtb_early_pa __initdata; phys_addr_t dma32_phys_limit __initdata; -static void __init zone_sizes_init(void) +void __init arch_zone_limits_init(unsigned long *max_zone_pfns) { - unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; - #ifdef CONFIG_ZONE_DMA32 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(dma32_phys_limit); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; - - free_area_init(max_zone_pfns); } #if defined(CONFIG_MMU) && defined(CONFIG_DEBUG_VM) @@ -171,7 +160,7 @@ static void __init print_vm_layout(void) static void print_vm_layout(void) { } #endif /* CONFIG_DEBUG_VM */ -void __init mem_init(void) +void __init arch_mm_preinit(void) { bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit); #ifdef CONFIG_FLATMEM @@ -192,16 +181,12 @@ void __init mem_init(void) } swiotlb_init(swiotlb, SWIOTLB_VERBOSE); - memblock_free_all(); print_vm_layout(); } /* Limit the memory size via mem. */ static phys_addr_t memory_limit; -#ifdef CONFIG_XIP_KERNEL -#define memory_limit (*(phys_addr_t *)XIP_FIXUP(&memory_limit)) -#endif /* CONFIG_XIP_KERNEL */ static int __init early_mem(char *p) { @@ -225,10 +210,7 @@ static void __init setup_bootmem(void) phys_addr_t max_mapped_addr; phys_addr_t phys_ram_end, vmlinux_start; - if (IS_ENABLED(CONFIG_XIP_KERNEL)) - vmlinux_start = __pa_symbol(&_sdata); - else - vmlinux_start = __pa_symbol(&_start); + vmlinux_start = __pa_symbol(&_start); memblock_enforce_memory_limit(memory_limit); @@ -248,12 +230,10 @@ static void __init setup_bootmem(void) * Make sure we align the start of the memory on a PMD boundary so that * at worst, we map the linear mapping with PMD mappings. */ - if (!IS_ENABLED(CONFIG_XIP_KERNEL)) { - phys_ram_base = memblock_start_of_DRAM() & PMD_MASK; + phys_ram_base = memblock_start_of_DRAM() & PMD_MASK; #ifdef CONFIG_SPARSEMEM_VMEMMAP - vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; + vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; #endif - } /* * In 64-bit, any use of __va/__pa before this point is wrong as we @@ -295,10 +275,8 @@ static void __init setup_bootmem(void) phys_ram_end = memblock_end_of_DRAM(); min_low_pfn = PFN_UP(phys_ram_base); max_low_pfn = max_pfn = PFN_DOWN(phys_ram_end); - high_memory = (void *)(__va(PFN_PHYS(max_low_pfn))); dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); - set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); reserve_initrd_mem(); @@ -319,10 +297,46 @@ static void __init setup_bootmem(void) memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); dma_contiguous_reserve(dma32_phys_limit); - if (IS_ENABLED(CONFIG_64BIT)) - hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); } +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +static void __init relocate_kernel(void) +{ + Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; + /* + * This holds the offset between the linked virtual address and the + * relocated virtual address. + */ + uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR; + /* + * This holds the offset between kernel linked virtual address and + * physical address. + */ + uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr; + + for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { + Elf_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); + Elf_Addr relocated_addr = rela->r_addend; + + if (rela->r_info != R_RISCV_RELATIVE) + continue; + + /* + * Make sure to not relocate vdso symbols like rt_sigreturn + * which are linked from the address 0 in vmlinux since + * vdso symbol addresses are actually used as an offset from + * mm->context.vdso in VDSO_OFFSET macro. + */ + if (relocated_addr >= KERNEL_LINK_ADDR) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif /* CONFIG_RELOCATABLE */ + #ifdef CONFIG_MMU struct pt_alloc_ops pt_ops __meminitdata; @@ -332,17 +346,10 @@ static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define pt_ops (*(struct pt_alloc_ops *)XIP_FIXUP(&pt_ops)) -#define trampoline_pg_dir ((pgd_t *)XIP_FIXUP(trampoline_pg_dir)) -#define fixmap_pte ((pte_t *)XIP_FIXUP(fixmap_pte)) -#define early_pg_dir ((pgd_t *)XIP_FIXUP(early_pg_dir)) -#endif /* CONFIG_XIP_KERNEL */ - static const pgprot_t protection_map[16] = { [VM_NONE] = PAGE_NONE, [VM_READ] = PAGE_READ, - [VM_WRITE] = PAGE_COPY, + [VM_WRITE] = PAGE_SHADOWSTACK, [VM_WRITE | VM_READ] = PAGE_COPY, [VM_EXEC] = PAGE_EXEC, [VM_EXEC | VM_READ] = PAGE_READ_EXEC, @@ -407,9 +414,14 @@ static inline phys_addr_t __init alloc_pte_fixmap(uintptr_t va) static phys_addr_t __meminit alloc_pte_late(uintptr_t va) { - struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); - BUG_ON(!ptdesc || !pagetable_pte_ctor(ptdesc)); + /* + * We do not know which mm the PTE page is associated to at this point. + * Passing NULL to the ctor is the safe option, though it may result + * in unnecessary work (e.g. initialising the ptlock for init_mm). + */ + BUG_ON(!ptdesc || !pagetable_pte_ctor(NULL, ptdesc)); return __pa((pte_t *)ptdesc_address(ptdesc)); } @@ -430,32 +442,14 @@ static pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_pmd ((pmd_t *)XIP_FIXUP(trampoline_pmd)) -#define fixmap_pmd ((pmd_t *)XIP_FIXUP(fixmap_pmd)) -#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd)) -#endif /* CONFIG_XIP_KERNEL */ - static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss; static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss; static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d)) -#define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d)) -#define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d)) -#endif /* CONFIG_XIP_KERNEL */ - static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss; static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss; static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE); -#ifdef CONFIG_XIP_KERNEL -#define trampoline_pud ((pud_t *)XIP_FIXUP(trampoline_pud)) -#define fixmap_pud ((pud_t *)XIP_FIXUP(fixmap_pud)) -#define early_pud ((pud_t *)XIP_FIXUP(early_pud)) -#endif /* CONFIG_XIP_KERNEL */ - static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) { /* Before MMU is enabled */ @@ -487,9 +481,10 @@ static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) static phys_addr_t __meminit alloc_pmd_late(uintptr_t va) { - struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); - BUG_ON(!ptdesc || !pagetable_pmd_ctor(ptdesc)); + /* See comment in alloc_pte_late() regarding NULL passed the ctor */ + BUG_ON(!ptdesc || !pagetable_pmd_ctor(NULL, ptdesc)); return __pa((pmd_t *)ptdesc_address(ptdesc)); } @@ -551,11 +546,11 @@ static phys_addr_t __init alloc_pud_fixmap(uintptr_t va) static phys_addr_t __meminit alloc_pud_late(uintptr_t va) { - unsigned long vaddr; + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); - vaddr = __get_free_page(GFP_KERNEL); - BUG_ON(!vaddr); - return __pa(vaddr); + BUG_ON(!ptdesc); + pagetable_pud_ctor(ptdesc); + return __pa((pud_t *)ptdesc_address(ptdesc)); } static p4d_t *__init get_p4d_virt_early(phys_addr_t pa) @@ -589,11 +584,11 @@ static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va) static phys_addr_t __meminit alloc_p4d_late(uintptr_t va) { - unsigned long vaddr; + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); - vaddr = __get_free_page(GFP_KERNEL); - BUG_ON(!vaddr); - return __pa(vaddr); + BUG_ON(!ptdesc); + pagetable_p4d_ctor(ptdesc); + return __pa((p4d_t *)ptdesc_address(ptdesc)); } static void __meminit create_pud_mapping(pud_t *pudp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, @@ -725,21 +720,6 @@ static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr return PAGE_SIZE; } -#ifdef CONFIG_XIP_KERNEL -#define phys_ram_base (*(phys_addr_t *)XIP_FIXUP(&phys_ram_base)) -extern char _xiprom[], _exiprom[], __data_loc; - -/* called from head.S with MMU off */ -asmlinkage void __init __copy_data(void) -{ - void *from = (void *)(&__data_loc); - void *to = (void *)CONFIG_PHYS_RAM_BASE; - size_t sz = (size_t)((uintptr_t)(&_end) - (uintptr_t)(&_sdata)); - - memcpy(to, from, sz); -} -#endif - #ifdef CONFIG_STRICT_KERNEL_RWX static __meminit pgprot_t pgprot_from_va(uintptr_t va) { @@ -775,8 +755,9 @@ static __meminit pgprot_t pgprot_from_va(uintptr_t va) } #endif /* CONFIG_STRICT_KERNEL_RWX */ -#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_64BIT) u64 __pi_set_satp_mode_from_cmdline(uintptr_t dtb_pa); +u64 __pi_set_satp_mode_from_fdt(uintptr_t dtb_pa); static void __init disable_pgtable_l5(void) { @@ -811,21 +792,48 @@ static void __init set_mmap_rnd_bits_max(void) mmap_rnd_bits_max = MMAP_VA_BITS - PAGE_SHIFT - 3; } +static bool __init is_vaddr_valid(unsigned long va) +{ + unsigned long up = 0; + + switch (satp_mode) { + case SATP_MODE_39: + up = 1UL << 38; + break; + case SATP_MODE_48: + up = 1UL << 47; + break; + case SATP_MODE_57: + up = 1UL << 56; + break; + default: + return false; + } + + return (va < up) || (va >= (ULONG_MAX - up + 1)); +} + /* * There is a simple way to determine if 4-level is supported by the * underlying hardware: establish 1:1 mapping in 4-level page table mode * then read SATP to see if the configuration was taken into account * meaning sv48 is supported. + * The maximum SATP mode is limited by both the command line and the "mmu-type" + * property in the device tree, since some platforms may hang if an unsupported + * SATP mode is attempted. */ static __init void set_satp_mode(uintptr_t dtb_pa) { u64 identity_satp, hw_satp; uintptr_t set_satp_mode_pmd = ((unsigned long)set_satp_mode) & PMD_MASK; - u64 satp_mode_cmdline = __pi_set_satp_mode_from_cmdline(dtb_pa); + u64 satp_mode_limit = min_not_zero(__pi_set_satp_mode_from_cmdline(dtb_pa), + __pi_set_satp_mode_from_fdt(dtb_pa)); - if (satp_mode_cmdline == SATP_MODE_57) { + kernel_map.page_offset = PAGE_OFFSET_L5; + + if (satp_mode_limit == SATP_MODE_48) { disable_pgtable_l5(); - } else if (satp_mode_cmdline == SATP_MODE_48) { + } else if (satp_mode_limit == SATP_MODE_39) { disable_pgtable_l5(); disable_pgtable_l4(); return; @@ -846,6 +854,9 @@ static __init void set_satp_mode(uintptr_t dtb_pa) set_satp_mode_pmd + PMD_SIZE, PMD_SIZE, PAGE_KERNEL_EXEC); retry: + if (!is_vaddr_valid(set_satp_mode_pmd)) + goto out; + create_pgd_mapping(early_pg_dir, set_satp_mode_pmd, pgtable_l5_enabled ? @@ -868,6 +879,7 @@ retry: disable_pgtable_l4(); } +out: memset(early_pg_dir, 0, PAGE_SIZE); memset(early_p4d, 0, PAGE_SIZE); memset(early_pud, 0, PAGE_SIZE); @@ -893,66 +905,6 @@ retry: #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." #endif -#ifdef CONFIG_RELOCATABLE -extern unsigned long __rela_dyn_start, __rela_dyn_end; - -static void __init relocate_kernel(void) -{ - Elf64_Rela *rela = (Elf64_Rela *)&__rela_dyn_start; - /* - * This holds the offset between the linked virtual address and the - * relocated virtual address. - */ - uintptr_t reloc_offset = kernel_map.virt_addr - KERNEL_LINK_ADDR; - /* - * This holds the offset between kernel linked virtual address and - * physical address. - */ - uintptr_t va_kernel_link_pa_offset = KERNEL_LINK_ADDR - kernel_map.phys_addr; - - for ( ; rela < (Elf64_Rela *)&__rela_dyn_end; rela++) { - Elf64_Addr addr = (rela->r_offset - va_kernel_link_pa_offset); - Elf64_Addr relocated_addr = rela->r_addend; - - if (rela->r_info != R_RISCV_RELATIVE) - continue; - - /* - * Make sure to not relocate vdso symbols like rt_sigreturn - * which are linked from the address 0 in vmlinux since - * vdso symbol addresses are actually used as an offset from - * mm->context.vdso in VDSO_OFFSET macro. - */ - if (relocated_addr >= KERNEL_LINK_ADDR) - relocated_addr += reloc_offset; - - *(Elf64_Addr *)addr = relocated_addr; - } -} -#endif /* CONFIG_RELOCATABLE */ - -#ifdef CONFIG_XIP_KERNEL -static void __init create_kernel_page_table(pgd_t *pgdir, - __always_unused bool early) -{ - uintptr_t va, start_va, end_va; - - /* Map the flash resident part */ - end_va = kernel_map.virt_addr + kernel_map.xiprom_sz; - for (va = kernel_map.virt_addr; va < end_va; va += PMD_SIZE) - create_pgd_mapping(pgdir, va, - kernel_map.xiprom + (va - kernel_map.virt_addr), - PMD_SIZE, PAGE_KERNEL_EXEC); - - /* Map the data in RAM */ - start_va = kernel_map.virt_addr + (uintptr_t)&_sdata - (uintptr_t)&_start; - end_va = kernel_map.virt_addr + kernel_map.size; - for (va = start_va; va < end_va; va += PMD_SIZE) - create_pgd_mapping(pgdir, va, - kernel_map.phys_addr + (va - start_va), - PMD_SIZE, PAGE_KERNEL); -} -#else static void __init create_kernel_page_table(pgd_t *pgdir, bool early) { uintptr_t va, end_va; @@ -965,7 +917,6 @@ static void __init create_kernel_page_table(pgd_t *pgdir, bool early) early ? PAGE_KERNEL_EXEC : pgprot_from_va(va)); } -#endif /* * Setup a 4MB mapping that encompasses the device tree: for 64-bit kernel, @@ -1075,11 +1026,6 @@ static int __init print_nokaslr(char *p) return 0; } early_param("nokaslr", print_nokaslr); - -unsigned long kaslr_offset(void) -{ - return kernel_map.virt_offset; -} #endif asmlinkage void __init setup_vm(uintptr_t dtb_pa) @@ -1107,33 +1053,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset; -#ifdef CONFIG_XIP_KERNEL -#ifdef CONFIG_64BIT - kernel_map.page_offset = PAGE_OFFSET_L3; -#else - kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); -#endif - kernel_map.xiprom = (uintptr_t)CONFIG_XIP_PHYS_ADDR; - kernel_map.xiprom_sz = (uintptr_t)(&_exiprom) - (uintptr_t)(&_xiprom); - - phys_ram_base = CONFIG_PHYS_RAM_BASE; -#ifdef CONFIG_SPARSEMEM_VMEMMAP - vmemmap_start_pfn = round_down(phys_ram_base, VMEMMAP_ADDR_ALIGN) >> PAGE_SHIFT; -#endif - kernel_map.phys_addr = (uintptr_t)CONFIG_PHYS_RAM_BASE; - kernel_map.size = (uintptr_t)(&_end) - (uintptr_t)(&_start); - - kernel_map.va_kernel_xip_text_pa_offset = kernel_map.virt_addr - kernel_map.xiprom; - kernel_map.va_kernel_xip_data_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr - + (uintptr_t)&_sdata - (uintptr_t)&_start; -#else - kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL); kernel_map.phys_addr = (uintptr_t)(&_start); kernel_map.size = (uintptr_t)(&_end) - kernel_map.phys_addr; kernel_map.va_kernel_pa_offset = kernel_map.virt_addr - kernel_map.phys_addr; -#endif -#if defined(CONFIG_64BIT) && !defined(CONFIG_XIP_KERNEL) +#if defined(CONFIG_64BIT) set_satp_mode(dtb_pa); set_mmap_rnd_bits_max(); #endif @@ -1174,7 +1098,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) * makes the kernel cross over a PUD_SIZE boundary, raise a bug * since a part of the kernel would not get mapped. */ - BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size); + if (IS_ENABLED(CONFIG_64BIT)) + BUG_ON(PUD_SIZE - (kernel_map.virt_addr & (PUD_SIZE - 1)) < kernel_map.size); relocate_kernel(); #endif @@ -1205,13 +1130,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) if (pgtable_l4_enabled) create_pud_mapping(trampoline_pud, kernel_map.virt_addr, (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); -#ifdef CONFIG_XIP_KERNEL - create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, - kernel_map.xiprom, PMD_SIZE, PAGE_KERNEL_EXEC); -#else create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, kernel_map.phys_addr, PMD_SIZE, PAGE_KERNEL_EXEC); -#endif #else /* Setup trampoline PGD */ create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, @@ -1378,6 +1298,12 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) { dtb_early_va = (void *)dtb_pa; dtb_early_pa = dtb_pa; + +#ifdef CONFIG_RELOCATABLE + kernel_map.virt_addr = (uintptr_t)_start; + kernel_map.phys_addr = (uintptr_t)_start; + relocate_kernel(); +#endif } static inline void setup_vm_final(void) @@ -1396,21 +1322,19 @@ static void __init arch_reserve_crashkernel(void) { unsigned long long low_size = 0; unsigned long long crash_base, crash_size; - char *cmdline = boot_command_line; bool high = false; int ret; if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) return; - ret = parse_crashkernel(cmdline, memblock_phys_mem_size(), + ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), &crash_size, &crash_base, - &low_size, &high); + &low_size, NULL, &high); if (ret) return; - reserve_crashkernel_generic(cmdline, crash_size, crash_base, - low_size, high); + reserve_crashkernel_generic(crash_size, crash_base, low_size, high); } void __init paging_init(void) @@ -1426,12 +1350,10 @@ void __init misc_mem_init(void) { early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); arch_numa_init(); - sparse_init(); #ifdef CONFIG_SPARSEMEM_VMEMMAP /* The entire VMEMMAP region has been populated. Flush TLB for this region */ local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END); #endif - zone_sizes_init(); arch_reserve_crashkernel(); memblock_dump_all(); } @@ -1453,6 +1375,8 @@ int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node, int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { + WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); + /* * Note that SPARSEMEM_VMEMMAP is only selected for rv64 and that we * can't use hugepage mappings for 2-level page table because in case of @@ -1622,7 +1546,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) if (PageReserved(page)) free_reserved_page(page); else - free_pages((unsigned long)page_address(page), 0); + __free_pages(page, 0); p4d_clear(p4d); } @@ -1644,7 +1568,7 @@ static void __meminit free_vmemmap_storage(struct page *page, size_t size, return; } - free_pages((unsigned long)page_address(page), order); + __free_pages(page, order); } static void __meminit remove_pte_mapping(pte_t *pte_base, unsigned long addr, unsigned long end, diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index 41c635d6aca4..c4a2a9e5586e 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -533,4 +533,5 @@ void __init kasan_init(void) csr_write(CSR_SATP, PFN_DOWN(__pa(swapper_pg_dir)) | satp_mode); local_flush_tlb_all(); + kasan_init_generic(); } diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index d815448758a1..3f76db3d2769 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -299,7 +299,7 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, if (ret) goto unlock; - ret = walk_page_range_novma(&init_mm, lm_start, lm_end, + ret = walk_kernel_page_table_range(lm_start, lm_end, &pageattr_ops, NULL, &masks); if (ret) goto unlock; @@ -317,13 +317,13 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, if (ret) goto unlock; - ret = walk_page_range_novma(&init_mm, lm_start, lm_end, + ret = walk_kernel_page_table_range(lm_start, lm_end, &pageattr_ops, NULL, &masks); if (ret) goto unlock; } - ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, + ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, &masks); unlock: @@ -335,7 +335,7 @@ unlock: */ flush_tlb_all(); #else - ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, + ret = walk_kernel_page_table_range(start, end, &pageattr_ops, NULL, &masks); mmap_write_unlock(&init_mm); diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 4ae67324f992..9c4427d0b187 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -9,8 +9,16 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) { + if (!pte_same(ptep_get(ptep), entry)) { + __set_pte_at(vma->vm_mm, ptep, entry); + /* Here only not svadu is impacted */ + flush_tlb_page(vma, address); + return true; + } + + return false; + } if (!pte_same(ptep_get(ptep), entry)) __set_pte_at(vma->vm_mm, ptep, entry); @@ -19,24 +27,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma, * the case that the PTE changed and the spurious fault case. */ return true; - -svvptc: - if (!pte_same(ptep_get(ptep), entry)) { - __set_pte_at(vma->vm_mm, ptep, entry); - /* Here only not svadu is impacted */ - flush_tlb_page(vma, address); - return true; - } - - return false; } -int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep) +bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { if (!pte_young(ptep_get(ptep))) - return 0; + return false; return test_and_clear_bit(_PAGE_ACCESSED_OFFSET, &pte_val(*ptep)); } EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); @@ -49,6 +46,7 @@ pud_t *pud_offset(p4d_t *p4d, unsigned long address) return (pud_t *)p4d; } +EXPORT_SYMBOL_GPL(pud_offset); p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { @@ -57,6 +55,7 @@ p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) return (p4d_t *)pgd; } +EXPORT_SYMBOL_GPL(p4d_offset); #endif #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP @@ -154,4 +153,30 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, flush_tlb_mm(vma->vm_mm); return pmd; } + +pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, + pud_t *pudp) +{ + VM_WARN_ON_ONCE(!pud_present(*pudp)); + pud_t old = pudp_establish(vma, address, pudp, pud_mkinvalid(*pudp)); + + flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE); + return old; +} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma) +{ + if (vma->vm_flags & VM_SHADOW_STACK) + return pte_mkwrite_shstk(pte); + + return pte_mkwrite_novma(pte); +} + +pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) +{ + if (vma->vm_flags & VM_SHADOW_STACK) + return pmd_mkwrite_shstk(pmd); + + return pmd_mkwrite_novma(pmd); +} diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c index 18706f457da7..b1836e9481ce 100644 --- a/arch/riscv/mm/physaddr.c +++ b/arch/riscv/mm/physaddr.c @@ -9,10 +9,10 @@ phys_addr_t __virt_to_phys(unsigned long x) { /* - * Boundary checking aginst the kernel linear mapping space. + * Boundary checking against the kernel linear mapping space. */ WARN(!is_linear_mapping(x) && !is_kernel_mapping(x), - "virt_to_phys used for non-linear address: %pK (%pS)\n", + "virt_to_phys used for non-linear address: %p (%pS)\n", (void *)x, (void *)x); return __va_to_pa_nodebug(x); @@ -25,7 +25,7 @@ phys_addr_t __phys_addr_symbol(unsigned long x) unsigned long kernel_end = kernel_start + kernel_map.size; /* - * Boundary checking aginst the kernel image mapping. + * Boundary checking against the kernel image mapping. * __pa_symbol should only be used on kernel symbol addresses. */ VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end); diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 9d5f657a251b..34299c2b231f 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -6,7 +6,6 @@ #include <linux/efi.h> #include <linux/init.h> #include <linux/debugfs.h> -#include <linux/memory_hotplug.h> #include <linux/seq_file.h> #include <linux/ptdump.h> @@ -22,7 +21,7 @@ #define pt_dump_seq_puts(m, fmt) \ ({ \ if (m) \ - seq_printf(m, fmt); \ + seq_puts(m, fmt); \ }) /* @@ -318,6 +317,38 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr, } } +static void note_page_pte(struct ptdump_state *pt_st, unsigned long addr, pte_t pte) +{ + note_page(pt_st, addr, 4, pte_val(pte)); +} + +static void note_page_pmd(struct ptdump_state *pt_st, unsigned long addr, pmd_t pmd) +{ + note_page(pt_st, addr, 3, pmd_val(pmd)); +} + +static void note_page_pud(struct ptdump_state *pt_st, unsigned long addr, pud_t pud) +{ + note_page(pt_st, addr, 2, pud_val(pud)); +} + +static void note_page_p4d(struct ptdump_state *pt_st, unsigned long addr, p4d_t p4d) +{ + note_page(pt_st, addr, 1, p4d_val(p4d)); +} + +static void note_page_pgd(struct ptdump_state *pt_st, unsigned long addr, pgd_t pgd) +{ + note_page(pt_st, addr, 0, pgd_val(pgd)); +} + +static void note_page_flush(struct ptdump_state *pt_st) +{ + pte_t pte_zero = {0}; + + note_page(pt_st, 0, -1, pte_val(pte_zero)); +} + static void ptdump_walk(struct seq_file *s, struct ptd_mm_info *pinfo) { struct pg_state st = { @@ -325,7 +356,12 @@ static void ptdump_walk(struct seq_file *s, struct ptd_mm_info *pinfo) .marker = pinfo->markers, .level = -1, .ptdump = { - .note_page = note_page, + .note_page_pte = note_page_pte, + .note_page_pmd = note_page_pmd, + .note_page_pud = note_page_pud, + .note_page_p4d = note_page_p4d, + .note_page_pgd = note_page_pgd, + .note_page_flush = note_page_flush, .range = (struct ptdump_range[]) { {pinfo->base_addr, pinfo->end}, {0, 0} @@ -347,7 +383,12 @@ bool ptdump_check_wx(void) .level = -1, .check_wx = true, .ptdump = { - .note_page = note_page, + .note_page_pte = note_page_pte, + .note_page_pmd = note_page_pmd, + .note_page_pud = note_page_pud, + .note_page_p4d = note_page_p4d, + .note_page_pgd = note_page_pgd, + .note_page_flush = note_page_flush, .range = (struct ptdump_range[]) { {KERN_VIRT_START, ULONG_MAX}, {0, 0} @@ -371,9 +412,7 @@ bool ptdump_check_wx(void) static int ptdump_show(struct seq_file *m, void *v) { - get_online_mems(); ptdump_walk(m, m->private); - put_online_mems(); return 0; } diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 9b6e86ce3867..8404530ec00f 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -4,8 +4,30 @@ #include <linux/smp.h> #include <linux/sched.h> #include <linux/hugetlb.h> +#include <linux/mmu_notifier.h> #include <asm/sbi.h> #include <asm/mmu_context.h> +#include <asm/cpufeature.h> + +#define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) + +static inline void local_sfence_inval_ir(void) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval(void) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma(unsigned long vma, unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} /* * Flush entire TLB if number of entries to be flushed is greater @@ -26,6 +48,16 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start, return; } + if (has_svinval()) { + local_sfence_w_inval(); + for (i = 0; i < nr_ptes_in_range; ++i) { + local_sinval_vma(start, asid); + start += stride; + } + local_sfence_inval_ir(); + return; + } + for (i = 0; i < nr_ptes_in_range; ++i) { local_flush_tlb_page_asid(start, asid); start += stride; @@ -78,10 +110,17 @@ static void __ipi_flush_tlb_range_asid(void *info) local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); } -static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid, +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static void __flush_tlb_range(struct mm_struct *mm, + const struct cpumask *cmask, unsigned long start, unsigned long size, unsigned long stride) { + unsigned long asid = get_mm_asid(mm); unsigned int cpu; if (cpumask_empty(cmask)) @@ -105,30 +144,26 @@ static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid, } put_cpu(); -} -static inline unsigned long get_mm_asid(struct mm_struct *mm) -{ - return cntx2asid(atomic_long_read(&mm->context.id)); + if (mm) + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size); } void flush_tlb_mm(struct mm_struct *mm) { - __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), - 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); + __flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); } void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int page_size) { - __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), - start, end - start, page_size); + __flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), addr, PAGE_SIZE, PAGE_SIZE); } @@ -161,13 +196,13 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, } } - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), start, end - start, stride_size); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - __flush_tlb_range(cpu_online_mask, FLUSH_TLB_NO_ASID, + __flush_tlb_range(NULL, cpu_online_mask, start, end - start, PAGE_SIZE); } @@ -175,9 +210,16 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), start, end - start, PMD_SIZE); } + +void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), + start, end - start, PUD_SIZE); +} #endif bool arch_tlbbatch_should_defer(struct mm_struct *mm) @@ -186,20 +228,15 @@ bool arch_tlbbatch_should_defer(struct mm_struct *mm) } void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, - struct mm_struct *mm, - unsigned long uaddr) + struct mm_struct *mm, unsigned long start, unsigned long end) { cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); -} - -void arch_flush_tlb_batched_pending(struct mm_struct *mm) -{ - flush_tlb_mm(mm); + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0, - FLUSH_TLB_MAX_SIZE, PAGE_SIZE); + __flush_tlb_range(NULL, &batch->cpumask, + 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); cpumask_clear(&batch->cpumask); } diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 1d1c78d4cff1..da0271790244 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -11,23 +11,16 @@ #include <linux/bpf.h> #include <linux/filter.h> -#include <asm/cacheflush.h> + +/* verify runtime detection extension status */ +#define rv_ext_enabled(ext) \ + (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) static inline bool rvc_enabled(void) { return IS_ENABLED(CONFIG_RISCV_ISA_C); } -static inline bool rvzba_enabled(void) -{ - return IS_ENABLED(CONFIG_RISCV_ISA_ZBA) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBA); -} - -static inline bool rvzbb_enabled(void) -{ - return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB); -} - enum { RV_REG_ZERO = 0, /* The constant value 0 */ RV_REG_RA = 1, /* Return address */ @@ -84,6 +77,8 @@ struct rv_jit_context { int epilogue_offset; int *offset; /* BPF to RV */ int nexentries; + int ex_insn_off; + int ex_jmp_off; unsigned long flags; int stack_size; u64 arena_vm_start; @@ -109,11 +104,6 @@ static inline void bpf_fill_ill_insns(void *area, unsigned int size) memset(area, 0, size); } -static inline void bpf_flush_icache(void *start, void *end) -{ - flush_icache_range((unsigned long)start, (unsigned long)end); -} - /* Emit a 4-byte riscv instruction. */ static inline void emit(const u32 insn, struct rv_jit_context *ctx) { @@ -608,6 +598,21 @@ static inline u32 rv_fence(u8 pred, u8 succ) return rv_i_insn(imm11_0, 0, 0, 0, 0xf); } +static inline void emit_fence_r_rw(struct rv_jit_context *ctx) +{ + emit(rv_fence(0x2, 0x3), ctx); +} + +static inline void emit_fence_rw_w(struct rv_jit_context *ctx) +{ + emit(rv_fence(0x3, 0x1), ctx); +} + +static inline void emit_fence_rw_rw(struct rv_jit_context *ctx) +{ + emit(rv_fence(0x3, 0x3), ctx); +} + static inline u32 rv_nop(void) { return rv_i_insn(0, 0, 0, 0, 0x13); @@ -742,6 +747,17 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2) return rv_css_insn(0x6, imm, rs2, 0x2); } +/* RVZACAS instructions. */ +static inline u32 rvzacas_amocas_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) +{ + return rv_amo_insn(0x5, aq, rl, rs2, rs1, 2, rd, 0x2f); +} + +static inline u32 rvzacas_amocas_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) +{ + return rv_amo_insn(0x5, aq, rl, rs2, rs1, 3, rd, 0x2f); +} + /* RVZBA instructions. */ static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2) { @@ -1108,7 +1124,7 @@ static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) { - if (rvzba_enabled()) { + if (rv_ext_enabled(ZBA)) { emit(rvzba_sh2add(rd, rs1, rs2), ctx); return; } @@ -1119,7 +1135,7 @@ static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) { - if (rvzba_enabled()) { + if (rv_ext_enabled(ZBA)) { emit(rvzba_sh3add(rd, rs1, rs2), ctx); return; } @@ -1169,7 +1185,7 @@ static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) { - if (rvzbb_enabled()) { + if (rv_ext_enabled(ZBB)) { emit(rvzbb_sextb(rd, rs), ctx); return; } @@ -1180,7 +1196,7 @@ static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) { - if (rvzbb_enabled()) { + if (rv_ext_enabled(ZBB)) { emit(rvzbb_sexth(rd, rs), ctx); return; } @@ -1196,7 +1212,7 @@ static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) { - if (rvzbb_enabled()) { + if (rv_ext_enabled(ZBB)) { emit(rvzbb_zexth(rd, rs), ctx); return; } @@ -1207,7 +1223,7 @@ static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) { - if (rvzba_enabled()) { + if (rv_ext_enabled(ZBA)) { emit(rvzba_zextw(rd, rs), ctx); return; } @@ -1218,7 +1234,7 @@ static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) { - if (rvzbb_enabled()) { + if (rv_ext_enabled(ZBB)) { int bits = 64 - imm; emit(rvzbb_rev8(rd, rd), ctx); @@ -1274,6 +1290,35 @@ out_be: emit_mv(rd, RV_REG_T2, ctx); } +static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx) +{ + int jmp_offset; + + if (rv_ext_enabled(ZACAS)) { + ctx->ex_insn_off = ctx->ninsns; + emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) : + rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; + if (!is64) + emit_zextw(r0, r0, ctx); + return; + } + + if (is64) + emit_mv(RV_REG_T2, r0, ctx); + else + emit_addiw(RV_REG_T2, r0, 0, ctx); + emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : + rv_lr_w(r0, 0, rd, 0, 0), ctx); + jmp_offset = ninsns_rvoff(8); + emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); + emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) : + rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); + jmp_offset = ninsns_rvoff(-6); + emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); + emit_fence_rw_rw(ctx); +} + #endif /* __riscv_xlen == 64 */ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index ca60db75199d..2f1109dbf105 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,7 +18,7 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI_CLANG) ? 1 : 0) +#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF @@ -469,15 +469,152 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) { - if (IS_ENABLED(CONFIG_CFI_CLANG)) + if (IS_ENABLED(CONFIG_CFI)) emit(hash, ctx); } -static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, - struct rv_jit_context *ctx) +static void emit_ldx_insn(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, + struct rv_jit_context *ctx) { - u8 r0; - int jmp_offset; + switch (size) { + case BPF_B: + emit(sign_ext ? rv_lb(rd, off, rs) : rv_lbu(rd, off, rs), ctx); + break; + case BPF_H: + emit(sign_ext ? rv_lh(rd, off, rs) : rv_lhu(rd, off, rs), ctx); + break; + case BPF_W: + emit(sign_ext ? rv_lw(rd, off, rs) : rv_lwu(rd, off, rs), ctx); + break; + case BPF_DW: + emit_ld(rd, off, rs, ctx); + break; + } + +} + +static void emit_stx_insn(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) +{ + switch (size) { + case BPF_B: + emit(rv_sb(rd, off, rs), ctx); + break; + case BPF_H: + emit(rv_sh(rd, off, rs), ctx); + break; + case BPF_W: + emit_sw(rd, off, rs, ctx); + break; + case BPF_DW: + emit_sd(rd, off, rs, ctx); + break; + } +} + +static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, + struct rv_jit_context *ctx) +{ + if (is_12b_int(off)) { + ctx->ex_insn_off = ctx->ninsns; + emit_ldx_insn(rd, off, rs, size, sign_ext, ctx); + ctx->ex_jmp_off = ctx->ninsns; + return; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); + ctx->ex_insn_off = ctx->ninsns; + emit_ldx_insn(rd, 0, RV_REG_T1, size, sign_ext, ctx); + ctx->ex_jmp_off = ctx->ninsns; +} + +static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) +{ + emit_imm(RV_REG_T1, imm, ctx); + if (is_12b_int(off)) { + ctx->ex_insn_off = ctx->ninsns; + emit_stx_insn(rd, off, RV_REG_T1, size, ctx); + ctx->ex_jmp_off = ctx->ninsns; + return; + } + + emit_imm(RV_REG_T2, off, ctx); + emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); + ctx->ex_insn_off = ctx->ninsns; + emit_stx_insn(RV_REG_T2, 0, RV_REG_T1, size, ctx); + ctx->ex_jmp_off = ctx->ninsns; +} + +static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) +{ + if (is_12b_int(off)) { + ctx->ex_insn_off = ctx->ninsns; + emit_stx_insn(rd, off, rs, size, ctx); + ctx->ex_jmp_off = ctx->ninsns; + return; + } + + emit_imm(RV_REG_T1, off, ctx); + emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); + ctx->ex_insn_off = ctx->ninsns; + emit_stx_insn(RV_REG_T1, 0, rs, size, ctx); + ctx->ex_jmp_off = ctx->ninsns; +} + +static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, + struct rv_jit_context *ctx) +{ + u8 code = insn->code; + s32 imm = insn->imm; + s16 off = insn->off; + + switch (imm) { + /* dst_reg = load_acquire(src_reg + off16) */ + case BPF_LOAD_ACQ: + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { + emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); + rs = RV_REG_T2; + } + + emit_ldx(rd, off, rs, BPF_SIZE(code), false, ctx); + emit_fence_r_rw(ctx); + + /* If our next insn is a redundant zext, return 1 to tell + * build_body() to skip it. + */ + if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) + return 1; + break; + /* store_release(dst_reg + off16, src_reg) */ + case BPF_STORE_REL: + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { + emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); + rd = RV_REG_T2; + } + + emit_fence_rw_w(ctx); + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + break; + default: + pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm); + return -EINVAL; + } + + return 0; +} + +static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn, + struct rv_jit_context *ctx) +{ + u8 code = insn->code; + s16 off = insn->off; + s32 imm = insn->imm; + bool is64 = BPF_SIZE(code) == BPF_DW; + + if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) { + pr_err_once("bpf-jit: 1- and 2-byte RMW atomics are not supported\n"); + return -EINVAL; + } if (off) { if (is_12b_int(off)) { @@ -489,74 +626,122 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, rd = RV_REG_T1; } + if (BPF_MODE(code) == BPF_PROBE_ATOMIC) { + emit_add(RV_REG_T1, rd, RV_REG_ARENA, ctx); + rd = RV_REG_T1; + } + switch (imm) { /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ case BPF_ADD: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); + ctx->ex_jmp_off = ctx->ninsns; break; case BPF_AND: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); + ctx->ex_jmp_off = ctx->ninsns; break; case BPF_OR: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); + ctx->ex_jmp_off = ctx->ninsns; break; case BPF_XOR: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); + ctx->ex_jmp_off = ctx->ninsns; break; /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ case BPF_ADD | BPF_FETCH: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) : rv_amoadd_w(rs, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; if (!is64) emit_zextw(rs, rs, ctx); break; case BPF_AND | BPF_FETCH: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) : rv_amoand_w(rs, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; if (!is64) emit_zextw(rs, rs, ctx); break; case BPF_OR | BPF_FETCH: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) : rv_amoor_w(rs, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; if (!is64) emit_zextw(rs, rs, ctx); break; case BPF_XOR | BPF_FETCH: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) : rv_amoxor_w(rs, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; if (!is64) emit_zextw(rs, rs, ctx); break; /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ case BPF_XCHG: + ctx->ex_insn_off = ctx->ninsns; emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) : rv_amoswap_w(rs, rs, rd, 1, 1), ctx); + ctx->ex_jmp_off = ctx->ninsns; if (!is64) emit_zextw(rs, rs, ctx); break; /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ case BPF_CMPXCHG: - r0 = bpf_to_rv_reg(BPF_REG_0, ctx); - if (is64) - emit_mv(RV_REG_T2, r0, ctx); - else - emit_addiw(RV_REG_T2, r0, 0, ctx); - emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : - rv_lr_w(r0, 0, rd, 0, 0), ctx); - jmp_offset = ninsns_rvoff(8); - emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); - emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) : - rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); - jmp_offset = ninsns_rvoff(-6); - emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); - emit(rv_fence(0x3, 0x3), ctx); + emit_cmpxchg(rd, rs, regmap[BPF_REG_0], is64, ctx); + break; + default: + pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm); + return -EINVAL; + } + + return 0; +} + +/* + * Sign-extend the register if necessary + */ +static int sign_extend(u8 rd, u8 rs, u8 sz, bool sign, struct rv_jit_context *ctx) +{ + if (!sign && (sz == 1 || sz == 2)) { + if (rd != rs) + emit_mv(rd, rs, ctx); + return 0; + } + + switch (sz) { + case 1: + emit_sextb(rd, rs, ctx); + break; + case 2: + emit_sexth(rd, rs, ctx); + break; + case 4: + emit_sextw(rd, rs, ctx); + break; + case 8: + if (rd != rs) + emit_mv(rd, rs, ctx); break; + default: + pr_err("bpf-jit: invalid size %d for sign_extend\n", sz); + return -EINVAL; } + + return 0; } #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) @@ -577,9 +762,8 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, } /* For accesses to BTF pointers, add an entry to the exception table */ -static int add_exception_handler(const struct bpf_insn *insn, - struct rv_jit_context *ctx, - int dst_reg, int insn_len) +static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, + struct rv_jit_context *ctx) { struct exception_table_entry *ex; unsigned long pc; @@ -587,21 +771,23 @@ static int add_exception_handler(const struct bpf_insn *insn, off_t fixup_offset; if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || - (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && - BPF_MODE(insn->code) != BPF_PROBE_MEM32)) + ctx->ex_insn_off <= 0 || ctx->ex_jmp_off <= 0) return 0; - if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) - return -EINVAL; + if (BPF_MODE(insn->code) != BPF_PROBE_MEM && + BPF_MODE(insn->code) != BPF_PROBE_MEMSX && + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) + return 0; - if (WARN_ON_ONCE(insn_len > ctx->ninsns)) + if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) return -EINVAL; - if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) + if (WARN_ON_ONCE(ctx->ex_insn_off > ctx->ninsns || ctx->ex_jmp_off > ctx->ninsns)) return -EINVAL; ex = &ctx->prog->aux->extable[ctx->nexentries]; - pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; + pc = (unsigned long)&ctx->ro_insns[ctx->ex_insn_off]; /* * This is the relative offset of the instruction that may fault from @@ -625,7 +811,7 @@ static int add_exception_handler(const struct bpf_insn *insn, * that may fault. The execution will jump to this after handling the * fault. */ - fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); + fixup_offset = (long)&ex->fixup - (long)&ctx->ro_insns[ctx->ex_jmp_off]; if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) return -ERANGE; @@ -642,6 +828,8 @@ static int add_exception_handler(const struct bpf_insn *insn, FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); ex->type = EX_TYPE_BPF; + ctx->ex_insn_off = 0; + ctx->ex_jmp_off = 0; ctx->nexentries++; return 0; } @@ -664,17 +852,19 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); } -int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, - void *old_addr, void *new_addr) +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, + enum bpf_text_poke_type new_t, void *old_addr, + void *new_addr) { u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; - bool is_call = poke_type == BPF_MOD_CALL; + bool is_call; int ret; if (!is_kernel_text((unsigned long)ip) && !is_bpf_text_address((unsigned long)ip)) return -ENOTSUPP; + is_call = old_t == BPF_MOD_CALL; ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); if (ret) return ret; @@ -682,6 +872,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, if (memcmp(ip, old_insns, RV_FENTRY_NBYTES)) return -EFAULT; + is_call = new_t == BPF_MOD_CALL; ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); if (ret) return ret; @@ -735,6 +926,14 @@ static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off, } } +static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64, + struct rv_jit_context *ctx) +{ + /* Load imm64 into reg and store it at [FP + stack_off]. */ + emit_imm(reg, (s64)imm64, ctx); + emit_sd(RV_REG_FP, stack_off, reg, ctx); +} + static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) { @@ -742,12 +941,10 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of struct bpf_prog *p = l->link.prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); - if (l->cookie) { - emit_imm(RV_REG_T1, l->cookie, ctx); - emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); - } else { + if (l->cookie) + emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, l->cookie, ctx); + else emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); - } /* arg1: prog */ emit_imm(RV_REG_A0, (const s64)p, ctx); @@ -799,6 +996,29 @@ static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_of return ret; } +static int invoke_bpf(struct bpf_tramp_links *tl, int args_off, int retval_off, + int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta, + int cookie_off, struct rv_jit_context *ctx) +{ + int i, cur_cookie = (cookie_off - args_off) / 8; + + for (i = 0; i < tl->nr_links; i++) { + int err; + + if (bpf_prog_calls_session_cookie(tl->links[i])) { + u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx); + cur_cookie--; + } + err = invoke_bpf_prog(tl->links[i], args_off, retval_off, run_ctx_off, + save_ret, ctx); + if (err) + return err; + } + return 0; +} + static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_links *tlinks, @@ -808,13 +1028,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int i, ret, offset; int *branches_off = NULL; int stack_size = 0, nr_arg_slots = 0; - int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; + int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; + int cookie_off, cookie_cnt; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; void *orig_call = func_addr; bool save_ret; + u64 func_meta; u32 insn; /* Two types of generated trampoline stack layout: @@ -845,10 +1067,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * [ ... ] * FP - args_off [ arg1 ] * - * FP - nregs_off [ regs count ] + * FP - func_meta_off [ regs count, etc ] * * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG * + * [ stack cookie N ] + * [ ... ] + * FP - cookie_off [ stack cookie 1 ] + * * FP - run_ctx_off [ bpf_tramp_run_ctx ] * * FP - sreg_off [ callee saved reg ] @@ -873,22 +1099,27 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 16; save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); - if (save_ret) { + if (save_ret) stack_size += 16; /* Save both A5 (BPF R0) and A0 */ - retval_off = stack_size; - } + retval_off = stack_size; stack_size += nr_arg_slots * 8; args_off = stack_size; + /* function metadata, such as regs count */ stack_size += 8; - nregs_off = stack_size; + func_meta_off = stack_size; if (flags & BPF_TRAMP_F_IP_ARG) { stack_size += 8; ip_off = stack_size; } + cookie_cnt = bpf_fsession_cookie_cnt(tlinks); + /* room for session cookies */ + stack_size += cookie_cnt * 8; + cookie_off = stack_size; + stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); run_ctx_off = stack_size; @@ -933,19 +1164,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); /* store ip address of the traced function */ - if (flags & BPF_TRAMP_F_IP_ARG) { - emit_imm(RV_REG_T1, (const s64)func_addr, ctx); - emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); - } + if (flags & BPF_TRAMP_F_IP_ARG) + emit_store_stack_imm64(RV_REG_T1, -ip_off, (u64)func_addr, ctx); - emit_li(RV_REG_T1, nr_arg_slots, ctx); - emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); + func_meta = nr_arg_slots; + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); store_args(nr_arg_slots, args_off, ctx); - /* skip to actual body of traced function */ - if (flags & BPF_TRAMP_F_SKIP_FRAME) - orig_call += RV_FENTRY_NINSNS * 4; + if (bpf_fsession_cnt(tlinks)) { + /* clear all session cookies' value */ + for (i = 0; i < cookie_cnt; i++) + emit_sd(RV_REG_FP, -cookie_off + 8 * i, RV_REG_ZERO, ctx); + /* clear return value to make sure fentry always get 0 */ + emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); + } if (flags & BPF_TRAMP_F_CALL_ORIG) { emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); @@ -954,15 +1187,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return ret; } - for (i = 0; i < fentry->nr_links; i++) { - ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, - flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); + if (fentry->nr_links) { + ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off, + flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx); if (ret) return ret; } if (fmod_ret->nr_links) { - branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); + branches_off = kzalloc_objs(int, fmod_ret->nr_links); if (!branches_off) return -ENOMEM; @@ -981,6 +1214,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_CALL_ORIG) { + /* skip to actual body of traced function */ + orig_call += RV_FENTRY_NINSNS * 4; restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); ret = emit_call((const u64)orig_call, true, ctx); @@ -1001,9 +1236,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, *(u32 *)(ctx->insns + branches_off[i]) = insn; } - for (i = 0; i < fexit->nr_links; i++) { - ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, - run_ctx_off, false, ctx); + /* set "is_return" flag for fsession */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (bpf_fsession_cnt(tlinks)) + emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); + + if (fexit->nr_links) { + ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off, + false, func_meta, cookie_off, ctx); if (ret) goto out; } @@ -1020,8 +1260,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); if (save_ret) { - emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); + if (is_struct_ops) { + ret = sign_extend(RV_REG_A0, regmap[BPF_REG_0], m->ret_size, + m->ret_flags & BTF_FMODEL_SIGNED_ARG, ctx); + if (ret) + goto out; + } else { + emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); + } } emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); @@ -1114,7 +1361,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, goto out; } - bpf_flush_icache(ro_image, ro_image_end); out: kvfree(image); return ret < 0 ? ret : size; @@ -1150,7 +1396,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_mv(rd, rs, ctx); #ifdef CONFIG_SMP /* Load current CPU number in T1 */ - emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu), + emit_lw(RV_REG_T1, offsetof(struct thread_info, cpu), RV_REG_TP, ctx); /* Load address of __per_cpu_offset array in T2 */ emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx); @@ -1557,7 +1803,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, */ if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { /* Load current CPU number in R0 */ - emit_ld(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu), + emit_lw(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu), RV_REG_TP, ctx); break; } @@ -1650,7 +1896,6 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: { - int insn_len, insns_start; bool sign_ext; sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || @@ -1661,84 +1906,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rs = RV_REG_T2; } - switch (BPF_SIZE(code)) { - case BPF_B: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lb(rd, off, rs), ctx); - else - emit(rv_lbu(rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lb(rd, 0, RV_REG_T1), ctx); - else - emit(rv_lbu(rd, 0, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_H: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lh(rd, off, rs), ctx); - else - emit(rv_lhu(rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lh(rd, 0, RV_REG_T1), ctx); - else - emit(rv_lhu(rd, 0, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_W: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lw(rd, off, rs), ctx); - else - emit(rv_lwu(rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - insns_start = ctx->ninsns; - if (sign_ext) - emit(rv_lw(rd, 0, RV_REG_T1), ctx); - else - emit(rv_lwu(rd, 0, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_DW: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit_ld(rd, off, rs, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); - insns_start = ctx->ninsns; - emit_ld(rd, 0, RV_REG_T1, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - ret = add_exception_handler(insn, ctx, rd, insn_len); + ret = add_exception_handler(insn, rd, ctx); if (ret) return ret; @@ -1746,260 +1916,73 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return 1; break; } + /* speculation barrier */ case BPF_ST | BPF_NOSPEC: break; /* ST: *(size *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - emit_imm(RV_REG_T1, imm, ctx); - if (is_12b_int(off)) { - emit(rv_sb(rd, off, RV_REG_T1), ctx); - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); - break; - case BPF_ST | BPF_MEM | BPF_H: - emit_imm(RV_REG_T1, imm, ctx); - if (is_12b_int(off)) { - emit(rv_sh(rd, off, RV_REG_T1), ctx); - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); - break; case BPF_ST | BPF_MEM | BPF_W: - emit_imm(RV_REG_T1, imm, ctx); - if (is_12b_int(off)) { - emit_sw(rd, off, RV_REG_T1, ctx); - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); - break; case BPF_ST | BPF_MEM | BPF_DW: - emit_imm(RV_REG_T1, imm, ctx); - if (is_12b_int(off)) { - emit_sd(rd, off, RV_REG_T1, ctx); - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); - break; - + /* ST | PROBE_MEM32: *(size *)(dst + RV_REG_ARENA + off) = imm */ case BPF_ST | BPF_PROBE_MEM32 | BPF_B: case BPF_ST | BPF_PROBE_MEM32 | BPF_H: case BPF_ST | BPF_PROBE_MEM32 | BPF_W: case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: - { - int insn_len, insns_start; - - emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); - rd = RV_REG_T3; - - /* Load imm to a register then store it */ - emit_imm(RV_REG_T1, imm, ctx); - - switch (BPF_SIZE(code)) { - case BPF_B: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit(rv_sb(rd, off, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - insns_start = ctx->ninsns; - emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_H: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit(rv_sh(rd, off, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - insns_start = ctx->ninsns; - emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_W: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit_sw(rd, off, RV_REG_T1, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - insns_start = ctx->ninsns; - emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_DW: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit_sd(rd, off, RV_REG_T1, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T2, off, ctx); - emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); - insns_start = ctx->ninsns; - emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); - insn_len = ctx->ninsns - insns_start; - break; + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); + rd = RV_REG_T3; } - ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, - insn_len); + emit_st(rd, off, imm, BPF_SIZE(code), ctx); + + ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) return ret; - break; - } /* STX: *(size *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_B: - if (is_12b_int(off)) { - emit(rv_sb(rd, off, rs), ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - emit(rv_sb(RV_REG_T1, 0, rs), ctx); - break; case BPF_STX | BPF_MEM | BPF_H: - if (is_12b_int(off)) { - emit(rv_sh(rd, off, rs), ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - emit(rv_sh(RV_REG_T1, 0, rs), ctx); - break; case BPF_STX | BPF_MEM | BPF_W: - if (is_12b_int(off)) { - emit_sw(rd, off, rs, ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - emit_sw(RV_REG_T1, 0, rs, ctx); - break; case BPF_STX | BPF_MEM | BPF_DW: - if (is_12b_int(off)) { - emit_sd(rd, off, rs, ctx); - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - emit_sd(RV_REG_T1, 0, rs, ctx); - break; - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - emit_atomic(rd, rs, off, imm, - BPF_SIZE(code) == BPF_DW, ctx); - break; - + /* STX | PROBE_MEM32: *(size *)(dst + RV_REG_ARENA + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_B: case BPF_STX | BPF_PROBE_MEM32 | BPF_H: case BPF_STX | BPF_PROBE_MEM32 | BPF_W: case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: - { - int insn_len, insns_start; - - emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); - rd = RV_REG_T2; - - switch (BPF_SIZE(code)) { - case BPF_B: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit(rv_sb(rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } - - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - insns_start = ctx->ninsns; - emit(rv_sb(RV_REG_T1, 0, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_H: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit(rv_sh(rd, off, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - } + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); + rd = RV_REG_T2; + } - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - insns_start = ctx->ninsns; - emit(rv_sh(RV_REG_T1, 0, rs), ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_W: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit_sw(rd, off, rs, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - insns_start = ctx->ninsns; - emit_sw(RV_REG_T1, 0, rs, ctx); - insn_len = ctx->ninsns - insns_start; - break; - case BPF_DW: - if (is_12b_int(off)) { - insns_start = ctx->ninsns; - emit_sd(rd, off, rs, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } + ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + if (ret) + return ret; + break; - emit_imm(RV_REG_T1, off, ctx); - emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); - insns_start = ctx->ninsns; - emit_sd(RV_REG_T1, 0, rs, ctx); - insn_len = ctx->ninsns - insns_start; - break; - } + /* Atomics */ + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_H: + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: + if (bpf_atomic_is_load_store(insn)) + ret = emit_atomic_ld_st(rd, rs, insn, ctx); + else + ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, - insn_len); + ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) return ret; - break; - } default: pr_err("bpf-jit: unknown opcode %02x\n", code); @@ -2127,6 +2110,25 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) +{ + if (in_arena) { + switch (insn->code) { + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + if (insn->imm == BPF_CMPXCHG) + return rv_ext_enabled(ZACAS); + break; + case BPF_LDX | BPF_MEMSX | BPF_B: + case BPF_LDX | BPF_MEMSX | BPF_H: + case BPF_LDX | BPF_MEMSX | BPF_W: + return false; + } + } + + return true; +} + bool bpf_jit_supports_percpu_insn(void) { return true; @@ -2141,3 +2143,8 @@ bool bpf_jit_inlines_helper_call(s32 imm) return false; } } + +bool bpf_jit_supports_fsession(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index f8cd2f70a7fb..4365d07aaf54 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -26,9 +26,8 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) int ret; ret = bpf_jit_emit_insn(insn, ctx, extra_pass); - /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ if (ret > 0) - i++; + i++; /* skip the next instruction */ if (offset) offset[i] = ctx->ninsns; if (ret < 0) @@ -42,32 +41,22 @@ bool bpf_jit_needs_zext(void) return true; } -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; - bool tmp_blinded = false, extra_pass = false; - struct bpf_prog *tmp, *orig_prog = prog; + bool extra_pass = false; int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { - jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + jit_data = kzalloc_obj(*jit_data); if (!jit_data) { - prog = orig_prog; - goto out; + return prog; } prog->aux->jit_data = jit_data; } @@ -83,16 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; - ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); - if (!ctx->offset) { - prog = orig_prog; + ctx->offset = kzalloc_objs(int, prog->len); + if (!ctx->offset) goto out_offset; - } - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_offset; - } for (i = 0; i < prog->len; i++) { prev_ninsns += 32; @@ -106,10 +91,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); ctx->prologue_len = ctx->ninsns; - if (build_body(ctx, extra_pass, ctx->offset)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, ctx->offset)) goto out_offset; - } ctx->epilogue_offset = ctx->ninsns; bpf_jit_build_epilogue(ctx); @@ -127,10 +110,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, bpf_fill_ill_insns); - if (!jit_data->ro_header) { - prog = orig_prog; + if (!jit_data->ro_header) goto out_offset; - } /* * Use the image(RW) for writing the JITed instructions. But also save @@ -151,7 +132,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (i == NR_JIT_ITERATIONS) { pr_err("bpf-jit: image did not converge in <%d passes!\n", i); - prog = orig_prog; goto out_free_hdr; } @@ -164,33 +144,27 @@ skip_init_ctx: ctx->nexentries = 0; bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_free_hdr; - } bpf_jit_build_epilogue(ctx); if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); - prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); - prog->jited = 1; - prog->jited_len = prog_size - cfi_get_offset(); - if (!prog->is_func || extra_pass) { if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { /* ro_header has been freed */ jit_data->ro_header = NULL; - prog = orig_prog; - goto out_offset; + jit_data->header = NULL; + goto out_free_hdr; } - /* - * The instructions have now been copied to the ROX region from - * where they will execute. - * Write any modified data cache blocks out to memory and - * invalidate the corresponding blocks in the instruction cache. - */ - bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns); + } + + prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); + prog->jited = 1; + prog->jited_len = prog_size - cfi_get_offset(); + + if (!prog->is_func || extra_pass) { for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); @@ -199,14 +173,15 @@ out_offset: kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); return prog; out_free_hdr: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } if (jit_data->header) { bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, sizeof(jit_data->header->size)); diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile index fb9c917c9b45..b0358a78f11a 100644 --- a/arch/riscv/purgatory/Makefile +++ b/arch/riscv/purgatory/Makefile @@ -2,7 +2,7 @@ purgatory-y := purgatory.o sha256.o entry.o string.o ctype.o memcpy.o memset.o ifeq ($(CONFIG_KASAN_GENERIC)$(CONFIG_KASAN_SW_TAGS),) -purgatory-y += strcmp.o strlen.o strncmp.o +purgatory-y += strcmp.o strlen.o strncmp.o strnlen.o strchr.o strrchr.o endif targets += $(purgatory-y) @@ -32,6 +32,15 @@ $(obj)/strncmp.o: $(srctree)/arch/riscv/lib/strncmp.S FORCE $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE $(call if_changed_rule,cc_o_c) +$(obj)/strnlen.o: $(srctree)/arch/riscv/lib/strnlen.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strchr.o: $(srctree)/arch/riscv/lib/strchr.S FORCE + $(call if_changed_rule,as_o_S) + +$(obj)/strrchr.o: $(srctree)/arch/riscv/lib/strrchr.S FORCE + $(call if_changed_rule,as_o_S) + CFLAGS_sha256.o := -D__DISABLE_EXPORTS -D__NO_FORTIFY CFLAGS_string.o := -D__DISABLE_EXPORTS CFLAGS_ctype.o := -D__DISABLE_EXPORTS @@ -53,7 +62,7 @@ targets += purgatory.ro purgatory.chk PURGATORY_CFLAGS_REMOVE := -mcmodel=kernel PURGATORY_CFLAGS := -mcmodel=medany -ffreestanding -fno-zero-initialized-in-bss -PURGATORY_CFLAGS += $(DISABLE_STACKLEAK_PLUGIN) -DDISABLE_BRANCH_PROFILING +PURGATORY_CFLAGS += $(DISABLE_KSTACK_ERASE) -DDISABLE_BRANCH_PROFILING PURGATORY_CFLAGS += -fno-stack-protector -g0 # Default KBUILD_CFLAGS can have -pg option set when FTRACE is enabled. That @@ -71,7 +80,7 @@ ifdef CONFIG_STACKPROTECTOR_STRONG PURGATORY_CFLAGS_REMOVE += -fstack-protector-strong endif -ifdef CONFIG_CFI_CLANG +ifdef CONFIG_CFI PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI) endif diff --git a/arch/riscv/purgatory/entry.S b/arch/riscv/purgatory/entry.S index 0e6ca6d5ae4b..c5db2f072c34 100644 --- a/arch/riscv/purgatory/entry.S +++ b/arch/riscv/purgatory/entry.S @@ -12,6 +12,7 @@ .text +.align 2 SYM_CODE_START(purgatory_start) lla sp, .Lstack diff --git a/arch/riscv/purgatory/kexec-purgatory.S b/arch/riscv/purgatory/kexec-purgatory.S index 0e9188815718..32c53581b8f2 100644 --- a/arch/riscv/purgatory/kexec-purgatory.S +++ b/arch/riscv/purgatory/kexec-purgatory.S @@ -6,9 +6,9 @@ kexec_purgatory: .globl kexec_purgatory .incbin "arch/riscv/purgatory/purgatory.ro" -.Lkexec_purgatroy_end: +.Lkexec_purgatory_end: .align 8 kexec_purgatory_size: .globl kexec_purgatory_size - .quad .Lkexec_purgatroy_end - kexec_purgatory + .quad .Lkexec_purgatory_end - kexec_purgatory diff --git a/arch/riscv/purgatory/purgatory.c b/arch/riscv/purgatory/purgatory.c index 80596ab5fb62..bbd5cfa4d741 100644 --- a/arch/riscv/purgatory/purgatory.c +++ b/arch/riscv/purgatory/purgatory.c @@ -20,14 +20,14 @@ struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX] __section(".kex static int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; - struct sha256_state ss; + struct sha256_ctx sctx; u8 digest[SHA256_DIGEST_SIZE]; - sha256_init(&ss); + sha256_init(&sctx); end = purgatory_sha_regions + ARRAY_SIZE(purgatory_sha_regions); for (ptr = purgatory_sha_regions; ptr < end; ptr++) - sha256_update(&ss, (uint8_t *)(ptr->start), ptr->len); - sha256_final(&ss, digest); + sha256_update(&sctx, (uint8_t *)(ptr->start), ptr->len); + sha256_final(&sctx, digest); if (memcmp(digest, purgatory_sha256_digest, sizeof(digest)) != 0) return 1; return 0; diff --git a/arch/riscv/tools/relocs_check.sh b/arch/riscv/tools/relocs_check.sh index baeb2e7b2290..742993e6a8cb 100755 --- a/arch/riscv/tools/relocs_check.sh +++ b/arch/riscv/tools/relocs_check.sh @@ -14,7 +14,9 @@ bad_relocs=$( ${srctree}/scripts/relocs_check.sh "$@" | # These relocations are okay # R_RISCV_RELATIVE - grep -F -w -v 'R_RISCV_RELATIVE' + # R_RISCV_NONE + grep -F -w -v 'R_RISCV_RELATIVE +R_RISCV_NONE' ) if [ -z "$bad_relocs" ]; then |