diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-30 09:48:45 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-30 09:48:45 -1000 |
| commit | 17fa96334e15c6b38e8d9dd3627f3a6c5ca034f8 (patch) | |
| tree | 5260bdaf3379444d54901567005349f77e5935e6 | |
| parent | 6e07889d8fedf729282ed14ff202e06dbb67444b (diff) | |
| parent | 1689c80fca308978ea1c9f03d15a500f4e44e2e1 (diff) | |
| download | linux-17fa96334e15c6b38e8d9dd3627f3a6c5ca034f8.tar.gz linux-17fa96334e15c6b38e8d9dd3627f3a6c5ca034f8.zip | |
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
Pull more KVM updates from Paolo Bonzini:
"ARM64:
- Fix a couple of embarrassing merge mistakes in kvmarm-7.4
- Cleanup the HCR_EL2 bit fields to exclusively use the generated
definitions, fixings a nasty regression introduced in 6.16, where
barriers were not upgraded from NSH to ISH anymore
- Fix publication of state when a vcpu is forced out of the guest
- Let vcpu pausing nest correctly
- Sanitise ID_AA64MMFR3_EL1.S1P[IO]E to follow the rules of the
architecture
- Don't warn on SMCCC filter reserved range allocation failure
- Prevent GICv5 and TLBI hypercalls to be used from the host when
pKVM is active
- Enforce that only GICv3 guests are supported for pKVM
- Don't try to access GICv5 EL2 sysregs from EL1
- Compute architectrally correct values for ID_AA64MMFR3_EL1.S1P[IO]E
by checking the feature dependencies
- Require matching VGIC and ITS model
- Cleanup TRACE_REMOTE dependencies
- Fix random comments
ARM64 selftests:
- selftests: Skip unwritable ID fields in the failed-run test
- Move pkvm_selftests() after all CPUs have initialized EL2
RISC-V:
- Allow Zicclsm, Ziccamoa, Ziccif, Za64rs for KVM guest
- Implement KVM_PRE_FAULT_MEMORY ioctl()
- Replace imsic switch-case macros with range checks
RISC-V fixes:
- Avoid redundant interrupt check for pending timers
- Reject vCPU creation after AIA initialization
- Fix build with vector support disabled
- Fix IPI delivery for out-of-order vcpu_id
- Use a raw lock for IMSIC wakeup state
- Reject vCPU creation after AIA initialization
RISC-V selftests:
- Lazy V-extension enablement for KVM selftests
- Show sepc/scause/stval CSRs in KVM selftests vcpu dump"
* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (42 commits)
KVM: selftests: Reject GICv3 ITS with a GICv5 VGIC
KVM: arm64: vgic: Require matching VGIC and ITS models
KVM: arm64: gic-v5: Drop unused addr variable
KVM: arm64: Sanitise ID_AA64MMFR3_EL1.S1P[IO]E
KVM: arm64: selftests: Skip unwritable ID fields in the failed-run test
tracing/remotes: Fix TRACE_REMOTE build when TRACING=n
KVM: arm64: Move pkvm_selftests() after all CPUs have initialized EL2
KVM: arm64: Get rid of compatibility HCR_EL2 aliases
KVM: arm64: Convert HCR_xyz to HCR_EL2_xyz
KVM: arm64: Set HCR_EL2.BSU to Inner Shareable for guests
KVM: arm64: Fix stale VGICv3 comments in the nVHE world switch
KVM: arm64: vgic: Do not access the GICv5 CPU interface from EL1
KVM: arm64: Validate the host-provided vgic model in pKVM
KVM: arm64: Reject the stage-2 flush hypercalls under pKVM
KVM: arm64: Return on kvm_gmem_get_pfn() failure in gmem_abort()
KVM: riscv: selftest: add Zicclsm, Ziccamoa, Ziccif, Za64rs
RISC-V: KVM: Allow Zicclsm, Ziccamoa, Ziccif, Za64rs in guest
MAINTAINERS: name the kvm-riscv/linux riscv_kvm_next branch
RISC-V: KVM: Don't write MMIO load result to x0
RISC-V: KVM: Use a raw lock for IMSIC wakeup state
...
61 files changed, 767 insertions, 464 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 3f7bbb9c2915..adbe6d9c5658 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14681,7 +14681,7 @@ L: kvm@vger.kernel.org L: kvm-riscv@lists.infradead.org L: linux-riscv@lists.infradead.org S: Maintained -T: git https://github.com/kvm-riscv/linux.git +T: git https://github.com/kvm-riscv/linux.git riscv_kvm_next F: arch/riscv/include/asm/kvm* F: arch/riscv/include/uapi/asm/kvm* F: arch/riscv/kvm/ diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 1da277baacf7..12a500181291 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -61,7 +61,7 @@ b.ne .LnVHE_\@ // Otherwise, we know that nVHE works. .LnE2H0_\@: - orr x0, x0, #HCR_E2H + orr x0, x0, #HCR_EL2_E2H msr_hcr_el2 x0 .LnVHE_\@: .endm @@ -102,7 +102,7 @@ /* Check if running in host at EL2 mode, i.e., (h)VHE. Jump to fail if not. */ .macro __check_hvhe fail, tmp mrs \tmp, hcr_el2 - and \tmp, \tmp, #HCR_E2H + and \tmp, \tmp, #HCR_EL2_E2H cbz \tmp, \fail .endm diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index 77d6b8c63d4e..024e5a24973e 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -40,8 +40,8 @@ do { \ } \ \ ___hcr = read_sysreg(hcr_el2); \ - if (!(___hcr & HCR_TGE)) { \ - write_sysreg_hcr(___hcr | HCR_TGE); \ + if (!(___hcr & HCR_EL2_TGE)) { \ + write_sysreg_hcr(___hcr | HCR_EL2_TGE); \ isb(); \ } \ /* \ @@ -81,7 +81,7 @@ do { \ * we secured things for it. \ */ \ barrier(); \ - if (!___ctx->cnt && !(___hcr & HCR_TGE)) \ + if (!___ctx->cnt && !(___hcr & HCR_EL2_TGE)) \ write_sysreg_hcr(___hcr); \ } while (0) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 678f0daed445..ba0062849be9 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -15,71 +15,6 @@ #include <asm/sysreg.h> /* - * Because I'm terribly lazy and that repainting the whole of the KVM - * code with the proper names is a pain, use a helper to map the names - * inherited from AArch32 with the new fancy nomenclature. One day... - */ -#define __HCR(x) HCR_EL2_##x - -#define HCR_TID5 __HCR(TID5) -#define HCR_DCT __HCR(DCT) -#define HCR_ATA_SHIFT __HCR(ATA_SHIFT) -#define HCR_ATA __HCR(ATA) -#define HCR_TTLBOS __HCR(TTLBOS) -#define HCR_TTLBIS __HCR(TTLBIS) -#define HCR_ENSCXT __HCR(EnSCXT) -#define HCR_TOCU __HCR(TOCU) -#define HCR_AMVOFFEN __HCR(AMVOFFEN) -#define HCR_TICAB __HCR(TICAB) -#define HCR_TID4 __HCR(TID4) -#define HCR_FIEN __HCR(FIEN) -#define HCR_FWB __HCR(FWB) -#define HCR_NV2 __HCR(NV2) -#define HCR_AT __HCR(AT) -#define HCR_NV1 __HCR(NV1) -#define HCR_NV __HCR(NV) -#define HCR_API __HCR(API) -#define HCR_APK __HCR(APK) -#define HCR_TEA __HCR(TEA) -#define HCR_TERR __HCR(TERR) -#define HCR_TLOR __HCR(TLOR) -#define HCR_E2H __HCR(E2H) -#define HCR_ID __HCR(ID) -#define HCR_CD __HCR(CD) -#define HCR_RW __HCR(RW) -#define HCR_TRVM __HCR(TRVM) -#define HCR_HCD __HCR(HCD) -#define HCR_TDZ __HCR(TDZ) -#define HCR_TGE __HCR(TGE) -#define HCR_TVM __HCR(TVM) -#define HCR_TTLB __HCR(TTLB) -#define HCR_TPU __HCR(TPU) -#define HCR_TPC __HCR(TPCP) -#define HCR_TSW __HCR(TSW) -#define HCR_TACR __HCR(TACR) -#define HCR_TIDCP __HCR(TIDCP) -#define HCR_TSC __HCR(TSC) -#define HCR_TID3 __HCR(TID3) -#define HCR_TID2 __HCR(TID2) -#define HCR_TID1 __HCR(TID1) -#define HCR_TID0 __HCR(TID0) -#define HCR_TWE __HCR(TWE) -#define HCR_TWI __HCR(TWI) -#define HCR_DC __HCR(DC) -#define HCR_BSU __HCR(BSU) -#define HCR_BSU_IS __HCR(BSU_IS) -#define HCR_FB __HCR(FB) -#define HCR_VSE __HCR(VSE) -#define HCR_VI __HCR(VI) -#define HCR_VF __HCR(VF) -#define HCR_AMO __HCR(AMO) -#define HCR_IMO __HCR(IMO) -#define HCR_FMO __HCR(FMO) -#define HCR_PTW __HCR(PTW) -#define HCR_SWIO __HCR(SWIO) -#define HCR_VM __HCR(VM) - -/* * The bits we set in HCR: * TLOR: Trap LORegion register accesses * RW: 64bit by default, can be overridden for 32bit VMs @@ -99,13 +34,13 @@ * TID3: Trap EL1 reads of group 3 ID registers * TID1: Trap REVIDR_EL1, AIDR_EL1, and SMIDR_EL1 */ -#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ - HCR_BSU_IS | HCR_FB | HCR_TACR | \ - HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ - HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1) -#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) -#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) -#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H | HCR_AMO | HCR_IMO | HCR_FMO) +#define HCR_GUEST_FLAGS (HCR_EL2_TSC | HCR_EL2_TSW | HCR_EL2_TWE | HCR_EL2_TWI | HCR_EL2_VM | \ + SYS_FIELD_PREP_ENUM(HCR_EL2, BSU, IS) | HCR_EL2_FB | HCR_EL2_TACR | \ + HCR_EL2_AMO | HCR_EL2_SWIO | HCR_EL2_TIDCP | HCR_EL2_RW | HCR_EL2_TLOR | \ + HCR_EL2_FMO | HCR_EL2_IMO | HCR_EL2_PTW | HCR_EL2_TID3 | HCR_EL2_TID1) +#define HCR_HOST_NVHE_FLAGS (HCR_EL2_RW | HCR_EL2_API | HCR_EL2_APK) +#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_EL2_TSC) +#define HCR_HOST_VHE_FLAGS (HCR_EL2_RW | HCR_EL2_TGE | HCR_EL2_E2H | HCR_EL2_AMO | HCR_EL2_IMO | HCR_EL2_FMO) #define MPAMHCR_HOST_FLAGS 0 diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 7d799cd05dcd..abda476e023d 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -63,6 +63,17 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, __KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend, + __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, + __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, + __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), @@ -71,12 +82,6 @@ enum __kvm_host_smccc_func { /* Hypercalls that are always available and common to [nh]VHE/pKVM. */ __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, - __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, - __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, __KVM_HOST_SMCCC_FUNC___tracing_load, __KVM_HOST_SMCCC_FUNC___tracing_unload, @@ -88,11 +93,6 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___tracing_write_event, __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, - __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, - __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, - __KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend, - __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, - __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, __KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup, __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim, __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable, diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 85289a4b8d52..334211b8b2ee 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -95,7 +95,7 @@ static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu) #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__) static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) { - return !(vcpu->arch.hcr_el2 & HCR_RW); + return !(vcpu->arch.hcr_el2 & HCR_EL2_RW); } #else static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) @@ -116,7 +116,7 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) * then. */ if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_TVM; + vcpu->arch.hcr_el2 |= HCR_EL2_TVM; } static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) @@ -227,12 +227,12 @@ static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu) static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu) { return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) || - (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H)); + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_E2H)); } static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu) { - return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE; + return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_TGE; } static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) @@ -251,7 +251,7 @@ static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu)) return true; - return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO; + return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_AMO; } static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) @@ -264,8 +264,8 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - e2h = (hcr & HCR_E2H); - tge = (hcr & HCR_TGE); + e2h = (hcr & HCR_EL2_E2H); + tge = (hcr & HCR_EL2_TGE); /* * We are in a hypervisor context if the vcpu mode is EL2 or @@ -376,8 +376,8 @@ static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu) if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) return false; - return ((is_wfe && (hcr_el2 & HCR_TWE)) || - (!is_wfe && (hcr_el2 & HCR_TWI))); + return ((is_wfe && (hcr_el2 & HCR_EL2_TWE)) || + (!is_wfe && (hcr_el2 & HCR_EL2_TWI))); } static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 6dd8462145c6..97332ffcb130 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -967,11 +967,10 @@ struct kvm_vcpu_arch { /* * Don't run the guest (internal implementation need). * - * Contrary to the flags above, this is set/cleared outside of - * a vcpu context, and thus cannot be mixed with the flags - * themselves (or the flag accesses need to be made atomic). + * Contrary to the flags above, this is updated outside of + * a vcpu context, and thus cannot be mixed with the flags. */ - bool pause; + atomic_t pause; /* * We maintain more than a single set of debug registers to support @@ -1355,6 +1354,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); +void kvm_pause_vcpu(struct kvm_vcpu *vcpu); +void kvm_resume_vcpu(struct kvm_vcpu *vcpu); #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid)) @@ -1746,6 +1747,7 @@ static inline bool kvm_arch_has_irq_bypass(void) return true; } +void kvm_idregs_enforce_dependencies(struct kvm *kvm); void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt); struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg); void check_feature_map(void); diff --git a/arch/arm64/include/asm/kvm_mte.h b/arch/arm64/include/asm/kvm_mte.h index 3171963ad25c..502d6df91691 100644 --- a/arch/arm64/include/asm/kvm_mte.h +++ b/arch/arm64/include/asm/kvm_mte.h @@ -16,7 +16,7 @@ alternative_if_not ARM64_MTE b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + tbz \reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@ mrs_s \reg1, SYS_RGSR_EL1 str \reg1, [\h_ctxt, #CPU_RGSR_EL1] @@ -36,7 +36,7 @@ alternative_if_not ARM64_MTE b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + tbz \reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@ mrs_s \reg1, SYS_RGSR_EL1 str \reg1, [\g_ctxt, #CPU_RGSR_EL1] diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index e7562770e7cb..fd042d2b5462 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -17,7 +17,6 @@ #define HYP_MEMBLOCK_REGIONS 128 -void pkvm_selftests(void); int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); diff --git a/arch/arm64/include/asm/kvm_ptrauth.h b/arch/arm64/include/asm/kvm_ptrauth.h index e50987b32483..24f75457a712 100644 --- a/arch/arm64/include/asm/kvm_ptrauth.h +++ b/arch/arm64/include/asm/kvm_ptrauth.h @@ -71,7 +71,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - and \reg1, \reg1, #(HCR_API | HCR_APK) + and \reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK) cbz \reg1, .L__skip_switch\@ add \reg1, \g_ctxt, #CPU_APIAKEYLO_EL1 ptrauth_restore_state \reg1, \reg2, \reg3 @@ -83,7 +83,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - and \reg1, \reg1, #(HCR_API | HCR_APK) + and \reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK) cbz \reg1, .L__skip_switch\@ add \reg1, \g_ctxt, #CPU_APIAKEYLO_EL1 ptrauth_save_state \reg1, \reg2, \reg3 diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 8951ce693552..d6ba3c0af811 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -298,7 +298,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) isb 0: - init_el2_hcr HCR_HOST_NVHE_FLAGS | HCR_ATA + init_el2_hcr HCR_HOST_NVHE_FLAGS | HCR_EL2_ATA init_el2_state /* Hypervisor stub */ @@ -309,7 +309,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) mov_q x1, INIT_SCTLR_EL1_MMU_OFF mrs x0, hcr_el2 - and x0, x0, #HCR_E2H + and x0, x0, #HCR_EL2_E2H cbz x0, 2f /* Set a sane SCTLR_EL1, the VHE way */ diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 48d531e302d3..cecea0820e52 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -71,7 +71,7 @@ if NVHE_EL2_DEBUG config NVHE_EL2_TRACING bool - depends on TRACING && FTRACE + depends on FTRACE select TRACE_REMOTE default y diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d30378a50c53..157bf4b81202 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -590,6 +590,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_arm_pvtime_vcpu_init(&vcpu->arch); vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; + atomic_set(&vcpu->arch.pause, 0); /* * This vCPU may have been created after mpidr_data was initialized. @@ -650,11 +651,11 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) u64 val; val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + val &= (HCR_EL2_API | HCR_EL2_APK); + vcpu->arch.hcr_el2 &= ~(HCR_EL2_API | HCR_EL2_APK); vcpu->arch.hcr_el2 |= val; } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK); } /* @@ -662,7 +663,7 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) * to use pauth, as the entry code will reload the guest * keys in that case. */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + if (vcpu->arch.hcr_el2 & (HCR_EL2_API | HCR_EL2_APK)) { struct kvm_cpu_context *ctxt; ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); @@ -736,14 +737,14 @@ static void vcpu_put_mmu(struct kvm_vcpu *vcpu) static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) { if (kvm_vcpu_should_clear_twe(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_TWE; + vcpu->arch.hcr_el2 &= ~HCR_EL2_TWE; else - vcpu->arch.hcr_el2 |= HCR_TWE; + vcpu->arch.hcr_el2 |= HCR_EL2_TWE; if (kvm_vcpu_should_clear_twi(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_TWI; + vcpu->arch.hcr_el2 &= ~HCR_EL2_TWI; else - vcpu->arch.hcr_el2 |= HCR_TWI; + vcpu->arch.hcr_el2 |= HCR_EL2_TWI; } static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) @@ -954,6 +955,11 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, return ret; } +static bool vcpu_can_run(struct kvm_vcpu *vcpu) +{ + return !kvm_arm_vcpu_stopped(vcpu) && !atomic_read(&vcpu->arch.pause); +} + /** * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled * @v: The VCPU pointer @@ -963,14 +969,13 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, */ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) { - bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE); + bool irq_lines = *vcpu_hcr(v) & (HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_VSE); irq_lines |= (!irqchip_in_kernel(v->kvm) && (kvm_timer_should_notify_user(v) || kvm_pmu_should_notify_user(v))); - return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) - && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); + return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) && vcpu_can_run(v)); } bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) @@ -1127,13 +1132,28 @@ bool kvm_arch_intc_initialized(struct kvm *kvm) return vgic_initialized(kvm); } +void kvm_pause_vcpu(struct kvm_vcpu *vcpu) +{ + DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS) = { }; + + atomic_inc(&vcpu->arch.pause); + __set_bit(vcpu->vcpu_idx, vcpu_bitmap); + kvm_make_vcpus_request_mask(vcpu->kvm, KVM_REQ_SLEEP, vcpu_bitmap); +} + +void kvm_resume_vcpu(struct kvm_vcpu *vcpu) +{ + if (atomic_dec_and_test(&vcpu->arch.pause)) + __kvm_vcpu_wake_up(vcpu); +} + void kvm_arm_halt_guest(struct kvm *kvm) { unsigned long i; struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) - vcpu->arch.pause = true; + atomic_inc(&vcpu->arch.pause); kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); } @@ -1142,21 +1162,17 @@ void kvm_arm_resume_guest(struct kvm *kvm) unsigned long i; struct kvm_vcpu *vcpu; - kvm_for_each_vcpu(i, vcpu, kvm) { - vcpu->arch.pause = false; - __kvm_vcpu_wake_up(vcpu); - } + kvm_for_each_vcpu(i, vcpu, kvm) + kvm_resume_vcpu(vcpu); } static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) { struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); - rcuwait_wait_event(wait, - (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), - TASK_INTERRUPTIBLE); + rcuwait_wait_event(wait, vcpu_can_run(vcpu), TASK_INTERRUPTIBLE); - if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) { + if (!vcpu_can_run(vcpu)) { /* Awaken to handle a signal, request we sleep again later. */ kvm_make_request(KVM_REQ_SLEEP, vcpu); } @@ -1437,38 +1453,40 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) */ preempt_disable(); - kvm_nested_flush_hwstate(vcpu); - if (kvm_vcpu_has_pmu(vcpu)) kvm_pmu_flush_hwstate(vcpu); - local_irq_disable(); - - kvm_vgic_flush_hwstate(vcpu); + kvm_nested_flush_hwstate(vcpu); - kvm_pmu_update_vcpu_events(vcpu); + local_irq_disable(); /* * Ensure we set mode to IN_GUEST_MODE after we disable * interrupts and before the final VCPU requests check. * See the comment in kvm_vcpu_exiting_guest_mode() and * Documentation/virt/kvm/vcpu-requests.rst + * + * Requests to self must be posted before this point, or + * won't be taken into account until the next exit. No state + * loading other than what happens at vcpu_load() time can + * be done before we have checked for pending requests. */ smp_store_mb(vcpu->mode, IN_GUEST_MODE); if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) { - vcpu->mode = OUTSIDE_GUEST_MODE; - isb(); /* Ensure work in x_flush_hwstate is committed */ - if (kvm_vcpu_has_pmu(vcpu)) - kvm_pmu_sync_hwstate(vcpu); if (unlikely(!irqchip_in_kernel(vcpu->kvm))) kvm_timer_sync_user(vcpu); - kvm_vgic_sync_hwstate(vcpu); + smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); local_irq_enable(); preempt_enable(); continue; } + /* From here onwards, we are commited to entering the guest */ + kvm_vgic_flush_hwstate(vcpu); + + kvm_pmu_update_vcpu_events(vcpu); + kvm_arch_vcpu_ctxflush_fp(vcpu); /************************************************************** @@ -1479,7 +1497,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) ret = kvm_arm_vcpu_enter_exit(vcpu); - vcpu->mode = OUTSIDE_GUEST_MODE; vcpu->stat.exits++; /* * Back from guest @@ -1514,6 +1531,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_arch_vcpu_ctxsync_fp(vcpu); /* + * The state that was pushed into the CPU on entering + * the guest has been synchronised back, let's + * advertise we're outside of the guest. + */ + smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); + + /* * We must ensure that any pending interrupts are taken before * we exit guest timing so that timer ticks are accounted as * guest time. Transiently unmask interrupts so that any @@ -1592,9 +1616,9 @@ static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) unsigned long *hcr; if (number == KVM_ARM_IRQ_CPU_IRQ) - bit_index = __ffs(HCR_VI); + bit_index = __ffs(HCR_EL2_VI); else /* KVM_ARM_IRQ_CPU_FIQ */ - bit_index = __ffs(HCR_VF); + bit_index = __ffs(HCR_EL2_VF); hcr = vcpu_hcr(vcpu); if (level) @@ -2384,12 +2408,12 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) params->hcr_el2 = HCR_HOST_NVHE_FLAGS; if (system_supports_mte()) - params->hcr_el2 |= HCR_ATA; + params->hcr_el2 |= HCR_EL2_ATA; else - params->hcr_el2 |= HCR_TID5; + params->hcr_el2 |= HCR_EL2_TID5; if (cpus_have_final_cap(ARM64_KVM_HVHE)) - params->hcr_el2 |= HCR_E2H; + params->hcr_el2 |= HCR_EL2_E2H; params->vttbr = params->vtcr = 0; /* @@ -3109,8 +3133,6 @@ static int __init init_hyp_mode(void) kvm_err("Failed to init hyp memory protection\n"); goto out_err; } - - pkvm_selftests(); } return 0; diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 0926426b8798..30d8f062b2c2 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -229,7 +229,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, if (vcpu_has_nv(vcpu)) { hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); + wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_EL2_VM | HCR_EL2_DC)); } else { WARN_ON_ONCE(wi->regime != TR_EL10); wi->s2 = false; @@ -305,7 +305,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, * is EL1&0, then it means that either {E2H,TGE} == {1,0} or * {0,x}, and we only need to test for TGE == 1. */ - if (hcr & (HCR_DC | HCR_TGE)) { + if (hcr & (HCR_EL2_DC | HCR_EL2_TGE)) { wr->level = S1_MMU_DISABLED; break; } @@ -870,7 +870,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, s1_parattr = FIELD_GET(SYS_PAR_EL1_ATTR, s1_par); s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc); - if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_FWB) { + if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_FWB) { if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP)) s2_memattr &= ~BIT(3); @@ -924,7 +924,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, } } - if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_CD) && + if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_CD) && !MEMATTR_IS_DEVICE(final_attr)) final_attr = MEMATTR(NC, NC); @@ -956,7 +956,7 @@ static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, par |= wr->pa & SYS_PAR_EL1_PA; if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && - (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) { + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_DC)) { par |= FIELD_PREP(SYS_PAR_EL1_ATTR, MEMATTR(WbRaWa, WbRaWa)); par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_NSH); @@ -1548,11 +1548,11 @@ int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) bool fail; val = hcr = read_sysreg(hcr_el2); - val &= ~HCR_TGE; - val |= HCR_VM; + val &= ~HCR_EL2_TGE; + val |= HCR_EL2_VM; if (!vcpu_el2_e2h_is_set(vcpu)) - val |= HCR_NV | HCR_NV1; + val |= HCR_EL2_NV | HCR_EL2_NV1; write_sysreg_hcr(val); isb(); @@ -1635,7 +1635,7 @@ int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) * early. Same thing if {VM,DC}=={0,0}. */ if (compute_translation_regime(vcpu, op) == TR_EL20 || - !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM | HCR_DC))) + !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_VM | HCR_EL2_DC))) return 0; /* Do the stage-2 translation */ diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index e93919cadf86..adaf26ab93b8 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -59,6 +59,7 @@ struct reg_feat_map_desc { const struct reg_bits_to_feat_map feat_map; const struct reg_bits_to_feat_map *bit_feat_map; const unsigned int bit_feat_map_sz; + const unsigned int idreg_idx; }; #define __NEEDS_FEAT_3(m, f, w, id, fld, lim) \ @@ -130,6 +131,21 @@ struct reg_feat_map_desc { .bit_feat_map_sz = ARRAY_SIZE(m), \ } +/* + * Declare the dependency between the features advertised in an id + * register, and the set of other features they depend on. This + * generates a struct reg_feat_map_desc. + */ +#define DECLARE_IDREG_FEAT_MAP(n, r, m, f) \ + struct reg_feat_map_desc n = { \ + .name = #r, \ + .feat_map = NEEDS_FEAT(~(r##_RES0 | \ + r##_RES1), f), \ + .bit_feat_map = m, \ + .bit_feat_map_sz = ARRAY_SIZE(m), \ + .idreg_idx = IDREG_IDX(SYS_##r), \ + } + #define FEAT_SPE ID_AA64DFR0_EL1, PMSVer, IMP #define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP #define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP @@ -205,6 +221,7 @@ struct reg_feat_map_desc { #define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP #define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT #define FEAT_HDBSS ID_AA64MMFR1_EL1, HAFDBS, HDBSS +#define FEAT_HPDS ID_AA64MMFR1_EL1, HPDS, IMP #define FEAT_HPDS2 ID_AA64MMFR1_EL1, HPDS, HPDS2 #define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP #define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP @@ -280,6 +297,25 @@ static bool feat_anerr(struct kvm *kvm) kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SNERR, FEAT_ANERR)); } +/* + * S1PIE and S1POE imply TCR2 and ATS1A, and S1POE also implies HPDS, but + * userspace can set them independently. + */ +static bool feat_s1pie(struct kvm *kvm) +{ + return (kvm_has_feat(kvm, FEAT_S1PIE) && + kvm_has_feat(kvm, FEAT_TCR2) && + kvm_has_feat(kvm, FEAT_ATS1A)); +} + +static bool feat_s1poe(struct kvm *kvm) +{ + return (kvm_has_feat(kvm, FEAT_S1POE) && + kvm_has_feat(kvm, FEAT_TCR2) && + kvm_has_feat(kvm, FEAT_ATS1A) && + kvm_has_feat(kvm, FEAT_HPDS)); +} + static bool feat_sme_smps(struct kvm *kvm) { /* @@ -1299,6 +1335,14 @@ static const struct reg_bits_to_feat_map vtcr_el2_feat_map[] = { static const DECLARE_FEAT_MAP(vtcr_el2_desc, VTCR_EL2, vtcr_el2_feat_map, FEAT_AA64EL2); +static const struct reg_bits_to_feat_map id_aa64mmfr3_el1_feat_map[] = { + NEEDS_FEAT(ID_AA64MMFR3_EL1_S1PIE, feat_s1pie), + NEEDS_FEAT(ID_AA64MMFR3_EL1_S1POE, feat_s1poe), +}; + +static const DECLARE_IDREG_FEAT_MAP(id_aa64mmfr3_el1_desc, ID_AA64MMFR3_EL1, + id_aa64mmfr3_el1_feat_map, FEAT_AA64EL1); + static const struct reg_bits_to_feat_map ich_hfgrtr_feat_map[] = { NEEDS_FEAT(ICH_HFGRTR_EL2_ICC_APR_EL1 | ICH_HFGRTR_EL2_ICC_IDRn_EL1 | @@ -1408,6 +1452,40 @@ void __init check_feature_map(void) check_reg_desc(&ich_hfgitr_desc); } +static const struct reg_feat_map_desc *idreg_feat_maps[] = { + &id_aa64mmfr3_el1_desc, +}; + +static bool idreg_apply_deps(struct kvm *kvm, const struct reg_feat_map_desc *r) +{ + struct resx resx = {}; + u64 val; + + for (int i = 0; i < r->bit_feat_map_sz; i++) { + if ((r->bit_feat_map[i].flags & CALL_FUNC) && + !r->bit_feat_map[i].match(kvm)) + resx.res0 |= reg_feat_map_bits(&r->bit_feat_map[i]); + } + + val = kvm->arch.id_regs[r->idreg_idx]; + kvm->arch.id_regs[r->idreg_idx] &= ~resx.res0; + kvm->arch.id_regs[r->idreg_idx] |= resx.res1; + + return val != kvm->arch.id_regs[r->idreg_idx]; +} + +void kvm_idregs_enforce_dependencies(struct kvm *kvm) +{ + bool again; + + do { + again = false; + + for (int i = 0; i < ARRAY_SIZE(idreg_feat_maps); i++) + again |= idreg_apply_deps(kvm, idreg_feat_maps[i]); + } while (again); +} + static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map) { u64 regval = kvm->arch.id_regs[map->regidx]; diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index fc23c793ef17..2bec81668361 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -145,176 +145,176 @@ enum cgt_group_id { static const struct trap_bits coarse_trap_bits[] = { [CGT_HCR_TID1] = { .index = HCR_EL2, - .value = HCR_TID1, - .mask = HCR_TID1, + .value = HCR_EL2_TID1, + .mask = HCR_EL2_TID1, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_TID2] = { .index = HCR_EL2, - .value = HCR_TID2, - .mask = HCR_TID2, + .value = HCR_EL2_TID2, + .mask = HCR_EL2_TID2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID3] = { .index = HCR_EL2, - .value = HCR_TID3, - .mask = HCR_TID3, + .value = HCR_EL2_TID3, + .mask = HCR_EL2_TID3, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_IMO] = { .index = HCR_EL2, - .value = HCR_IMO, - .mask = HCR_IMO, + .value = HCR_EL2_IMO, + .mask = HCR_EL2_IMO, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_FMO] = { .index = HCR_EL2, - .value = HCR_FMO, - .mask = HCR_FMO, + .value = HCR_EL2_FMO, + .mask = HCR_EL2_FMO, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_TIDCP] = { .index = HCR_EL2, - .value = HCR_TIDCP, - .mask = HCR_TIDCP, + .value = HCR_EL2_TIDCP, + .mask = HCR_EL2_TIDCP, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TACR] = { .index = HCR_EL2, - .value = HCR_TACR, - .mask = HCR_TACR, + .value = HCR_EL2_TACR, + .mask = HCR_EL2_TACR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TSW] = { .index = HCR_EL2, - .value = HCR_TSW, - .mask = HCR_TSW, + .value = HCR_EL2_TSW, + .mask = HCR_EL2_TSW, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */ .index = HCR_EL2, - .value = HCR_TPC, - .mask = HCR_TPC, + .value = HCR_EL2_TPCP, + .mask = HCR_EL2_TPCP, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TPU] = { .index = HCR_EL2, - .value = HCR_TPU, - .mask = HCR_TPU, + .value = HCR_EL2_TPU, + .mask = HCR_EL2_TPU, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLB] = { .index = HCR_EL2, - .value = HCR_TTLB, - .mask = HCR_TTLB, + .value = HCR_EL2_TTLB, + .mask = HCR_EL2_TTLB, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TVM] = { .index = HCR_EL2, - .value = HCR_TVM, - .mask = HCR_TVM, + .value = HCR_EL2_TVM, + .mask = HCR_EL2_TVM, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_TDZ] = { .index = HCR_EL2, - .value = HCR_TDZ, - .mask = HCR_TDZ, + .value = HCR_EL2_TDZ, + .mask = HCR_EL2_TDZ, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TRVM] = { .index = HCR_EL2, - .value = HCR_TRVM, - .mask = HCR_TRVM, + .value = HCR_EL2_TRVM, + .mask = HCR_EL2_TRVM, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_TLOR] = { .index = HCR_EL2, - .value = HCR_TLOR, - .mask = HCR_TLOR, + .value = HCR_EL2_TLOR, + .mask = HCR_EL2_TLOR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TERR] = { .index = HCR_EL2, - .value = HCR_TERR, - .mask = HCR_TERR, + .value = HCR_EL2_TERR, + .mask = HCR_EL2_TERR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_APK] = { .index = HCR_EL2, .value = 0, - .mask = HCR_APK, + .mask = HCR_EL2_APK, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV] = { .index = HCR_EL2, - .value = HCR_NV, - .mask = HCR_NV, + .value = HCR_EL2_NV, + .mask = HCR_EL2_NV, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV_nNV2] = { .index = HCR_EL2, - .value = HCR_NV, - .mask = HCR_NV | HCR_NV2, + .value = HCR_EL2_NV, + .mask = HCR_EL2_NV | HCR_EL2_NV2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV1_nNV2] = { .index = HCR_EL2, - .value = HCR_NV | HCR_NV1, - .mask = HCR_NV | HCR_NV1 | HCR_NV2, + .value = HCR_EL2_NV | HCR_EL2_NV1, + .mask = HCR_EL2_NV | HCR_EL2_NV1 | HCR_EL2_NV2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_AT] = { .index = HCR_EL2, - .value = HCR_AT, - .mask = HCR_AT, + .value = HCR_EL2_AT, + .mask = HCR_EL2_AT, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_nFIEN] = { .index = HCR_EL2, .value = 0, - .mask = HCR_FIEN, + .mask = HCR_EL2_FIEN, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID4] = { .index = HCR_EL2, - .value = HCR_TID4, - .mask = HCR_TID4, + .value = HCR_EL2_TID4, + .mask = HCR_EL2_TID4, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TICAB] = { .index = HCR_EL2, - .value = HCR_TICAB, - .mask = HCR_TICAB, + .value = HCR_EL2_TICAB, + .mask = HCR_EL2_TICAB, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TOCU] = { .index = HCR_EL2, - .value = HCR_TOCU, - .mask = HCR_TOCU, + .value = HCR_EL2_TOCU, + .mask = HCR_EL2_TOCU, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_ENSCXT] = { .index = HCR_EL2, .value = 0, - .mask = HCR_ENSCXT, + .mask = HCR_EL2_EnSCXT, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLBIS] = { .index = HCR_EL2, - .value = HCR_TTLBIS, - .mask = HCR_TTLBIS, + .value = HCR_EL2_TTLBIS, + .mask = HCR_EL2_TTLBIS, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLBOS] = { .index = HCR_EL2, - .value = HCR_TTLBOS, - .mask = HCR_TTLBOS, + .value = HCR_EL2_TTLBOS, + .mask = HCR_EL2_TTLBOS, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID5] = { .index = HCR_EL2, - .value = HCR_TID5, - .mask = HCR_TID5, + .value = HCR_EL2_TID5, + .mask = HCR_EL2_TID5, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_MDCR_TPMCR] = { @@ -528,7 +528,7 @@ static bool is_nested_nv2_guest(struct kvm_vcpu *vcpu) u64 val; val = __vcpu_sys_reg(vcpu, HCR_EL2); - return ((val & (HCR_E2H | HCR_TGE | HCR_NV2 | HCR_NV1 | HCR_NV)) == (HCR_E2H | HCR_NV2 | HCR_NV)); + return ((val & (HCR_EL2_E2H | HCR_EL2_TGE | HCR_EL2_NV2 | HCR_EL2_NV1 | HCR_EL2_NV)) == (HCR_EL2_E2H | HCR_EL2_NV2 | HCR_EL2_NV)); } static enum trap_behaviour check_cnthctl_el1nvpct(struct kvm_vcpu *vcpu) @@ -2722,7 +2722,7 @@ static bool forward_hcr_traps(struct kvm_vcpu *vcpu, u64 control_bit) bool forward_smc_trap(struct kvm_vcpu *vcpu) { - return forward_hcr_traps(vcpu, HCR_TSC); + return forward_hcr_traps(vcpu, HCR_EL2_TSC); } static bool forward_mdcr_traps(struct kvm_vcpu *vcpu, u64 control_bit) @@ -2944,7 +2944,7 @@ int kvm_inject_nested_irq(struct kvm_vcpu *vcpu) */ if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) && - !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO)) + !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_IMO)) return 1; /* esr_el2 value doesn't matter for exits due to irqs. */ diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 1079136d5cd7..878ea4500c5c 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -755,7 +755,7 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) { events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); - events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) || + events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_EL2_VSE) || vcpu_get_flag(vcpu, NESTED_SERROR_PENDING); if (events->exception.serror_pending && events->exception.serror_has_esr) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index db37678dcb05..9931192735d4 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -43,7 +43,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu) /* Forward hvc instructions to the virtual EL2 if the guest has EL2. */ if (vcpu_has_nv(vcpu)) { - if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_HCD) + if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_HCD) kvm_inject_undefined(vcpu); else kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu)); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index a2a054fec2e2..e0c55644224c 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -403,11 +403,11 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr) { if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) - hcr |= HCR_TVM; + hcr |= HCR_EL2_TVM; write_sysreg_hcr(hcr); - if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) { + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_EL2_VSE)) { u64 vsesr; /* @@ -454,9 +454,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) * Additionally, when in a nested context we need to propagate the * updated state to the guest hypervisor's HCR_EL2. */ - if (*hcr & HCR_VSE) { - *hcr &= ~HCR_VSE; - *hcr |= read_sysreg(hcr_el2) & HCR_VSE; + if (*hcr & HCR_EL2_VSE) { + *hcr &= ~HCR_EL2_VSE; + *hcr |= read_sysreg(hcr_el2) & HCR_EL2_VSE; } } @@ -655,7 +655,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); /* Skip restoring fpexc32 for AArch64 guests */ - if (!(read_sysreg(hcr_el2) & HCR_RW)) + if (!(read_sysreg(hcr_el2) & HCR_EL2_RW)) write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; @@ -680,7 +680,7 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) * The normal sysreg handling code expects to see the traps, * let's not do anything here. */ - if (vcpu->arch.hcr_el2 & HCR_TVM) + if (vcpu->arch.hcr_el2 & HCR_EL2_TVM) return false; switch (sysreg) { diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 4373e9b63e87..0261b2bc681f 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -14,13 +14,14 @@ #include <nvhe/spinlock.h> /* - * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest - * are EL2's own and nothing the host sets there reaches the guest. + * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. A protected + * VM also takes TWI and TWE, at vCPU load. The rest are EL2's own and nothing + * the host sets there reaches the guest. */ -#define PKVM_HCR_EL2_HOST_PVM (HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE) -#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF | \ - HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 | \ - HCR_EL2_TID5 | HCR_EL2_TTLBOS) +#define PKVM_HCR_EL2_HOST_PVM HCR_EL2_VSE +#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_TWI | HCR_EL2_TWE | \ + HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_TVM | HCR_EL2_TID2 | \ + HCR_EL2_TID4 | HCR_EL2_TID5 | HCR_EL2_TTLBOS) /* * Holds the relevant data for maintaining the vcpu state completely at hyp. diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 9393fe3ea6a1..50581822ec74 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -123,7 +123,7 @@ SYM_FUNC_START(__hyp_do_panic) #ifdef CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC /* Ensure host stage-2 is disabled */ mrs x0, hcr_el2 - bic x0, x0, #HCR_VM + bic x0, x0, #HCR_EL2_VM msr_hcr_el2 x0 tlbi vmalls12e1 dsb nsh diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 0b3e0b28dfc7..49127e03cf93 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -102,7 +102,7 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) ldr x1, [x0, #NVHE_INIT_HCR_EL2] msr_hcr_el2 x1 - mov x2, #HCR_E2H + mov x2, #HCR_EL2_E2H and x2, x1, x2 cbz x2, 1f diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 137a8144d09e..179d7c8277b8 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -809,8 +809,8 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) } /* Cleared by hardware once the guest takes the vSError. */ - host_vcpu->arch.hcr_el2 &= ~HCR_VSE; - host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; + host_vcpu->arch.hcr_el2 &= ~HCR_EL2_VSE; + host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_EL2_VSE; sync_hyp_vgic_state(hyp_vcpu); sync_hyp_timer_state(hyp_vcpu); @@ -834,8 +834,8 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, u64 dfr0 = read_sysreg(id_aa64dfr0_el1); /* Propagate WFx trapping flags */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); - hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_EL2_TWE | HCR_EL2_TWI); + hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_EL2_TWE | HCR_EL2_TWI); /* HPMN == 0 is reserved without FEAT_HPMN0. */ if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) @@ -1615,7 +1615,8 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) /* * If pKVM has been initialised then reject any calls to the - * early "privileged" hypercalls. Note that we cannot reject + * early "privileged" hypercalls, and to the ones the host has no + * use for in protected mode. Note that we cannot reject * calls to __pkvm_prot_finalize for two reasons: (1) The static * key used to determine initialisation must be toggled prior to * finalisation and (2) finalisation is performed on a per-CPU @@ -1737,12 +1738,12 @@ static bool handle_host_mte(u64 esr) case SYS_TFSR_EL1: case SYS_TFSRE0_EL1: /* If we're here for any reason other than MTE, it's a bug. */ - if (read_sysreg(HCR_EL2) & HCR_ATA) + if (read_sysreg(HCR_EL2) & HCR_EL2_ATA) return false; break; case SYS_GMID_EL1: /* If we're here for any reason other than MTE, it's a bug. */ - if (!(read_sysreg(HCR_EL2) & HCR_TID5)) + if (!(read_sysreg(HCR_EL2) & HCR_EL2_TID5)) return false; break; default: diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index c3035aa5a470..d4c332a91803 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -344,14 +344,14 @@ int __pkvm_prot_finalize(void) struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu; struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); - if (params->hcr_el2 & HCR_VM) + if (params->hcr_el2 & HCR_EL2_VM) return -EPERM; params->vttbr = kvm_get_vttbr(mmu); params->vtcr = mmu->vtcr; - params->hcr_el2 |= HCR_VM; + params->hcr_el2 |= HCR_EL2_VM; if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - params->hcr_el2 |= HCR_FWB; + params->hcr_el2 |= HCR_EL2_FWB; /* * The CMO below not only cleans the updated params to the diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 5168321713d9..fc0475f0877d 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -38,23 +38,26 @@ unsigned int kvm_host_sve_max_vl; */ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); -/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */ +/* + * The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on + * each entry, and a protected vCPU's TWI and TWE at vCPU load. + */ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; if (has_hvhe()) - vcpu->arch.hcr_el2 |= HCR_E2H; + vcpu->arch.hcr_el2 |= HCR_EL2_E2H; if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ - vcpu->arch.hcr_el2 |= HCR_TEA; + vcpu->arch.hcr_el2 |= HCR_EL2_TEA; /* trap error record accesses */ - vcpu->arch.hcr_el2 |= HCR_TERR; + vcpu->arch.hcr_el2 |= HCR_EL2_TERR; } if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_FWB; + vcpu->arch.hcr_el2 |= HCR_EL2_FWB; /* * Without AArch32 EL1, leave RW set and let the entry fail with an @@ -66,7 +69,7 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; if (vcpu_has_ptrauth(vcpu)) - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK); } static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) @@ -75,14 +78,14 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) u64 val = vcpu->arch.hcr_el2; /* No support for AArch32. */ - val |= HCR_RW; + val |= HCR_EL2_RW; /* * Always trap: * - Feature id registers: to control features exposed to guests * - Implementation-defined features */ - val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1; + val |= HCR_EL2_TACR | HCR_EL2_TIDCP | HCR_EL2_TID3 | HCR_EL2_TID1; if (cpus_have_final_cap(ARM64_HAS_EVT) && !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && @@ -92,20 +95,20 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) val |= HCR_EL2_TID2; if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { - val |= HCR_TERR | HCR_TEA; - val &= ~(HCR_FIEN); + val |= HCR_EL2_TERR | HCR_EL2_TEA; + val &= ~(HCR_EL2_FIEN); } if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP)) - val &= ~(HCR_AMVOFFEN); + val &= ~(HCR_EL2_AMVOFFEN); if (!kvm_has_mte(kvm)) { - val |= HCR_TID5; - val &= ~(HCR_DCT | HCR_ATA); + val |= HCR_EL2_TID5; + val &= ~(HCR_EL2_DCT | HCR_EL2_ATA); } if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) - val |= HCR_TLOR; + val |= HCR_EL2_TLOR; vcpu->arch.hcr_el2 = val; } @@ -392,13 +395,19 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc { struct kvm *kvm = &hyp_vm->kvm; unsigned long host_arch_flags = READ_ONCE(host_kvm->arch.flags); + u32 vgic_model = READ_ONCE(host_kvm->arch.vgic.vgic_model); DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES); /* CTR_EL0 is always under host control, even for protected VMs. */ hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0; - /* Preserve the vgic model so that GICv3 emulation works */ - hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model; + /* + * pKVM runs only GICv3. Anything else becomes 0, the model of a VM + * with no in-kernel irqchip, which keeps EL2 off the GICv5 registers. + */ + if (vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) + vgic_model = 0; + hyp_vm->kvm.arch.vgic.vgic_model = vgic_model; /* No restrictions for non-protected VMs. */ if (!kvm_vm_is_protected(kvm)) { diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 91e1edf40bcd..89fb9dd351ad 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -110,7 +110,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) write_sysreg(__kvm_hyp_host_vector, vbar_el2); } -/* Save VGICv3 state on non-VHE systems */ +/* Save vgic state on non-VHE systems */ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) { if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) { @@ -125,7 +125,7 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) } } -/* Restore VGICv3 state on non-VHE systems */ +/* Restore vgic state on non-VHE systems */ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) { if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) { diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 74d4a509f1de..2dfb5ff9db57 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -464,7 +464,7 @@ bool __vgic_v3_get_gic_config(void) if (has_vhe()) { flags = local_daif_save(); } else { - sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO); + sysreg_clear_set_hcr(0, HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO); isb(); } @@ -479,7 +479,7 @@ bool __vgic_v3_get_gic_config(void) if (has_vhe()) { local_daif_restore(flags); } else { - sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0); + sysreg_clear_set_hcr(HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO, 0); isb(); } diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 4c965f9b1e1d..3f2390a13173 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -47,7 +47,7 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); * - FIEN: no way we let a guest have access to the RAS "Common Fault * Injection" thing, whatever that does */ -#define NV_HCR_GUEST_EXCLUDE (HCR_TGE | HCR_API | HCR_APK | HCR_FIEN) +#define NV_HCR_GUEST_EXCLUDE (HCR_EL2_TGE | HCR_EL2_API | HCR_EL2_APK | HCR_EL2_FIEN) static u64 __compute_hcr(struct kvm_vcpu *vcpu) { @@ -65,10 +65,10 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) if (is_hyp_ctxt(vcpu)) { host_data_set_flag(VCPU_IN_HYP_CONTEXT); - hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB; + hcr |= HCR_EL2_NV | HCR_EL2_NV2 | HCR_EL2_AT | HCR_EL2_TTLB; if (!vcpu_el2_e2h_is_set(vcpu)) - hcr |= HCR_NV1; + hcr |= HCR_EL2_NV1; /* Publish the guest's view of HCR_EL2 to the HW */ if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) @@ -91,7 +91,7 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) host_data_clear_flag(VCPU_IN_HYP_CONTEXT); guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - if (guest_hcr & HCR_NV) { + if (guest_hcr & HCR_EL2_NV) { u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id())); /* Inherit the low bits from the actual register */ @@ -99,14 +99,14 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) write_sysreg_s(va, SYS_VNCR_EL2); /* Force NV2 in case the guest is forgetful... */ - guest_hcr |= HCR_NV2; + guest_hcr |= HCR_EL2_NV2; } /* * Exclude the guest's TWED configuration if it hasn't set TWE * to avoid potentially delaying traps for the host. */ - if (!(guest_hcr & HCR_TWE)) + if (!(guest_hcr & HCR_EL2_TWE)) guest_hcr &= ~(HCR_EL2_TWEDEn | HCR_EL2_TWEDEL); } diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index c386d9f1c101..33ac9a8c954e 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -62,7 +62,7 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu, */ __load_stage2(mmu); val = read_sysreg(hcr_el2); - val &= ~HCR_TGE; + val &= ~HCR_EL2_TGE; write_sysreg_hcr(val); isb(); } diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index f4b932b7bf12..c00b0044ef11 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -198,8 +198,10 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user if (!kvm_smccc_filter_configured(kvm)) { r = kvm_smccc_filter_insert_reserved(kvm); - if (WARN_ON_ONCE(r)) + if (r) { + WARN_ON_ONCE(r != -ENOMEM); goto out_unlock; + } } r = mtree_insert_range(&kvm->arch.smccc_filter, start, end, diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 96bf926ca520..aa2011cc0d5a 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -228,7 +228,7 @@ static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu) { - if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA)) + if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_TGE | HCR_EL2_TEA)) return true; if (!vcpu_mode_priv(vcpu)) @@ -272,7 +272,7 @@ int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr) { u64 esr; - if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM)) + if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM)) return kvm_inject_nested_excl_atomic(vcpu, addr); __kvm_inject_sea(vcpu, false, addr); @@ -387,6 +387,6 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) } vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK); - *vcpu_hcr(vcpu) |= HCR_VSE; + *vcpu_hcr(vcpu) |= HCR_EL2_VSE; return 1; } diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 5b9ff769729e..1db10d8d6db7 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1781,6 +1781,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, s2vi.vma_pagesize, write_fault, exec_fault, false); kfree(mapping); + return ret; } if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) @@ -2952,18 +2953,18 @@ void kvm_set_way_flush(struct kvm_vcpu *vcpu) /* * If this is the first time we do a S/W operation - * (i.e. HCR_TVM not set) flush the whole memory, and set the + * (i.e. HCR_EL2_TVM not set) flush the whole memory, and set the * VM trapping. * * Otherwise, rely on the VM trapping to wait for the MMU + * Caches to be turned off. At that point, we'll be able to * clean the caches again. */ - if (!(hcr & HCR_TVM)) { + if (!(hcr & HCR_EL2_TVM)) { trace_kvm_set_way_flush(*vcpu_pc(vcpu), vcpu_has_cache_enabled(vcpu)); stage2_flush_vm(vcpu->kvm); - *vcpu_hcr(vcpu) = hcr | HCR_TVM; + *vcpu_hcr(vcpu) = hcr | HCR_EL2_TVM; } } @@ -2981,7 +2982,7 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) /* Caches are now on, stop trapping VM ops (until a S/W op) */ if (now_enabled) - *vcpu_hcr(vcpu) &= ~HCR_TVM; + *vcpu_hcr(vcpu) &= ~HCR_EL2_TVM; trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f1c414f1c3fa..4d83281a7a28 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -777,7 +777,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); - nested_stage2_enabled = hcr & HCR_VM; + nested_stage2_enabled = hcr & HCR_EL2_VM; /* Don't consider the CnP bit for the vttbr match */ vttbr &= ~VTTBR_CNP_BIT; @@ -858,7 +858,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) */ s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); - s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; + s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM; out: atomic_inc(&s2_mmu->refcnt); @@ -979,7 +979,7 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); } - if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) + if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); } } @@ -2181,6 +2181,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) * * What follows is some slop to (1) react to exception routing / masking and (2) * preserve the pending SError state across translation regimes. + * + * Also generate the maintenance interrupt state, and finally check that if + * we need to take an interrupt targetting the guest hypervisor. */ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) { @@ -2189,6 +2192,11 @@ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); + + vgic_v3_nested_update_mi(vcpu); + + if (vgic_state_is_nested(vcpu) && kvm_vgic_vcpu_pending_irq(vcpu)) + kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu); } void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) @@ -2202,7 +2210,7 @@ void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) * We previously decided that an SError was deliverable to the guest. * Reap the pending state from HCR_EL2 and... */ - if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr))) + if (unlikely(__test_and_clear_bit(__ffs(HCR_EL2_VSE), hcr))) vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); /* diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 380dfb1db0be..016997288fd2 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -333,7 +333,7 @@ static int __init pkvm_drop_host_privileges(void) return ret; } -void __init pkvm_selftests(void) +static void __init pkvm_selftests(void) { #ifdef CONFIG_NVHE_EL2_DEBUG int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest); @@ -387,6 +387,8 @@ static int __init finalize_pkvm(void) if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) return 0; + pkvm_selftests(); + /* * Exclude HYP sections from kmemleak so that they don't get peeked * at, which would end badly once inaccessible. diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index edfc3be1d4b3..850eba167aad 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -586,7 +586,7 @@ static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift) } /* - * Generic accessor for VM registers. Only called as long as HCR_TVM + * Generic accessor for VM registers. Only called as long as HCR_EL2_TVM * is set. If the guest enables the MMU, we stop trapping the VM * sys_regs and leave it in complete control of the caches. */ @@ -2893,7 +2893,7 @@ static u64 reset_hcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) u64 val = r->val; if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1)) - val |= HCR_E2H; + val |= HCR_EL2_E2H; __vcpu_assign_sys_reg(vcpu, r->reg, val); @@ -5798,31 +5798,31 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu) struct kvm *kvm = vcpu->kvm; if (has_vhe() || has_hvhe()) - vcpu->arch.hcr_el2 |= HCR_E2H; + vcpu->arch.hcr_el2 |= HCR_EL2_E2H; if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ - vcpu->arch.hcr_el2 |= HCR_TEA; + vcpu->arch.hcr_el2 |= HCR_EL2_TEA; /* trap error record accesses */ - vcpu->arch.hcr_el2 |= HCR_TERR; + vcpu->arch.hcr_el2 |= HCR_EL2_TERR; } if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_FWB; + vcpu->arch.hcr_el2 |= HCR_EL2_FWB; if (cpus_have_final_cap(ARM64_HAS_EVT) && !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_sanitised_ftr_reg(SYS_CTR_EL0)) - vcpu->arch.hcr_el2 |= HCR_TID4; + vcpu->arch.hcr_el2 |= HCR_EL2_TID4; else - vcpu->arch.hcr_el2 |= HCR_TID2; + vcpu->arch.hcr_el2 |= HCR_EL2_TID2; if (vcpu_el1_is_32bit(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_RW; + vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; if (kvm_has_mte(vcpu->kvm)) - vcpu->arch.hcr_el2 |= HCR_ATA; + vcpu->arch.hcr_el2 |= HCR_EL2_ATA; else - vcpu->arch.hcr_el2 |= HCR_TID5; + vcpu->arch.hcr_el2 |= HCR_EL2_TID5; /* * In the absence of FGT, we cannot independently trap TLBI @@ -5830,7 +5830,7 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu) * TLBIs would be far worse. Live with it... */ if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) - vcpu->arch.hcr_el2 |= HCR_TTLBOS; + vcpu->arch.hcr_el2 |= HCR_EL2_TTLBOS; } void kvm_calculate_traps(struct kvm_vcpu *vcpu) @@ -5874,6 +5874,8 @@ static int kvm_vm_finalize_sys_regs(struct kvm *kvm) if (kvm_id_regs_final(kvm)) return 0; + kvm_idregs_enforce_dependencies(kvm); + /* * This hacks into the ID registers, so only perform it when the * first vcpu runs, or the kvm_set_vm_id_reg() helper will scream. diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 979301c3dbd8..c9991e24cb6b 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -121,6 +121,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } + if (kvm->arch.vgic.its_model && kvm->arch.vgic.its_model != type) { + ret = -ENODEV; + goto out_unlock; + } + if (kvm_id_regs_final(kvm)) goto out_unlock; ret = 0; @@ -433,6 +438,8 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) return ret; switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + break; case KVM_DEV_TYPE_ARM_VGIC_V3: /* * If we are creating a VCPU with a GICv3 we must also register @@ -451,6 +458,9 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) */ if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus) return -EINVAL; + break; + default: + unreachable(); } return ret; @@ -550,6 +560,10 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; + /* A VM may be destroyed without ever creating a VGIC. */ + if (!dist->vgic_model) + goto out; + switch (dist->vgic_model) { case KVM_DEV_TYPE_ARM_VGIC_V2: dist->vgic_cpu_base = VGIC_ADDR_UNDEF; @@ -567,8 +581,11 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) kfree(dist->vgic_v5_irs_data); dist->vgic_v5_irs_data = NULL; break; + default: + unreachable(); } +out: xa_destroy(&dist->lpi_xa); } diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 1874c3c289a6..8d0ad74ebc64 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -1134,7 +1134,7 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) const struct vgic_register_region *region; struct vgic_reg_attr reg_attr; struct kvm_vcpu *vcpu; - gpa_t addr, offset; + gpa_t offset; int ret, align; ret = vgic_v5_parse_attr(dev, attr, ®_attr); @@ -1142,7 +1142,6 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) return ret; vcpu = reg_attr.vcpu; - addr = reg_attr.addr; if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS) return vgic_v5_has_cpu_sysregs_attr(vcpu, attr); diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index e5657cb60356..7e9b265b1f52 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -776,6 +776,7 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite) its_unmap_vlpi(ite->irq->host_irq); irq->hw = false; + irq->mapped = false; } vgic_put_irq(kvm, ite->irq); @@ -784,6 +785,23 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite) kfree(ite); } +/* Mapping one LPI from two events is UNPREDICTABLE: the first ITE keeps it. */ +static bool vgic_its_map_lpi(struct kvm *kvm, struct its_ite *ite, + struct vgic_irq *irq, struct kvm_vcpu *vcpu) +{ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (!irq->mapped) { + irq->mapped = true; + irq->target_vcpu = vcpu; + ite->irq = irq; + return true; + } + } + + vgic_put_irq(kvm, irq); + return false; +} + static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size) { return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1); @@ -815,12 +833,35 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, struct its_device *device = find_its_device(its, device_id); int ite_esz = vgic_its_get_abi(its)->ite_esz; gpa_t gpa = device->itt_addr + ite->event_id * ite_esz; + struct vgic_irq *irq = ite->irq; + struct kvm_vcpu *vcpu, *target; + + vgic_its_invalidate_cache(its); + /* - * Though the spec talks about removing the pending state, we - * don't bother here since we clear the ITTE anyway and the - * pending state is a property of the ITTE struct. + * Only an LPI on a vCPU's AP list can be in its list registers, + * which the exit path folds back, and that exit moves the LPI + * to its target vCPU if the target has changed: pause both. */ - vgic_its_invalidate_cache(its); + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + vcpu = irq->vcpu; + target = irq->target_vcpu; + } + + if (vcpu) { + if (target != vcpu) + kvm_pause_vcpu(target); + kvm_pause_vcpu(vcpu); + } + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->pending_latch = false; + + if (vcpu) { + kvm_resume_vcpu(vcpu); + if (target != vcpu) + kvm_resume_vcpu(target); + } its_free_ite(kvm, ite); @@ -1093,13 +1134,14 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, vcpu = collection_to_vcpu(kvm, collection); irq = vgic_add_lpi(kvm, lpi_nr, vcpu); + if (!IS_ERR(irq) && !vgic_its_map_lpi(kvm, ite, irq, vcpu)) + irq = ERR_PTR(-EEXIST); if (IS_ERR(irq)) { if (new_coll) vgic_its_free_collection(its, coll_id); its_free_ite(kvm, ite); return PTR_ERR(irq); } - ite->irq = irq; return 0; } @@ -1382,6 +1424,15 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, if (vcpu1 == vcpu2) return 0; + /* + * Bulk operations such as MOVALL are a pain, as they can clash + * badly locking-wise with other vcpus entering and exiting the + * guest, should they be affected by it. Stopping the guest is a + * safer bet to ensure uncontended access and ultimately forward + * progress. Yeah... + */ + kvm_arm_halt_guest(kvm); + xa_for_each(&dist->lpi_xa, intid, irq) { irq = vgic_get_irq(kvm, intid); if (!irq) @@ -1394,6 +1445,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, vgic_its_invalidate_cache(its); + kvm_arm_resume_guest(kvm); + return 0; } @@ -1875,6 +1928,19 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) mutex_lock(&dev->kvm->arch.config_lock); + /* + * Reject creation of a GICv3 ITS if: + * - The VM already has a VGIC other than GICv3. + * - An existing ITS requires a VGIC model other than GICv3. + */ + if ((irqchip_in_kernel(dev->kvm) && !vgic_is_v3(dev->kvm)) || + (dev->kvm->arch.vgic.its_model && + dev->kvm->arch.vgic.its_model != KVM_DEV_TYPE_ARM_VGIC_V3)) { + mutex_unlock(&dev->kvm->arch.config_lock); + kfree(its); + return -ENODEV; + } + if (vgic_initialized(dev->kvm)) { ret = vgic_v4_init(dev->kvm); if (ret < 0) { @@ -1902,7 +1968,7 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) xa_init(&its->translation_cache); dev->kvm->arch.vgic.msis_require_devid = true; - dev->kvm->arch.vgic.has_its = true; + dev->kvm->arch.vgic.its_model = KVM_DEV_TYPE_ARM_VGIC_V3; its->enabled = false; its->dev = dev; @@ -2208,7 +2274,10 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id, its_free_ite(kvm, ite); return PTR_ERR(irq); } - ite->irq = irq; + + /* Drop an ITE whose LPI another ITE maps, as MAPTI does. */ + if (!vgic_its_map_lpi(kvm, ite, irq, vcpu)) + its_free_ite(kvm, ite); return offset; } diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5ab789accc9d..f9ff553bbbf1 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -42,10 +42,7 @@ bool vgic_has_its(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) - return false; - - return dist->has_its; + return dist->its_model && dist->its_model == dist->vgic_model; } bool vgic_supports_direct_msis(struct kvm *kvm) @@ -303,9 +300,19 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, if (ctlr != GICR_CTLR_ENABLE_LPIS) return; + /* + * Yes, disabling LPIs is painful, since it can be done from + * a *remote* vcpu! So let's not take any chance, and make + * sure that the target vCPU has written their LRs back to the + * irq structures, and release any reference they would have. + * + * If it hurts, don't do it. + */ + kvm_pause_vcpu(vcpu); vgic_flush_pending_lpis(vcpu); vgic_its_invalidate_all_caches(vcpu->kvm); atomic_set_release(&vgic_cpu->ctlr, 0); + kvm_resume_vcpu(vcpu); } else { ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0, GICR_CTLR_ENABLE_LPIS); diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index e3e84d7f5ad5..82bfc4d4f122 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -120,8 +120,8 @@ bool vgic_state_is_nested(struct kvm_vcpu *vcpu) u64 xmo; if (is_nested_ctxt(vcpu)) { - xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO); - WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO), + xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_IMO | HCR_EL2_FMO); + WARN_ONCE(xmo && xmo != (HCR_EL2_IMO | HCR_EL2_FMO), "Separate virtual IRQ/FIQ settings not supported\n"); return !!xmo; diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index c2726d4c06e6..bea848c93a99 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -1071,11 +1071,10 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) static inline bool can_access_vgic_from_kernel(void) { /* - * GICv2 can always be accessed from the kernel because it is - * memory-mapped, and VHE systems can access GICv3 EL2 system - * registers. + * GICv2 is memory-mapped. Every other CPU interface uses EL2 system + * registers, which only VHE can access from the kernel. */ - return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe(); + return has_vhe() || kvm_vgic_global_state.type == VGIC_V2; } static inline void vgic_save_state(struct kvm_vcpu *vcpu) @@ -1167,7 +1166,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) * * We do that by placing a request to ourselves which will * abort the entry procedure and inject the exception at the - * beginning of the run loop. + * beginning of the run loop (see kvm_nested_flush_hwstate()). * * - Otherwise, do exactly *NOTHING* apart from enabling the virtual * CPU interface. The guest state is already loaded, and we can @@ -1177,16 +1176,10 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) * maintenance interrupt state, as it may fire. */ if (vgic_state_is_nested(vcpu)) { - if (kvm_vgic_vcpu_pending_irq(vcpu)) - kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu); - vgic_v3_flush_nested(vcpu); return; } - if (vcpu_has_nv(vcpu)) - vgic_v3_nested_update_mi(vcpu); - DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); vgic_flush_state(vcpu); diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 6371d5ea5392..50d4789b9cac 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -11,9 +11,9 @@ #define __KVM_VCPU_RISCV_VECTOR_H #include <linux/types.h> +#include <asm/vector.h> #ifdef CONFIG_RISCV_ISA_V -#include <asm/vector.h> #include <asm/kvm_host.h> static __always_inline void __kvm_riscv_vector_save(struct kvm_cpu_context *context) @@ -55,6 +55,10 @@ static inline void kvm_riscv_v_exit(void) struct kvm_cpu_context; +static __always_inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *context) +{ +} + static inline void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) { } diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 20d9959ca44f..39b0f3f1d486 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -206,6 +206,10 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZALASR, KVM_RISCV_ISA_EXT_ZICFILP, KVM_RISCV_ISA_EXT_ZICFISS, + KVM_RISCV_ISA_EXT_ZICCLSM, + KVM_RISCV_ISA_EXT_ZICCAMOA, + KVM_RISCV_ISA_EXT_ZICCIF, + KVM_RISCV_ISA_EXT_ZA64RS, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 2e3c4e6c1cd7..f7e4d0f9b156 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -26,6 +26,7 @@ config KVM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING + select KVM_GENERIC_PRE_FAULT_MEMORY select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index f17dc559a709..302938972aa0 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -48,6 +48,8 @@ struct imsic { /* IMSIC VS-file */ rwlock_t vsfile_lock; + /* Writers hold both locks; scheduler-out only takes vsfile_hgei_lock. */ + raw_spinlock_t vsfile_hgei_lock; int vsfile_cpu; int vsfile_hgei; void __iomem *vsfile_va; @@ -67,36 +69,17 @@ struct imsic { __r; \ }) -#define imsic_read_switchcase(__ireg) \ - case __ireg: \ - return imsic_vs_csr_read(__ireg); -#define imsic_read_switchcase_2(__ireg) \ - imsic_read_switchcase(__ireg + 0) \ - imsic_read_switchcase(__ireg + 1) -#define imsic_read_switchcase_4(__ireg) \ - imsic_read_switchcase_2(__ireg + 0) \ - imsic_read_switchcase_2(__ireg + 2) -#define imsic_read_switchcase_8(__ireg) \ - imsic_read_switchcase_4(__ireg + 0) \ - imsic_read_switchcase_4(__ireg + 4) -#define imsic_read_switchcase_16(__ireg) \ - imsic_read_switchcase_8(__ireg + 0) \ - imsic_read_switchcase_8(__ireg + 8) -#define imsic_read_switchcase_32(__ireg) \ - imsic_read_switchcase_16(__ireg + 0) \ - imsic_read_switchcase_16(__ireg + 16) -#define imsic_read_switchcase_64(__ireg) \ - imsic_read_switchcase_32(__ireg + 0) \ - imsic_read_switchcase_32(__ireg + 32) +static inline bool imsic_eix_ireg_is_valid(int ireg) +{ + return (ireg >= IMSIC_EIP0 && ireg <= IMSIC_EIP63) || + (ireg >= IMSIC_EIE0 && ireg <= IMSIC_EIE63); +} static unsigned long imsic_eix_read(int ireg) { - switch (ireg) { - imsic_read_switchcase_64(IMSIC_EIP0) - imsic_read_switchcase_64(IMSIC_EIE0) - } - - return 0; + if (!imsic_eix_ireg_is_valid(ireg)) + return 0; + return imsic_vs_csr_read(ireg); } #define imsic_vs_csr_swap(__c, __v) \ @@ -107,36 +90,11 @@ static unsigned long imsic_eix_read(int ireg) __r; \ }) -#define imsic_swap_switchcase(__ireg, __v) \ - case __ireg: \ - return imsic_vs_csr_swap(__ireg, __v); -#define imsic_swap_switchcase_2(__ireg, __v) \ - imsic_swap_switchcase(__ireg + 0, __v) \ - imsic_swap_switchcase(__ireg + 1, __v) -#define imsic_swap_switchcase_4(__ireg, __v) \ - imsic_swap_switchcase_2(__ireg + 0, __v) \ - imsic_swap_switchcase_2(__ireg + 2, __v) -#define imsic_swap_switchcase_8(__ireg, __v) \ - imsic_swap_switchcase_4(__ireg + 0, __v) \ - imsic_swap_switchcase_4(__ireg + 4, __v) -#define imsic_swap_switchcase_16(__ireg, __v) \ - imsic_swap_switchcase_8(__ireg + 0, __v) \ - imsic_swap_switchcase_8(__ireg + 8, __v) -#define imsic_swap_switchcase_32(__ireg, __v) \ - imsic_swap_switchcase_16(__ireg + 0, __v) \ - imsic_swap_switchcase_16(__ireg + 16, __v) -#define imsic_swap_switchcase_64(__ireg, __v) \ - imsic_swap_switchcase_32(__ireg + 0, __v) \ - imsic_swap_switchcase_32(__ireg + 32, __v) - static unsigned long imsic_eix_swap(int ireg, unsigned long val) { - switch (ireg) { - imsic_swap_switchcase_64(IMSIC_EIP0, val) - imsic_swap_switchcase_64(IMSIC_EIE0, val) - } - - return 0; + if (!imsic_eix_ireg_is_valid(ireg)) + return 0; + return imsic_vs_csr_swap(ireg, val); } #define imsic_vs_csr_write(__c, __v) \ @@ -145,35 +103,10 @@ do { \ csr_write(CSR_VSIREG, __v); \ } while (0) -#define imsic_write_switchcase(__ireg, __v) \ - case __ireg: \ - imsic_vs_csr_write(__ireg, __v); \ - break; -#define imsic_write_switchcase_2(__ireg, __v) \ - imsic_write_switchcase(__ireg + 0, __v) \ - imsic_write_switchcase(__ireg + 1, __v) -#define imsic_write_switchcase_4(__ireg, __v) \ - imsic_write_switchcase_2(__ireg + 0, __v) \ - imsic_write_switchcase_2(__ireg + 2, __v) -#define imsic_write_switchcase_8(__ireg, __v) \ - imsic_write_switchcase_4(__ireg + 0, __v) \ - imsic_write_switchcase_4(__ireg + 4, __v) -#define imsic_write_switchcase_16(__ireg, __v) \ - imsic_write_switchcase_8(__ireg + 0, __v) \ - imsic_write_switchcase_8(__ireg + 8, __v) -#define imsic_write_switchcase_32(__ireg, __v) \ - imsic_write_switchcase_16(__ireg + 0, __v) \ - imsic_write_switchcase_16(__ireg + 16, __v) -#define imsic_write_switchcase_64(__ireg, __v) \ - imsic_write_switchcase_32(__ireg + 0, __v) \ - imsic_write_switchcase_32(__ireg + 32, __v) - static void imsic_eix_write(int ireg, unsigned long val) { - switch (ireg) { - imsic_write_switchcase_64(IMSIC_EIP0, val) - imsic_write_switchcase_64(IMSIC_EIE0, val) - } + if (imsic_eix_ireg_is_valid(ireg)) + imsic_vs_csr_write(ireg, val); } #define imsic_vs_csr_set(__c, __v) \ @@ -182,35 +115,10 @@ do { \ csr_set(CSR_VSIREG, __v); \ } while (0) -#define imsic_set_switchcase(__ireg, __v) \ - case __ireg: \ - imsic_vs_csr_set(__ireg, __v); \ - break; -#define imsic_set_switchcase_2(__ireg, __v) \ - imsic_set_switchcase(__ireg + 0, __v) \ - imsic_set_switchcase(__ireg + 1, __v) -#define imsic_set_switchcase_4(__ireg, __v) \ - imsic_set_switchcase_2(__ireg + 0, __v) \ - imsic_set_switchcase_2(__ireg + 2, __v) -#define imsic_set_switchcase_8(__ireg, __v) \ - imsic_set_switchcase_4(__ireg + 0, __v) \ - imsic_set_switchcase_4(__ireg + 4, __v) -#define imsic_set_switchcase_16(__ireg, __v) \ - imsic_set_switchcase_8(__ireg + 0, __v) \ - imsic_set_switchcase_8(__ireg + 8, __v) -#define imsic_set_switchcase_32(__ireg, __v) \ - imsic_set_switchcase_16(__ireg + 0, __v) \ - imsic_set_switchcase_16(__ireg + 16, __v) -#define imsic_set_switchcase_64(__ireg, __v) \ - imsic_set_switchcase_32(__ireg + 0, __v) \ - imsic_set_switchcase_32(__ireg + 32, __v) - static void imsic_eix_set(int ireg, unsigned long val) { - switch (ireg) { - imsic_set_switchcase_64(IMSIC_EIP0, val) - imsic_set_switchcase_64(IMSIC_EIE0, val) - } + if (imsic_eix_ireg_is_valid(ireg)) + imsic_vs_csr_set(ireg, val); } static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, @@ -582,6 +490,17 @@ static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix, csr_write(CSR_VSISELECT, old_vsiselect); } +static void imsic_vsfile_set_hgei(struct imsic *imsic, int cpu, int hgei) +{ + unsigned long flags; + + /* Serialize the CPU/HGEI pair with scheduler-out's wakeup setup. */ + raw_spin_lock_irqsave(&imsic->vsfile_hgei_lock, flags); + imsic->vsfile_cpu = cpu; + imsic->vsfile_hgei = hgei; + raw_spin_unlock_irqrestore(&imsic->vsfile_hgei_lock, flags); +} + static void imsic_vsfile_cleanup(struct imsic *imsic) { int old_vsfile_hgei, old_vsfile_cpu; @@ -596,7 +515,7 @@ static void imsic_vsfile_cleanup(struct imsic *imsic) write_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; old_vsfile_cpu = imsic->vsfile_cpu; - imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic_vsfile_set_hgei(imsic, -1, -1); imsic->vsfile_va = NULL; imsic->vsfile_pa = 0; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -727,10 +646,10 @@ void kvm_riscv_vcpu_aia_imsic_put(struct kvm_vcpu *vcpu) if (!kvm_vcpu_is_blocking(vcpu)) return; - read_lock_irqsave(&imsic->vsfile_lock, flags); + raw_spin_lock_irqsave(&imsic->vsfile_hgei_lock, flags); if (imsic->vsfile_cpu > -1) csr_set(CSR_HGEIE, BIT(imsic->vsfile_hgei)); - read_unlock_irqrestore(&imsic->vsfile_lock, flags); + raw_spin_unlock_irqrestore(&imsic->vsfile_hgei_lock, flags); } void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) @@ -747,7 +666,7 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) write_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; old_vsfile_cpu = imsic->vsfile_cpu; - imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic_vsfile_set_hgei(imsic, -1, -1); imsic->vsfile_va = NULL; imsic->vsfile_pa = 0; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -861,8 +780,7 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) /* Update new IMSIC VS-file details in IMSIC context */ write_lock_irqsave(&imsic->vsfile_lock, flags); - imsic->vsfile_hgei = new_vsfile_hgei; - imsic->vsfile_cpu = vcpu->cpu; + imsic_vsfile_set_hgei(imsic, vcpu->cpu, new_vsfile_hgei); imsic->vsfile_va = new_vsfile_va; imsic->vsfile_pa = new_vsfile_pa; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -1119,6 +1037,7 @@ int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) /* Setup IMSIC context */ imsic->nr_msis = kvm->arch.aia.nr_ids + 1; rwlock_init(&imsic->vsfile_lock); + raw_spin_lock_init(&imsic->vsfile_hgei_lock); imsic->nr_eix = BITS_TO_U64(imsic->nr_msis); imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids); imsic->vsfile_hgei = imsic->vsfile_cpu = -1; diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index e5002cb9cbef..be70607a33ea 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -216,12 +216,10 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, pte_t *ptep; int ret; - out_map->addr = gpa; - out_map->level = 0; - ret = gstage_page_size_to_level(gstage, page_size, &out_map->level); if (ret) return ret; + out_map->addr = ALIGN_DOWN(gpa, page_size); /* * A RISC-V implementation can choose to either: @@ -280,6 +278,8 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, out_map->level, true); } else if (ALIGN_DOWN(PFN_PHYS(pte_pfn(ptep_get(ptep))), page_size) == hpa) { kvm_riscv_gstage_update_pte_prot(gstage, ptep_level, gpa, ptep, prot); + out_map->level = ptep_level; + out_map->pte = ptep_get(ptep); return 0; } } diff --git a/arch/riscv/kvm/isa.c b/arch/riscv/kvm/isa.c index 971f44655100..72e3dc15d294 100644 --- a/arch/riscv/kvm/isa.c +++ b/arch/riscv/kvm/isa.c @@ -38,6 +38,7 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZA64RS), KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), @@ -64,6 +65,9 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(ZICBOM), KVM_ISA_EXT_ARR(ZICBOP), KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCAMOA), + KVM_ISA_EXT_ARR(ZICCIF), + KVM_ISA_EXT_ARR(ZICCLSM), KVM_ISA_EXT_ARR(ZICCRSE), KVM_ISA_EXT_ARR(ZICFILP), KVM_ISA_EXT_ARR(ZICFISS), @@ -183,6 +187,7 @@ bool kvm_riscv_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SVINVAL: case KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_RISCV_ISA_EXT_ZA64RS: case KVM_RISCV_ISA_EXT_ZAAMO: case KVM_RISCV_ISA_EXT_ZABHA: case KVM_RISCV_ISA_EXT_ZACAS: @@ -206,6 +211,9 @@ bool kvm_riscv_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_ZFH: case KVM_RISCV_ISA_EXT_ZFHMIN: case KVM_RISCV_ISA_EXT_ZICBOP: + case KVM_RISCV_ISA_EXT_ZICCAMOA: + case KVM_RISCV_ISA_EXT_ZICCIF: + case KVM_RISCV_ISA_EXT_ZICCLSM: case KVM_RISCV_ISA_EXT_ZICCRSE: case KVM_RISCV_ISA_EXT_ZICNTR: case KVM_RISCV_ISA_EXT_ZICOND: diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 371fccaf9df0..738d9ee1eac9 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -754,6 +754,53 @@ out_unlock: return ret; } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + return 0; +} + +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, + struct kvm_pre_fault_memory *range) +{ + struct kvm_gstage_mapping out_map = { 0 }; + struct kvm_memory_slot *memslot; + unsigned long map_size; + unsigned long hva; + gpa_t end; + gfn_t gfn; + int ret; + + gfn = gpa_to_gfn(range->gpa); + memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); + if (!memslot) + return -ENOENT; + + hva = gfn_to_hva_memslot_prot(memslot, gfn, NULL); + if (kvm_is_error_hva(hva)) + return -ENOENT; + + for (;;) { + if (signal_pending(current)) + return -EINTR; + + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) + return -EIO; + + cond_resched(); + ret = kvm_riscv_mmu_map(vcpu, memslot, range->gpa, hva, false, &out_map); + if (ret) + return ret; + + if (pte_val(out_map.pte)) + break; + } + + map_size = PAGE_SIZE << (out_map.level * kvm_riscv_gstage_index_bits); + end = ALIGN_DOWN(range->gpa, map_size) + map_size; + + return min_t(u64, range->size, end - range->gpa); +} + int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) { struct page *pgd_page; diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 7f1636c44d41..dd786f758eb7 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -138,6 +138,9 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) { + if (kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + return 0; } diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index adbbb59a0f36..2a8dc2b88ba8 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -647,7 +647,7 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) vcpu->arch.mmio_decode.return_handled = 1; insn = vcpu->arch.mmio_decode.insn; - if (run->mmio.is_write) + if (run->mmio.is_write || !((insn >> SH_RD) & MASK_RX)) goto done; len = vcpu->arch.mmio_decode.len; diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 506a510b6bff..c2f1f7f26442 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -64,8 +64,17 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (tmp->vcpu_id < hbase) continue; hart_bit = tmp->vcpu_id - hbase; + /* + * kvm_for_each_vcpu() walks kvm->vcpus[] by + * vcpu_idx, i.e. the creation order, which has + * nothing to do with the vcpu_id (hart id) space + * that the SBI IPI operates on. vcpu_ids need not + * increase along the iteration, so harts outside + * the hart_mask window must be skipped instead of + * aborting the loop. + */ if (hart_bit >= __riscv_xlen) - goto done; + continue; if (!(hmask & (1UL << hart_bit))) continue; } @@ -76,7 +85,6 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, kvm_riscv_vcpu_pmu_incr_fw(tmp, SBI_PMU_FW_IPI_RCVD); } -done: if (hbase != -1UL && (hmask ^ sentmask)) retdata->err_val = SBI_ERR_INVALID_PARAM; diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index a2cd277a4059..adbd1617b78a 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -135,11 +135,7 @@ bool kvm_riscv_vcpu_timer_pending(struct kvm_vcpu *vcpu) struct kvm_vcpu_timer *t = &vcpu->arch.timer; struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; - if (!kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t) || - kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER)) - return true; - else - return false; + return !kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t); } static void kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index a9f083feeb76..58500a19b33b 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -187,6 +187,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_MP_STATE: case KVM_CAP_IMMEDIATE_EXIT: case KVM_CAP_SET_GUEST_DEBUG: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_NR_VCPUS: diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 7be81b936bf1..4b2c41917b77 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -292,6 +292,7 @@ struct vgic_irq { bool active:1; bool hw:1; /* Tied to HW IRQ */ bool on_lr:1; /* Present in a CPU LR */ + bool mapped:1; /* LPIs: mapped by an ITE */ refcount_t refcount; /* Used for LPIs */ u32 hwintid; /* HW INTID number */ unsigned int host_irq; /* linux irq corresponding to hwintid */ @@ -502,7 +503,9 @@ struct vgic_dist { bool initialized; /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */ - u32 vgic_model; + enum kvm_device_type vgic_model; + /* VGIC model required by the VM's ITS devices. */ + enum kvm_device_type its_model; /* Implementation revision as reported in the GICD_IIDR */ u32 implementation_rev; @@ -547,7 +550,6 @@ struct vgic_dist { struct vgic_io_device dist_iodev; struct vgic_io_device cpuif_iodev; - bool has_its; bool table_write_in_progress; /* diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index c41397eccee5..35d043b19332 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -1337,6 +1337,7 @@ source "kernel/trace/rv/Kconfig" config TRACE_REMOTE bool + select TRACING config SIMPLE_RING_BUFFER bool diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 3cb35a6963d5..17a4256e9081 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -242,6 +242,7 @@ TEST_GEN_PROGS_riscv += coalesced_io_test TEST_GEN_PROGS_riscv += dirty_log_perf_test TEST_GEN_PROGS_riscv += get-reg-list TEST_GEN_PROGS_riscv += mmu_stress_test +TEST_GEN_PROGS_riscv += pre_fault_memory_test TEST_GEN_PROGS_riscv += rseq_test TEST_GEN_PROGS_riscv += steal_time diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 10849d21c0dd..1d2cc25cb3f4 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c @@ -815,6 +815,10 @@ static void test_idreg_frozen_after_failed_run(void) SYS_REVIDR_EL1, SYS_AIDR_EL1, }; + u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE]; + struct reg_mask_range range = { + .addr = (__u64)masks, + }; struct kvm_vcpu_init init; struct kvm_vcpu *vcpu; struct kvm_vm *vm; @@ -831,6 +835,7 @@ static void test_idreg_frozen_after_failed_run(void) vm = vm_create(1); vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + vm_ioctl(vm, KVM_ARM_GET_REG_WRITABLE_MASKS, &range); kvm_get_default_vcpu_target(vm, &init); init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); @@ -852,7 +857,9 @@ static void test_idreg_frozen_after_failed_run(void) */ for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; - u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u32 reg_id = test_regs[i].reg; + u64 reg = KVM_ARM64_SYS_REG(reg_id); + u64 mask = masks[encoding_to_range_idx(reg_id)]; u64 val = vcpu_get_reg(vcpu, reg); for (int j = 0; ftr_bits[j].type != FTR_END; j++) { @@ -860,6 +867,10 @@ static void test_idreg_frozen_after_failed_run(void) u64 safe = get_safe_value(&ftr_bits[j], ftr); u64 new_val; + /* Only a field userspace could change can be frozen. */ + if ((mask & ftr_bits[j].mask) != ftr_bits[j].mask) + continue; + if (safe == ftr) continue; diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index d5df742b2300..5bf7d99d2914 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -2003,8 +2003,40 @@ int test_kvm_device(u32 gic_dev_type) return 0; } +static void test_vgic_v3_its_with_vgic_v5(void) +{ + struct kvm_vm *vm = vm_create_barebones(); + int gic_fd, its_fd, ret; + + if (__kvm_test_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS)) { + pr_info("No GICv3 ITS support; skipping mixed-model test.\n"); + kvm_vm_free(vm); + return; + } + + gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5); + ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS); + TEST_ASSERT(ret < 0 && errno == ENODEV, + "GICv3 ITS created after GICv5 VGIC: ret=%d errno=%d", + ret, errno); + close(gic_fd); + kvm_vm_free(vm); + + vm = vm_create_barebones(); + its_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS); + ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5); + TEST_ASSERT(ret < 0 && errno == ENODEV, + "GICv5 VGIC created after GICv3 ITS: ret=%d errno=%d", + ret, errno); + close(its_fd); + kvm_vm_free(vm); +} + void run_tests(u32 gic_dev_type) { + pr_info("Test VGICv5 and GICv3 ITS model mismatch\n"); + test_vgic_v3_its_with_vgic_v5(); + pr_info("Test VGICv5 address attrs\n"); test_vgic_v5_addr_attrs(); diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index e3acf2ae9881..a5f101039a21 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -12,6 +12,9 @@ #include <asm/vdso/processor.h> #include "kvm_util.h" +#define PAGE_SHIFT 12 +#define PAGE_SIZE BIT_ULL(PAGE_SHIFT) + #define INSN_OPCODE_MASK 0x007c #define INSN_OPCODE_SHIFT 2 #define INSN_OPCODE_SYSTEM 28 @@ -25,6 +28,19 @@ #define GET_RM(insn) (((insn) & INSN_MASK_FUNCT3) >> INSN_SHIFT_FUNCT3) #define GET_CSR_NUM(insn) (((insn) & INSN_CSR_MASK) >> INSN_CSR_SHIFT) +/* Vector (V) instruction decoding, matching arch/riscv/include/asm/insn.h */ +#define RV_INSN_OPCODE_MASK 0x7f +#define RVG_OPCODE_SYSTEM 0x73 +#define RVV_OPCODE_VECTOR 0x57 +#define RVV_OPCODE_VL 0x07 +#define RVV_OPCODE_VS 0x27 +#define RVV_VL_VS_WIDTH_8 0 +#define RVV_VL_VS_WIDTH_16 5 +#define RVV_VL_VS_WIDTH_32 6 +#define RVV_VL_VS_WIDTH_64 7 +#define RVV_EXTRACT_VL_VS_WIDTH(insn) (((insn) >> 12) & 0x7) +#define RVG_EXTRACT_SYSTEM_CSR(insn) (((insn) >> 20) & 0xfff) + static inline u64 __kvm_reg_id(u64 type, u64 subtype, u64 idx, u64 size) { return KVM_REG_RISCV | type | subtype | idx | size; @@ -127,6 +143,9 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector, exception_handl void vm_install_interrupt_handler(struct kvm_vm *vm, exception_handler_fn handler); +/* L4 index Bit[56:48] */ +#define PGTBL_L4_INDEX_MASK 0x01FF000000000000ULL +#define PGTBL_L4_INDEX_SHIFT 48 /* L3 index Bit[47:39] */ #define PGTBL_L3_INDEX_MASK 0x0000FF8000000000ULL #define PGTBL_L3_INDEX_SHIFT 39 diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index c175dcd14114..5478ac038029 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -17,6 +17,11 @@ static gva_t exception_handlers; +struct handlers { + exception_handler_fn exception_handlers[NR_VECTORS][NR_EXCEPTIONS]; + bool v_available; +}; + bool __vcpu_has_ext(struct kvm_vcpu *vcpu, u64 ext) { unsigned long value = 0; @@ -43,6 +48,7 @@ static u64 pte_index_mask[] = { PGTBL_L1_INDEX_MASK, PGTBL_L2_INDEX_MASK, PGTBL_L3_INDEX_MASK, + PGTBL_L4_INDEX_MASK, }; static u32 pte_index_shift[] = { @@ -50,6 +56,7 @@ static u32 pte_index_shift[] = { PGTBL_L1_INDEX_SHIFT, PGTBL_L2_INDEX_SHIFT, PGTBL_L3_INDEX_SHIFT, + PGTBL_L4_INDEX_SHIFT, }; static u64 pte_index(struct kvm_vm *vm, gva_t gva, int level) @@ -233,6 +240,7 @@ void riscv_vcpu_mmu_setup(struct kvm_vcpu *vcpu) void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) { struct kvm_riscv_core core; + struct kvm_riscv_csr general_csr; core.mode = vcpu_get_reg(vcpu, RISCV_CORE_REG(mode)); core.regs.pc = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.pc)); @@ -268,6 +276,10 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) core.regs.t5 = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t5)); core.regs.t6 = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t6)); + general_csr.sepc = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(sepc)); + general_csr.scause = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(scause)); + general_csr.stval = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(stval)); + fprintf(stream, " MODE: 0x%lx\n", core.mode); fprintf(stream, @@ -294,13 +306,9 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) fprintf(stream, " T3: 0x%016lx T4: 0x%016lx T5: 0x%016lx T6: 0x%016lx\n", core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6); -} - -static void __aligned(16) guest_unexp_trap(void) -{ - sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, - KVM_RISCV_SELFTESTS_SBI_UNEXP, - 0, 0, 0, 0, 0, 0); + fprintf(stream, + "SEPC: 0x%016lx SCAUSE: 0x%016lx STVAL: 0x%016lx\n", + general_csr.sepc, general_csr.scause, general_csr.stval); } void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) @@ -346,8 +354,26 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) /* Setup sscratch for guest_get_vcpuid() */ vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(sscratch), vcpu_id); - /* Setup default exception vector of guest */ - vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(stvec), (unsigned long)guest_unexp_trap); + /* + * Advertise V to KVM so -O2 auto-vectorization in guest code is valid; + * ignore errors since the tests work without V too. Use the full + * exception vector table (which lazily enables V in route_exception()) + * as the default handler; vm_init_vector_tables() is idempotent. + */ + __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(KVM_RISCV_ISA_EXT_V), 1); + vm_init_vector_tables(vm); + vcpu_init_vector_tables(vcpu); + + /* + * Record V availability for route_exception(), which runs in guest + * context. V is enabled uniformly for every vCPU, so this is a + * VM-wide property. + */ + { + struct handlers *h = addr_gva2hva(vm, vm->handlers); + + h->v_available = __vcpu_has_isa_ext(vcpu, KVM_RISCV_ISA_EXT_V); + } return vcpu; } @@ -406,19 +432,43 @@ void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) struct ucall uc; if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { + vcpu_dump(stderr, vcpu, 2); TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)", uc.args[0], uc.args[1]); } } -struct handlers { - exception_handler_fn exception_handlers[NR_VECTORS][NR_EXCEPTIONS]; -}; +static bool insn_is_vector(u32 insn) +{ + u32 opcode = insn & RV_INSN_OPCODE_MASK; + u32 width, csr; + + /* All V-related instructions are 4-byte, i.e. not compressed. */ + if ((insn & 0x3) != 0x3) + return false; + + switch (opcode) { + case RVV_OPCODE_VECTOR: + return true; + case RVV_OPCODE_VL: + case RVV_OPCODE_VS: + width = RVV_EXTRACT_VL_VS_WIDTH(insn); + return width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || + width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64; + case RVG_OPCODE_SYSTEM: + csr = RVG_EXTRACT_SYSTEM_CSR(insn); + return (csr >= CSR_VSTART && csr <= CSR_VCSR) || + (csr >= CSR_VL && csr <= CSR_VLENB); + } + + return false; +} void route_exception(struct pt_regs *regs) { struct handlers *handlers = (struct handlers *)exception_handlers; - int vector = 0, ec; + int vector = 0; + unsigned long ec; ec = regs->cause & ~CAUSE_IRQ_FLAG; if (ec >= NR_EXCEPTIONS) @@ -430,6 +480,36 @@ void route_exception(struct pt_regs *regs) ec = 0; } + /* + * Lazily enable V on the first vector instruction: if the faulting + * instruction decodes as vector while VS is off, set VS to Initial + * and re-execute it, like the kernel's riscv_v_first_use_handler(). + * Genuinely illegal instructions continue to the unexpected-exception + * path. + */ + if (!(regs->cause & CAUSE_IRQ_FLAG) && ec == EXC_INST_ILLEGAL && + handlers && handlers->v_available && !(regs->status & SR_VS)) { + u32 insn = (u32)regs->badaddr; + + /* + * stval is not guaranteed to hold the faulting instruction. + * Vector instructions are always 32-bit, so check the first + * 16-bit halfword and, for a 32-bit instruction, assemble it + * from two halfwords to avoid an unaligned 32-bit access. + */ + if (!insn) { + u16 half = *(u16 *)regs->epc; + + if ((half & 0x3) == 0x3) + insn = (u32)half | ((u32)*(u16 *)(regs->epc + 2) << 16); + } + + if (insn_is_vector(insn)) { + regs->status |= SR_VS_INITIAL; + return; + } + } + if (handlers && handlers->exception_handlers[vector][ec]) return handlers->exception_handlers[vector][ec](regs); @@ -446,6 +526,9 @@ void vcpu_init_vector_tables(struct kvm_vcpu *vcpu) void vm_init_vector_tables(struct kvm_vm *vm) { + if (vm->handlers) + return; + vm->handlers = __vm_alloc(vm, sizeof(struct handlers), vm->page_size, MEM_REGION_DATA); diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index bf3980e9db12..71e5969142b3 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -63,6 +63,7 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVPBMT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZA64RS: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAAMO: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZABHA: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZACAS: @@ -89,6 +90,9 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOP: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOZ: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCAMOA: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCIF: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCLSM: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCRSE: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICFILP: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICFISS: @@ -576,6 +580,7 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZA64RS), KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), @@ -602,6 +607,9 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) KVM_ISA_EXT_ARR(ZICBOM), KVM_ISA_EXT_ARR(ZICBOP), KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCAMOA), + KVM_ISA_EXT_ARR(ZICCIF), + KVM_ISA_EXT_ARR(ZICCLSM), KVM_ISA_EXT_ARR(ZICCRSE), KVM_ISA_EXT_ARR(ZICFILP), KVM_ISA_EXT_ARR(ZICFISS), @@ -1238,6 +1246,7 @@ KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL); KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT); KVM_ISA_EXT_SIMPLE_CONFIG(svpbmt, SVPBMT); KVM_ISA_EXT_SIMPLE_CONFIG(svvptc, SVVPTC); +KVM_ISA_EXT_SIMPLE_CONFIG(za64rs, ZA64RS); KVM_ISA_EXT_SIMPLE_CONFIG(zaamo, ZAAMO); KVM_ISA_EXT_SIMPLE_CONFIG(zabha, ZABHA); KVM_ISA_EXT_SIMPLE_CONFIG(zacas, ZACAS); @@ -1264,6 +1273,9 @@ KVM_ISA_EXT_SIMPLE_CONFIG(zfhmin, ZFHMIN); KVM_ISA_EXT_SUBLIST_CONFIG(zicbom, ZICBOM); KVM_ISA_EXT_SUBLIST_CONFIG(zicbop, ZICBOP); KVM_ISA_EXT_SUBLIST_CONFIG(zicboz, ZICBOZ); +KVM_ISA_EXT_SIMPLE_CONFIG(ziccamoa, ZICCAMOA); +KVM_ISA_EXT_SIMPLE_CONFIG(ziccif, ZICCIF); +KVM_ISA_EXT_SIMPLE_CONFIG(zicclsm, ZICCLSM); KVM_ISA_EXT_SIMPLE_CONFIG(ziccrse, ZICCRSE); KVM_ISA_EXT_SIMPLE_CONFIG(zicfilp, ZICFILP); KVM_ISA_EXT_SUBLIST_CONFIG(zicfiss, ZICFISS); @@ -1372,6 +1384,7 @@ struct vcpu_reg_list *vcpu_configs[] = { &config_svnapot, &config_svpbmt, &config_svvptc, + &config_za64rs, &config_zaamo, &config_zabha, &config_zacas, @@ -1398,6 +1411,9 @@ struct vcpu_reg_list *vcpu_configs[] = { &config_zicbom, &config_zicbop, &config_zicboz, + &config_ziccamoa, + &config_ziccif, + &config_zicclsm, &config_ziccrse, &config_zicfilp, &config_zicfiss, |