Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git

46 files changed, 561 insertions, 336 deletions

diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index 1da277baacf7..12a500181291 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -61,7 +61,7 @@
 	b.ne	.LnVHE_\@	// Otherwise, we know that nVHE works.
 
 .LnE2H0_\@:
-	orr	x0, x0, #HCR_E2H
+	orr	x0, x0, #HCR_EL2_E2H
 	msr_hcr_el2 x0
 .LnVHE_\@:
 .endm
@@ -102,7 +102,7 @@
 /* Check if running in host at EL2 mode, i.e., (h)VHE. Jump to fail if not. */
 .macro __check_hvhe fail, tmp
 	mrs	\tmp, hcr_el2
-	and	\tmp, \tmp, #HCR_E2H
+	and	\tmp, \tmp, #HCR_EL2_E2H
 	cbz	\tmp, \fail
 .endm
 
diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index 77d6b8c63d4e..024e5a24973e 100644
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -40,8 +40,8 @@ do {									\
 	}								\
 									\
 	___hcr = read_sysreg(hcr_el2);					\
-	if (!(___hcr & HCR_TGE)) {					\
-		write_sysreg_hcr(___hcr | HCR_TGE);			\
+	if (!(___hcr & HCR_EL2_TGE)) {					\
+		write_sysreg_hcr(___hcr | HCR_EL2_TGE);			\
 		isb();							\
 	}								\
 	/*								\
@@ -81,7 +81,7 @@ do {									\
 	 * we secured things for it.					\
 	 */								\
 	barrier();							\
-	if (!___ctx->cnt && !(___hcr & HCR_TGE))			\
+	if (!___ctx->cnt && !(___hcr & HCR_EL2_TGE))			\
 		write_sysreg_hcr(___hcr);				\
 } while (0)
 
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 678f0daed445..ba0062849be9 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -15,71 +15,6 @@
 #include <asm/sysreg.h>
 
 /*
- * Because I'm terribly lazy and that repainting the whole of the KVM
- * code with the proper names is a pain, use a helper to map the names
- * inherited from AArch32 with the new fancy nomenclature. One day...
- */
-#define	__HCR(x)	HCR_EL2_##x
-
-#define HCR_TID5	__HCR(TID5)
-#define HCR_DCT		__HCR(DCT)
-#define HCR_ATA_SHIFT	__HCR(ATA_SHIFT)
-#define HCR_ATA		__HCR(ATA)
-#define HCR_TTLBOS	__HCR(TTLBOS)
-#define HCR_TTLBIS	__HCR(TTLBIS)
-#define HCR_ENSCXT	__HCR(EnSCXT)
-#define HCR_TOCU	__HCR(TOCU)
-#define HCR_AMVOFFEN	__HCR(AMVOFFEN)
-#define HCR_TICAB	__HCR(TICAB)
-#define HCR_TID4	__HCR(TID4)
-#define HCR_FIEN	__HCR(FIEN)
-#define HCR_FWB		__HCR(FWB)
-#define HCR_NV2		__HCR(NV2)
-#define HCR_AT		__HCR(AT)
-#define HCR_NV1		__HCR(NV1)
-#define HCR_NV		__HCR(NV)
-#define HCR_API		__HCR(API)
-#define HCR_APK		__HCR(APK)
-#define HCR_TEA		__HCR(TEA)
-#define HCR_TERR	__HCR(TERR)
-#define HCR_TLOR	__HCR(TLOR)
-#define HCR_E2H		__HCR(E2H)
-#define HCR_ID		__HCR(ID)
-#define HCR_CD		__HCR(CD)
-#define HCR_RW		__HCR(RW)
-#define HCR_TRVM	__HCR(TRVM)
-#define HCR_HCD		__HCR(HCD)
-#define HCR_TDZ		__HCR(TDZ)
-#define HCR_TGE		__HCR(TGE)
-#define HCR_TVM		__HCR(TVM)
-#define HCR_TTLB	__HCR(TTLB)
-#define HCR_TPU		__HCR(TPU)
-#define HCR_TPC		__HCR(TPCP)
-#define HCR_TSW		__HCR(TSW)
-#define HCR_TACR	__HCR(TACR)
-#define HCR_TIDCP	__HCR(TIDCP)
-#define HCR_TSC		__HCR(TSC)
-#define HCR_TID3	__HCR(TID3)
-#define HCR_TID2	__HCR(TID2)
-#define HCR_TID1	__HCR(TID1)
-#define HCR_TID0	__HCR(TID0)
-#define HCR_TWE		__HCR(TWE)
-#define HCR_TWI		__HCR(TWI)
-#define HCR_DC		__HCR(DC)
-#define HCR_BSU		__HCR(BSU)
-#define HCR_BSU_IS	__HCR(BSU_IS)
-#define HCR_FB		__HCR(FB)
-#define HCR_VSE		__HCR(VSE)
-#define HCR_VI		__HCR(VI)
-#define HCR_VF		__HCR(VF)
-#define HCR_AMO		__HCR(AMO)
-#define HCR_IMO		__HCR(IMO)
-#define HCR_FMO		__HCR(FMO)
-#define HCR_PTW		__HCR(PTW)
-#define HCR_SWIO	__HCR(SWIO)
-#define HCR_VM		__HCR(VM)
-
-/*
  * The bits we set in HCR:
  * TLOR:	Trap LORegion register accesses
  * RW:		64bit by default, can be overridden for 32bit VMs
@@ -99,13 +34,13 @@
  * TID3:	Trap EL1 reads of group 3 ID registers
  * TID1:	Trap REVIDR_EL1, AIDR_EL1, and SMIDR_EL1
  */
-#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
-			 HCR_BSU_IS | HCR_FB | HCR_TACR | \
-			 HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
-			 HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1)
-#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
-#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)
-#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H | HCR_AMO | HCR_IMO | HCR_FMO)
+#define HCR_GUEST_FLAGS (HCR_EL2_TSC | HCR_EL2_TSW | HCR_EL2_TWE | HCR_EL2_TWI | HCR_EL2_VM | \
+			 SYS_FIELD_PREP_ENUM(HCR_EL2, BSU, IS) | HCR_EL2_FB | HCR_EL2_TACR | \
+			 HCR_EL2_AMO | HCR_EL2_SWIO | HCR_EL2_TIDCP | HCR_EL2_RW | HCR_EL2_TLOR | \
+			 HCR_EL2_FMO | HCR_EL2_IMO | HCR_EL2_PTW | HCR_EL2_TID3 | HCR_EL2_TID1)
+#define HCR_HOST_NVHE_FLAGS (HCR_EL2_RW | HCR_EL2_API | HCR_EL2_APK)
+#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_EL2_TSC)
+#define HCR_HOST_VHE_FLAGS (HCR_EL2_RW | HCR_EL2_TGE | HCR_EL2_E2H | HCR_EL2_AMO | HCR_EL2_IMO | HCR_EL2_FMO)
 
 #define MPAMHCR_HOST_FLAGS	0
 
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 7d799cd05dcd..abda476e023d 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -63,6 +63,17 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config,
 	__KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
+	__KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,
+	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa,
+	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh,
+	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid,
+	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,
+	__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
 
 	MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM),
 
@@ -71,12 +82,6 @@ enum __kvm_host_smccc_func {
 	/* Hypercalls that are always available and common to [nh]VHE/pKVM. */
 	__KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
 	__KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,
-	__KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,
-	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa,
-	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh,
-	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid,
-	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,
-	__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
 	__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
 	__KVM_HOST_SMCCC_FUNC___tracing_load,
 	__KVM_HOST_SMCCC_FUNC___tracing_unload,
@@ -88,11 +93,6 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___tracing_write_event,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
-	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident,
-	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident,
-	__KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend,
-	__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
-	__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
 	__KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup,
 	__KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim,
 	__KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable,
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 85289a4b8d52..334211b8b2ee 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -95,7 +95,7 @@ static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
 #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
 {
-	return !(vcpu->arch.hcr_el2 & HCR_RW);
+	return !(vcpu->arch.hcr_el2 & HCR_EL2_RW);
 }
 #else
 static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
@@ -116,7 +116,7 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 	 * then.
 	 */
 	if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
-		vcpu->arch.hcr_el2 |= HCR_TVM;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TVM;
 }
 
 static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
@@ -227,12 +227,12 @@ static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu)
 static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu)
 {
 	return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) ||
-		(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H));
+		(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_E2H));
 }
 
 static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu)
 {
-	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
+	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_TGE;
 }
 
 static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
@@ -251,7 +251,7 @@ static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
 	if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu))
 		return true;
 
-	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;
+	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_AMO;
 }
 
 static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
@@ -264,8 +264,8 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
 
 	hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
 
-	e2h = (hcr & HCR_E2H);
-	tge = (hcr & HCR_TGE);
+	e2h = (hcr & HCR_EL2_E2H);
+	tge = (hcr & HCR_EL2_TGE);
 
 	/*
 	 * We are in a hypervisor context if the vcpu mode is EL2 or
@@ -376,8 +376,8 @@ static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu)
 	if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu))
 		return false;
 
-	return ((is_wfe && (hcr_el2 & HCR_TWE)) ||
-		(!is_wfe && (hcr_el2 & HCR_TWI)));
+	return ((is_wfe && (hcr_el2 & HCR_EL2_TWE)) ||
+		(!is_wfe && (hcr_el2 & HCR_EL2_TWI)));
 }
 
 static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 6dd8462145c6..97332ffcb130 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -967,11 +967,10 @@ struct kvm_vcpu_arch {
 	/*
 	 * Don't run the guest (internal implementation need).
 	 *
-	 * Contrary to the flags above, this is set/cleared outside of
-	 * a vcpu context, and thus cannot be mixed with the flags
-	 * themselves (or the flag accesses need to be made atomic).
+	 * Contrary to the flags above, this is updated outside of
+	 * a vcpu context, and thus cannot be mixed with the flags.
 	 */
-	bool pause;
+	atomic_t pause;
 
 	/*
 	 * We maintain more than a single set of debug registers to support
@@ -1355,6 +1354,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
 
 void kvm_arm_halt_guest(struct kvm *kvm);
 void kvm_arm_resume_guest(struct kvm *kvm);
+void kvm_pause_vcpu(struct kvm_vcpu *vcpu);
+void kvm_resume_vcpu(struct kvm_vcpu *vcpu);
 
 #define vcpu_has_run_once(vcpu)	(!!READ_ONCE((vcpu)->pid))
 
@@ -1746,6 +1747,7 @@ static inline bool kvm_arch_has_irq_bypass(void)
 	return true;
 }
 
+void kvm_idregs_enforce_dependencies(struct kvm *kvm);
 void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt);
 struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg);
 void check_feature_map(void);
diff --git a/arch/arm64/include/asm/kvm_mte.h b/arch/arm64/include/asm/kvm_mte.h
index 3171963ad25c..502d6df91691 100644
--- a/arch/arm64/include/asm/kvm_mte.h
+++ b/arch/arm64/include/asm/kvm_mte.h
@@ -16,7 +16,7 @@ alternative_if_not ARM64_MTE
 	b	.L__skip_switch\@
 alternative_else_nop_endif
 	mrs	\reg1, hcr_el2
-	tbz	\reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@
+	tbz	\reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@
 
 	mrs_s	\reg1, SYS_RGSR_EL1
 	str	\reg1, [\h_ctxt, #CPU_RGSR_EL1]
@@ -36,7 +36,7 @@ alternative_if_not ARM64_MTE
 	b	.L__skip_switch\@
 alternative_else_nop_endif
 	mrs	\reg1, hcr_el2
-	tbz	\reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@
+	tbz	\reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@
 
 	mrs_s	\reg1, SYS_RGSR_EL1
 	str	\reg1, [\g_ctxt, #CPU_RGSR_EL1]
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index e7562770e7cb..fd042d2b5462 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -17,7 +17,6 @@
 
 #define HYP_MEMBLOCK_REGIONS 128
 
-void pkvm_selftests(void);
 int pkvm_init_host_vm(struct kvm *kvm);
 int pkvm_create_hyp_vm(struct kvm *kvm);
 bool pkvm_hyp_vm_is_created(struct kvm *kvm);
diff --git a/arch/arm64/include/asm/kvm_ptrauth.h b/arch/arm64/include/asm/kvm_ptrauth.h
index e50987b32483..24f75457a712 100644
--- a/arch/arm64/include/asm/kvm_ptrauth.h
+++ b/arch/arm64/include/asm/kvm_ptrauth.h
@@ -71,7 +71,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH
 	b	.L__skip_switch\@
 alternative_else_nop_endif
 	mrs	\reg1, hcr_el2
-	and	\reg1, \reg1, #(HCR_API | HCR_APK)
+	and	\reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK)
 	cbz	\reg1, .L__skip_switch\@
 	add	\reg1, \g_ctxt, #CPU_APIAKEYLO_EL1
 	ptrauth_restore_state	\reg1, \reg2, \reg3
@@ -83,7 +83,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH
 	b	.L__skip_switch\@
 alternative_else_nop_endif
 	mrs	\reg1, hcr_el2
-	and	\reg1, \reg1, #(HCR_API | HCR_APK)
+	and	\reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK)
 	cbz	\reg1, .L__skip_switch\@
 	add	\reg1, \g_ctxt, #CPU_APIAKEYLO_EL1
 	ptrauth_save_state	\reg1, \reg2, \reg3
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 8951ce693552..d6ba3c0af811 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -298,7 +298,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
 	isb
 0:
 
-	init_el2_hcr	HCR_HOST_NVHE_FLAGS | HCR_ATA
+	init_el2_hcr	HCR_HOST_NVHE_FLAGS | HCR_EL2_ATA
 	init_el2_state
 
 	/* Hypervisor stub */
@@ -309,7 +309,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
 	mov_q	x1, INIT_SCTLR_EL1_MMU_OFF
 
 	mrs	x0, hcr_el2
-	and	x0, x0, #HCR_E2H
+	and	x0, x0, #HCR_EL2_E2H
 	cbz	x0, 2f
 
 	/* Set a sane SCTLR_EL1, the VHE way */
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 48d531e302d3..cecea0820e52 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -71,7 +71,7 @@ if NVHE_EL2_DEBUG
 
 config NVHE_EL2_TRACING
 	bool
-	depends on TRACING && FTRACE
+	depends on FTRACE
 	select TRACE_REMOTE
 	default y
 
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index d30378a50c53..157bf4b81202 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -590,6 +590,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
 
 	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
+	atomic_set(&vcpu->arch.pause, 0);
 
 	/*
 	 * This vCPU may have been created after mpidr_data was initialized.
@@ -650,11 +651,11 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
 		u64 val;
 
 		val = __vcpu_sys_reg(vcpu, HCR_EL2);
-		val &= (HCR_API | HCR_APK);
-		vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
+		val &= (HCR_EL2_API | HCR_EL2_APK);
+		vcpu->arch.hcr_el2 &= ~(HCR_EL2_API | HCR_EL2_APK);
 		vcpu->arch.hcr_el2 |= val;
 	} else {
-		vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
+		vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK);
 	}
 
 	/*
@@ -662,7 +663,7 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
 	 * to use pauth, as the entry code will reload the guest
 	 * keys in that case.
 	 */
-	if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
+	if (vcpu->arch.hcr_el2 & (HCR_EL2_API | HCR_EL2_APK)) {
 		struct kvm_cpu_context *ctxt;
 
 		ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
@@ -736,14 +737,14 @@ static void vcpu_put_mmu(struct kvm_vcpu *vcpu)
 static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
 {
 	if (kvm_vcpu_should_clear_twe(vcpu))
-		vcpu->arch.hcr_el2 &= ~HCR_TWE;
+		vcpu->arch.hcr_el2 &= ~HCR_EL2_TWE;
 	else
-		vcpu->arch.hcr_el2 |= HCR_TWE;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TWE;
 
 	if (kvm_vcpu_should_clear_twi(vcpu))
-		vcpu->arch.hcr_el2 &= ~HCR_TWI;
+		vcpu->arch.hcr_el2 &= ~HCR_EL2_TWI;
 	else
-		vcpu->arch.hcr_el2 |= HCR_TWI;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TWI;
 }
 
 static void vcpu_load_pvtime(struct kvm_vcpu *vcpu)
@@ -954,6 +955,11 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 	return ret;
 }
 
+static bool vcpu_can_run(struct kvm_vcpu *vcpu)
+{
+	return !kvm_arm_vcpu_stopped(vcpu) && !atomic_read(&vcpu->arch.pause);
+}
+
 /**
  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
  * @v:		The VCPU pointer
@@ -963,14 +969,13 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  */
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 {
-	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
+	bool irq_lines = *vcpu_hcr(v) & (HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_VSE);
 
 	irq_lines |= (!irqchip_in_kernel(v->kvm) &&
 		      (kvm_timer_should_notify_user(v) ||
 		       kvm_pmu_should_notify_user(v)));
 
-	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
-		&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
+	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) && vcpu_can_run(v));
 }
 
 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
@@ -1127,13 +1132,28 @@ bool kvm_arch_intc_initialized(struct kvm *kvm)
 	return vgic_initialized(kvm);
 }
 
+void kvm_pause_vcpu(struct kvm_vcpu *vcpu)
+{
+	DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS) = { };
+
+	atomic_inc(&vcpu->arch.pause);
+	__set_bit(vcpu->vcpu_idx, vcpu_bitmap);
+	kvm_make_vcpus_request_mask(vcpu->kvm, KVM_REQ_SLEEP, vcpu_bitmap);
+}
+
+void kvm_resume_vcpu(struct kvm_vcpu *vcpu)
+{
+	if (atomic_dec_and_test(&vcpu->arch.pause))
+		__kvm_vcpu_wake_up(vcpu);
+}
+
 void kvm_arm_halt_guest(struct kvm *kvm)
 {
 	unsigned long i;
 	struct kvm_vcpu *vcpu;
 
 	kvm_for_each_vcpu(i, vcpu, kvm)
-		vcpu->arch.pause = true;
+		atomic_inc(&vcpu->arch.pause);
 	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
 }
 
@@ -1142,21 +1162,17 @@ void kvm_arm_resume_guest(struct kvm *kvm)
 	unsigned long i;
 	struct kvm_vcpu *vcpu;
 
-	kvm_for_each_vcpu(i, vcpu, kvm) {
-		vcpu->arch.pause = false;
-		__kvm_vcpu_wake_up(vcpu);
-	}
+	kvm_for_each_vcpu(i, vcpu, kvm)
+		kvm_resume_vcpu(vcpu);
 }
 
 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
 {
 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
 
-	rcuwait_wait_event(wait,
-			   (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
-			   TASK_INTERRUPTIBLE);
+	rcuwait_wait_event(wait, vcpu_can_run(vcpu), TASK_INTERRUPTIBLE);
 
-	if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
+	if (!vcpu_can_run(vcpu)) {
 		/* Awaken to handle a signal, request we sleep again later. */
 		kvm_make_request(KVM_REQ_SLEEP, vcpu);
 	}
@@ -1437,38 +1453,40 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		 */
 		preempt_disable();
 
-		kvm_nested_flush_hwstate(vcpu);
-
 		if (kvm_vcpu_has_pmu(vcpu))
 			kvm_pmu_flush_hwstate(vcpu);
 
-		local_irq_disable();
-
-		kvm_vgic_flush_hwstate(vcpu);
+		kvm_nested_flush_hwstate(vcpu);
 
-		kvm_pmu_update_vcpu_events(vcpu);
+		local_irq_disable();
 
 		/*
 		 * Ensure we set mode to IN_GUEST_MODE after we disable
 		 * interrupts and before the final VCPU requests check.
 		 * See the comment in kvm_vcpu_exiting_guest_mode() and
 		 * Documentation/virt/kvm/vcpu-requests.rst
+		 *
+		 * Requests to self must be posted before this point, or
+		 * won't be taken into account until the next exit. No state
+		 * loading other than what happens at vcpu_load() time can
+		 * be done before we have checked for pending requests.
 		 */
 		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
 
 		if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
-			vcpu->mode = OUTSIDE_GUEST_MODE;
-			isb(); /* Ensure work in x_flush_hwstate is committed */
-			if (kvm_vcpu_has_pmu(vcpu))
-				kvm_pmu_sync_hwstate(vcpu);
 			if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 				kvm_timer_sync_user(vcpu);
-			kvm_vgic_sync_hwstate(vcpu);
+			smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
 			local_irq_enable();
 			preempt_enable();
 			continue;
 		}
 
+		/* From here onwards, we are commited to entering the guest */
+		kvm_vgic_flush_hwstate(vcpu);
+
+		kvm_pmu_update_vcpu_events(vcpu);
+
 		kvm_arch_vcpu_ctxflush_fp(vcpu);
 
 		/**************************************************************
@@ -1479,7 +1497,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
 		ret = kvm_arm_vcpu_enter_exit(vcpu);
 
-		vcpu->mode = OUTSIDE_GUEST_MODE;
 		vcpu->stat.exits++;
 		/*
 		 * Back from guest
@@ -1514,6 +1531,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		kvm_arch_vcpu_ctxsync_fp(vcpu);
 
 		/*
+		 * The state that was pushed into the CPU on entering
+		 * the guest has been synchronised back, let's
+		 * advertise we're outside of the guest.
+		 */
+		smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
+
+		/*
 		 * We must ensure that any pending interrupts are taken before
 		 * we exit guest timing so that timer ticks are accounted as
 		 * guest time. Transiently unmask interrupts so that any
@@ -1592,9 +1616,9 @@ static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
 	unsigned long *hcr;
 
 	if (number == KVM_ARM_IRQ_CPU_IRQ)
-		bit_index = __ffs(HCR_VI);
+		bit_index = __ffs(HCR_EL2_VI);
 	else /* KVM_ARM_IRQ_CPU_FIQ */
-		bit_index = __ffs(HCR_VF);
+		bit_index = __ffs(HCR_EL2_VF);
 
 	hcr = vcpu_hcr(vcpu);
 	if (level)
@@ -2384,12 +2408,12 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
 		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
 
 	if (system_supports_mte())
-		params->hcr_el2 |= HCR_ATA;
+		params->hcr_el2 |= HCR_EL2_ATA;
 	else
-		params->hcr_el2 |= HCR_TID5;
+		params->hcr_el2 |= HCR_EL2_TID5;
 
 	if (cpus_have_final_cap(ARM64_KVM_HVHE))
-		params->hcr_el2 |= HCR_E2H;
+		params->hcr_el2 |= HCR_EL2_E2H;
 	params->vttbr = params->vtcr = 0;
 
 	/*
@@ -3109,8 +3133,6 @@ static int __init init_hyp_mode(void)
 			kvm_err("Failed to init hyp memory protection\n");
 			goto out_err;
 		}
-
-		pkvm_selftests();
 	}
 
 	return 0;
diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
index 0926426b8798..30d8f062b2c2 100644
--- a/arch/arm64/kvm/at.c
+++ b/arch/arm64/kvm/at.c
@@ -229,7 +229,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
 
 	if (vcpu_has_nv(vcpu)) {
 		hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
-		wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC));
+		wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_EL2_VM | HCR_EL2_DC));
 	} else {
 		WARN_ON_ONCE(wi->regime != TR_EL10);
 		wi->s2 = false;
@@ -305,7 +305,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
 		 * is EL1&0, then it means that either {E2H,TGE} == {1,0} or
 		 * {0,x}, and we only need to test for TGE == 1.
 		 */
-		if (hcr & (HCR_DC | HCR_TGE)) {
+		if (hcr & (HCR_EL2_DC | HCR_EL2_TGE)) {
 			wr->level = S1_MMU_DISABLED;
 			break;
 		}
@@ -870,7 +870,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par,
 	s1_parattr = FIELD_GET(SYS_PAR_EL1_ATTR, s1_par);
 	s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc);
 
-	if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_FWB) {
+	if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_FWB) {
 		if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP))
 			s2_memattr &= ~BIT(3);
 
@@ -924,7 +924,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par,
 		}
 	}
 
-	if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_CD) &&
+	if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_CD) &&
 	    !MEMATTR_IS_DEVICE(final_attr))
 		final_attr = MEMATTR(NC, NC);
 
@@ -956,7 +956,7 @@ static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
 		par |= wr->pa & SYS_PAR_EL1_PA;
 
 		if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) &&
-		    (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) {
+		    (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_DC)) {
 			par |= FIELD_PREP(SYS_PAR_EL1_ATTR,
 					  MEMATTR(WbRaWa, WbRaWa));
 			par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_NSH);
@@ -1548,11 +1548,11 @@ int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
 		bool fail;
 
 		val = hcr = read_sysreg(hcr_el2);
-		val &= ~HCR_TGE;
-		val |= HCR_VM;
+		val &= ~HCR_EL2_TGE;
+		val |= HCR_EL2_VM;
 
 		if (!vcpu_el2_e2h_is_set(vcpu))
-			val |= HCR_NV | HCR_NV1;
+			val |= HCR_EL2_NV | HCR_EL2_NV1;
 
 		write_sysreg_hcr(val);
 		isb();
@@ -1635,7 +1635,7 @@ int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
 	 * early. Same thing if {VM,DC}=={0,0}.
 	 */
 	if (compute_translation_regime(vcpu, op) == TR_EL20 ||
-	    !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM | HCR_DC)))
+	    !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_VM | HCR_EL2_DC)))
 		return 0;
 
 	/* Do the stage-2 translation */
diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c
index e93919cadf86..adaf26ab93b8 100644
--- a/arch/arm64/kvm/config.c
+++ b/arch/arm64/kvm/config.c
@@ -59,6 +59,7 @@ struct reg_feat_map_desc {
 	const struct reg_bits_to_feat_map feat_map;
 	const struct reg_bits_to_feat_map *bit_feat_map;
 	const unsigned int		  bit_feat_map_sz;
+	const unsigned int		  idreg_idx;
 };
 
 #define __NEEDS_FEAT_3(m, f, w, id, fld, lim)		\
@@ -130,6 +131,21 @@ struct reg_feat_map_desc {
 		.bit_feat_map_sz	= ARRAY_SIZE(m),		\
 	}
 
+/*
+ * Declare the dependency between the features advertised in an id
+ * register, and the set of other features they depend on. This
+ * generates a struct reg_feat_map_desc.
+ */
+#define DECLARE_IDREG_FEAT_MAP(n, r, m, f)				\
+	struct reg_feat_map_desc n = {					\
+		.name			= #r,				\
+		.feat_map		= NEEDS_FEAT(~(r##_RES0 |	\
+						       r##_RES1), f),	\
+		.bit_feat_map		= m,				\
+		.bit_feat_map_sz	= ARRAY_SIZE(m),		\
+		.idreg_idx		= IDREG_IDX(SYS_##r),		\
+	}
+
 #define FEAT_SPE		ID_AA64DFR0_EL1, PMSVer, IMP
 #define FEAT_BRBE		ID_AA64DFR0_EL1, BRBE, IMP
 #define FEAT_TRC_SR		ID_AA64DFR0_EL1, TraceVer, IMP
@@ -205,6 +221,7 @@ struct reg_feat_map_desc {
 #define FEAT_MEC		ID_AA64MMFR3_EL1, MEC, IMP
 #define FEAT_HAFT		ID_AA64MMFR1_EL1, HAFDBS, HAFT
 #define FEAT_HDBSS		ID_AA64MMFR1_EL1, HAFDBS, HDBSS
+#define FEAT_HPDS		ID_AA64MMFR1_EL1, HPDS, IMP
 #define FEAT_HPDS2		ID_AA64MMFR1_EL1, HPDS, HPDS2
 #define FEAT_BTI		ID_AA64PFR1_EL1, BT, IMP
 #define FEAT_ExS		ID_AA64MMFR0_EL1, EXS, IMP
@@ -280,6 +297,25 @@ static bool feat_anerr(struct kvm *kvm)
 		kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SNERR, FEAT_ANERR));
 }
 
+/*
+ * S1PIE and S1POE imply TCR2 and ATS1A, and S1POE also implies HPDS, but
+ * userspace can set them independently.
+ */
+static bool feat_s1pie(struct kvm *kvm)
+{
+	return (kvm_has_feat(kvm, FEAT_S1PIE) &&
+		kvm_has_feat(kvm, FEAT_TCR2) &&
+		kvm_has_feat(kvm, FEAT_ATS1A));
+}
+
+static bool feat_s1poe(struct kvm *kvm)
+{
+	return (kvm_has_feat(kvm, FEAT_S1POE) &&
+		kvm_has_feat(kvm, FEAT_TCR2) &&
+		kvm_has_feat(kvm, FEAT_ATS1A) &&
+		kvm_has_feat(kvm, FEAT_HPDS));
+}
+
 static bool feat_sme_smps(struct kvm *kvm)
 {
 	/*
@@ -1299,6 +1335,14 @@ static const struct reg_bits_to_feat_map vtcr_el2_feat_map[] = {
 static const DECLARE_FEAT_MAP(vtcr_el2_desc, VTCR_EL2,
 			      vtcr_el2_feat_map, FEAT_AA64EL2);
 
+static const struct reg_bits_to_feat_map id_aa64mmfr3_el1_feat_map[] = {
+	NEEDS_FEAT(ID_AA64MMFR3_EL1_S1PIE, feat_s1pie),
+	NEEDS_FEAT(ID_AA64MMFR3_EL1_S1POE, feat_s1poe),
+};
+
+static const DECLARE_IDREG_FEAT_MAP(id_aa64mmfr3_el1_desc, ID_AA64MMFR3_EL1,
+				    id_aa64mmfr3_el1_feat_map, FEAT_AA64EL1);
+
 static const struct reg_bits_to_feat_map ich_hfgrtr_feat_map[] = {
 	NEEDS_FEAT(ICH_HFGRTR_EL2_ICC_APR_EL1 |
 		   ICH_HFGRTR_EL2_ICC_IDRn_EL1 |
@@ -1408,6 +1452,40 @@ void __init check_feature_map(void)
 	check_reg_desc(&ich_hfgitr_desc);
 }
 
+static const struct reg_feat_map_desc *idreg_feat_maps[] = {
+	&id_aa64mmfr3_el1_desc,
+};
+
+static bool idreg_apply_deps(struct kvm *kvm, const struct reg_feat_map_desc *r)
+{
+	struct resx resx = {};
+	u64 val;
+
+	for (int i = 0; i < r->bit_feat_map_sz; i++) {
+		if ((r->bit_feat_map[i].flags & CALL_FUNC) &&
+		    !r->bit_feat_map[i].match(kvm))
+			resx.res0 |= reg_feat_map_bits(&r->bit_feat_map[i]);
+	}
+
+	val = kvm->arch.id_regs[r->idreg_idx];
+	kvm->arch.id_regs[r->idreg_idx] &= ~resx.res0;
+	kvm->arch.id_regs[r->idreg_idx] |= resx.res1;
+
+	return val != kvm->arch.id_regs[r->idreg_idx];
+}
+
+void kvm_idregs_enforce_dependencies(struct kvm *kvm)
+{
+	bool again;
+
+	do {
+		again = false;
+
+		for (int i = 0; i < ARRAY_SIZE(idreg_feat_maps); i++)
+			again |= idreg_apply_deps(kvm, idreg_feat_maps[i]);
+	} while (again);
+}
+
 static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
 {
 	u64 regval = kvm->arch.id_regs[map->regidx];
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index fc23c793ef17..2bec81668361 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -145,176 +145,176 @@ enum cgt_group_id {
 static const struct trap_bits coarse_trap_bits[] = {
 	[CGT_HCR_TID1] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TID1,
-		.mask		= HCR_TID1,
+		.value 		= HCR_EL2_TID1,
+		.mask		= HCR_EL2_TID1,
 		.behaviour	= BEHAVE_FORWARD_READ,
 	},
 	[CGT_HCR_TID2] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TID2,
-		.mask		= HCR_TID2,
+		.value 		= HCR_EL2_TID2,
+		.mask		= HCR_EL2_TID2,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TID3] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TID3,
-		.mask		= HCR_TID3,
+		.value 		= HCR_EL2_TID3,
+		.mask		= HCR_EL2_TID3,
 		.behaviour	= BEHAVE_FORWARD_READ,
 	},
 	[CGT_HCR_IMO] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_IMO,
-		.mask		= HCR_IMO,
+		.value 		= HCR_EL2_IMO,
+		.mask		= HCR_EL2_IMO,
 		.behaviour	= BEHAVE_FORWARD_WRITE,
 	},
 	[CGT_HCR_FMO] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_FMO,
-		.mask		= HCR_FMO,
+		.value 		= HCR_EL2_FMO,
+		.mask		= HCR_EL2_FMO,
 		.behaviour	= BEHAVE_FORWARD_WRITE,
 	},
 	[CGT_HCR_TIDCP] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TIDCP,
-		.mask		= HCR_TIDCP,
+		.value		= HCR_EL2_TIDCP,
+		.mask		= HCR_EL2_TIDCP,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TACR] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TACR,
-		.mask		= HCR_TACR,
+		.value		= HCR_EL2_TACR,
+		.mask		= HCR_EL2_TACR,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TSW] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TSW,
-		.mask		= HCR_TSW,
+		.value		= HCR_EL2_TSW,
+		.mask		= HCR_EL2_TSW,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */
 		.index		= HCR_EL2,
-		.value		= HCR_TPC,
-		.mask		= HCR_TPC,
+		.value		= HCR_EL2_TPCP,
+		.mask		= HCR_EL2_TPCP,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TPU] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TPU,
-		.mask		= HCR_TPU,
+		.value		= HCR_EL2_TPU,
+		.mask		= HCR_EL2_TPU,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TTLB] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TTLB,
-		.mask		= HCR_TTLB,
+		.value		= HCR_EL2_TTLB,
+		.mask		= HCR_EL2_TTLB,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TVM] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TVM,
-		.mask		= HCR_TVM,
+		.value		= HCR_EL2_TVM,
+		.mask		= HCR_EL2_TVM,
 		.behaviour	= BEHAVE_FORWARD_WRITE,
 	},
 	[CGT_HCR_TDZ] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TDZ,
-		.mask		= HCR_TDZ,
+		.value		= HCR_EL2_TDZ,
+		.mask		= HCR_EL2_TDZ,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TRVM] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TRVM,
-		.mask		= HCR_TRVM,
+		.value		= HCR_EL2_TRVM,
+		.mask		= HCR_EL2_TRVM,
 		.behaviour	= BEHAVE_FORWARD_READ,
 	},
 	[CGT_HCR_TLOR] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TLOR,
-		.mask		= HCR_TLOR,
+		.value		= HCR_EL2_TLOR,
+		.mask		= HCR_EL2_TLOR,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TERR] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TERR,
-		.mask		= HCR_TERR,
+		.value		= HCR_EL2_TERR,
+		.mask		= HCR_EL2_TERR,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_APK] = {
 		.index		= HCR_EL2,
 		.value		= 0,
-		.mask		= HCR_APK,
+		.mask		= HCR_EL2_APK,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_NV] = {
 		.index		= HCR_EL2,
-		.value		= HCR_NV,
-		.mask		= HCR_NV,
+		.value		= HCR_EL2_NV,
+		.mask		= HCR_EL2_NV,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_NV_nNV2] = {
 		.index		= HCR_EL2,
-		.value		= HCR_NV,
-		.mask		= HCR_NV | HCR_NV2,
+		.value		= HCR_EL2_NV,
+		.mask		= HCR_EL2_NV | HCR_EL2_NV2,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_NV1_nNV2] = {
 		.index		= HCR_EL2,
-		.value		= HCR_NV | HCR_NV1,
-		.mask		= HCR_NV | HCR_NV1 | HCR_NV2,
+		.value		= HCR_EL2_NV | HCR_EL2_NV1,
+		.mask		= HCR_EL2_NV | HCR_EL2_NV1 | HCR_EL2_NV2,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_AT] = {
 		.index		= HCR_EL2,
-		.value		= HCR_AT,
-		.mask		= HCR_AT,
+		.value		= HCR_EL2_AT,
+		.mask		= HCR_EL2_AT,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_nFIEN] = {
 		.index		= HCR_EL2,
 		.value		= 0,
-		.mask		= HCR_FIEN,
+		.mask		= HCR_EL2_FIEN,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TID4] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TID4,
-		.mask		= HCR_TID4,
+		.value 		= HCR_EL2_TID4,
+		.mask		= HCR_EL2_TID4,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TICAB] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TICAB,
-		.mask		= HCR_TICAB,
+		.value 		= HCR_EL2_TICAB,
+		.mask		= HCR_EL2_TICAB,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TOCU] = {
 		.index		= HCR_EL2,
-		.value 		= HCR_TOCU,
-		.mask		= HCR_TOCU,
+		.value 		= HCR_EL2_TOCU,
+		.mask		= HCR_EL2_TOCU,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_ENSCXT] = {
 		.index		= HCR_EL2,
 		.value 		= 0,
-		.mask		= HCR_ENSCXT,
+		.mask		= HCR_EL2_EnSCXT,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TTLBIS] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TTLBIS,
-		.mask		= HCR_TTLBIS,
+		.value		= HCR_EL2_TTLBIS,
+		.mask		= HCR_EL2_TTLBIS,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TTLBOS] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TTLBOS,
-		.mask		= HCR_TTLBOS,
+		.value		= HCR_EL2_TTLBOS,
+		.mask		= HCR_EL2_TTLBOS,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_HCR_TID5] = {
 		.index		= HCR_EL2,
-		.value		= HCR_TID5,
-		.mask		= HCR_TID5,
+		.value		= HCR_EL2_TID5,
+		.mask		= HCR_EL2_TID5,
 		.behaviour	= BEHAVE_FORWARD_RW,
 	},
 	[CGT_MDCR_TPMCR] = {
@@ -528,7 +528,7 @@ static bool is_nested_nv2_guest(struct kvm_vcpu *vcpu)
 	u64 val;
 
 	val = __vcpu_sys_reg(vcpu, HCR_EL2);
-	return ((val & (HCR_E2H | HCR_TGE | HCR_NV2 | HCR_NV1 | HCR_NV)) == (HCR_E2H | HCR_NV2 | HCR_NV));
+	return ((val & (HCR_EL2_E2H | HCR_EL2_TGE | HCR_EL2_NV2 | HCR_EL2_NV1 | HCR_EL2_NV)) == (HCR_EL2_E2H | HCR_EL2_NV2 | HCR_EL2_NV));
 }
 
 static enum trap_behaviour check_cnthctl_el1nvpct(struct kvm_vcpu *vcpu)
@@ -2722,7 +2722,7 @@ static bool forward_hcr_traps(struct kvm_vcpu *vcpu, u64 control_bit)
 
 bool forward_smc_trap(struct kvm_vcpu *vcpu)
 {
-	return forward_hcr_traps(vcpu, HCR_TSC);
+	return forward_hcr_traps(vcpu, HCR_EL2_TSC);
 }
 
 static bool forward_mdcr_traps(struct kvm_vcpu *vcpu, u64 control_bit)
@@ -2944,7 +2944,7 @@ int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
 	 */
 
 	if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
-	    !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
+	    !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_IMO))
 		return 1;
 
 	/* esr_el2 value doesn't matter for exits due to irqs. */
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index 1079136d5cd7..878ea4500c5c 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -755,7 +755,7 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
 			      struct kvm_vcpu_events *events)
 {
 	events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
-	events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) ||
+	events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_EL2_VSE) ||
 					   vcpu_get_flag(vcpu, NESTED_SERROR_PENDING);
 
 	if (events->exception.serror_pending && events->exception.serror_has_esr)
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index db37678dcb05..9931192735d4 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -43,7 +43,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu)
 
 	/* Forward hvc instructions to the virtual EL2 if the guest has EL2. */
 	if (vcpu_has_nv(vcpu)) {
-		if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_HCD)
+		if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_HCD)
 			kvm_inject_undefined(vcpu);
 		else
 			kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index a2a054fec2e2..e0c55644224c 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -403,11 +403,11 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu)
 static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr)
 {
 	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
-		hcr |= HCR_TVM;
+		hcr |= HCR_EL2_TVM;
 
 	write_sysreg_hcr(hcr);
 
-	if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) {
+	if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_EL2_VSE)) {
 		u64 vsesr;
 
 		/*
@@ -454,9 +454,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
 	 * Additionally, when in a nested context we need to propagate the
 	 * updated state to the guest hypervisor's HCR_EL2.
 	 */
-	if (*hcr & HCR_VSE) {
-		*hcr &= ~HCR_VSE;
-		*hcr |= read_sysreg(hcr_el2) & HCR_VSE;
+	if (*hcr & HCR_EL2_VSE) {
+		*hcr &= ~HCR_EL2_VSE;
+		*hcr |= read_sysreg(hcr_el2) & HCR_EL2_VSE;
 	}
 }
 
@@ -655,7 +655,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
 		write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR);
 
 	/* Skip restoring fpexc32 for AArch64 guests */
-	if (!(read_sysreg(hcr_el2) & HCR_RW))
+	if (!(read_sysreg(hcr_el2) & HCR_EL2_RW))
 		write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2);
 
 	*host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED;
@@ -680,7 +680,7 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu)
 	 * The normal sysreg handling code expects to see the traps,
 	 * let's not do anything here.
 	 */
-	if (vcpu->arch.hcr_el2 & HCR_TVM)
+	if (vcpu->arch.hcr_el2 & HCR_EL2_TVM)
 		return false;
 
 	switch (sysreg) {
diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
index 4373e9b63e87..0261b2bc681f 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
@@ -14,13 +14,14 @@
 #include <nvhe/spinlock.h>
 
 /*
- * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest
- * are EL2's own and nothing the host sets there reaches the guest.
+ * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. A protected
+ * VM also takes TWI and TWE, at vCPU load. The rest are EL2's own and nothing
+ * the host sets there reaches the guest.
  */
-#define PKVM_HCR_EL2_HOST_PVM	(HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE)
-#define PKVM_HCR_EL2_HOST_NPVM	(PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF |	\
-				 HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 |		\
-				 HCR_EL2_TID5 | HCR_EL2_TTLBOS)
+#define PKVM_HCR_EL2_HOST_PVM	HCR_EL2_VSE
+#define PKVM_HCR_EL2_HOST_NPVM	(PKVM_HCR_EL2_HOST_PVM | HCR_EL2_TWI | HCR_EL2_TWE |	\
+				 HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_TVM | HCR_EL2_TID2 |	\
+				 HCR_EL2_TID4 | HCR_EL2_TID5 | HCR_EL2_TTLBOS)
 
 /*
  * Holds the relevant data for maintaining the vcpu state completely at hyp.
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 9393fe3ea6a1..50581822ec74 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -123,7 +123,7 @@ SYM_FUNC_START(__hyp_do_panic)
 #ifdef CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC
 	/* Ensure host stage-2 is disabled */
 	mrs	x0, hcr_el2
-	bic	x0, x0, #HCR_VM
+	bic	x0, x0, #HCR_EL2_VM
 	msr_hcr_el2 x0
 	tlbi	vmalls12e1
 	dsb	nsh
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 0b3e0b28dfc7..49127e03cf93 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -102,7 +102,7 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init)
 	ldr	x1, [x0, #NVHE_INIT_HCR_EL2]
 	msr_hcr_el2 x1
 
-	mov	x2, #HCR_E2H
+	mov	x2, #HCR_EL2_E2H
 	and	x2, x1, x2
 	cbz	x2, 1f
 
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 137a8144d09e..179d7c8277b8 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -809,8 +809,8 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason)
 	}
 
 	/* Cleared by hardware once the guest takes the vSError. */
-	host_vcpu->arch.hcr_el2 &= ~HCR_VSE;
-	host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE;
+	host_vcpu->arch.hcr_el2 &= ~HCR_EL2_VSE;
+	host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_EL2_VSE;
 
 	sync_hyp_vgic_state(hyp_vcpu);
 	sync_hyp_timer_state(hyp_vcpu);
@@ -834,8 +834,8 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load,
 		u64 dfr0 = read_sysreg(id_aa64dfr0_el1);
 
 		/* Propagate WFx trapping flags */
-		hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI);
-		hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI);
+		hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_EL2_TWE | HCR_EL2_TWI);
+		hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_EL2_TWE | HCR_EL2_TWI);
 
 		/* HPMN == 0 is reserved without FEAT_HPMN0. */
 		if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0)))
@@ -1615,7 +1615,8 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
 
 	/*
 	 * If pKVM has been initialised then reject any calls to the
-	 * early "privileged" hypercalls. Note that we cannot reject
+	 * early "privileged" hypercalls, and to the ones the host has no
+	 * use for in protected mode. Note that we cannot reject
 	 * calls to __pkvm_prot_finalize for two reasons: (1) The static
 	 * key used to determine initialisation must be toggled prior to
 	 * finalisation and (2) finalisation is performed on a per-CPU
@@ -1737,12 +1738,12 @@ static bool handle_host_mte(u64 esr)
 	case SYS_TFSR_EL1:
 	case SYS_TFSRE0_EL1:
 		/* If we're here for any reason other than MTE, it's a bug. */
-		if (read_sysreg(HCR_EL2) & HCR_ATA)
+		if (read_sysreg(HCR_EL2) & HCR_EL2_ATA)
 			return false;
 		break;
 	case SYS_GMID_EL1:
 		/* If we're here for any reason other than MTE, it's a bug. */
-		if (!(read_sysreg(HCR_EL2) & HCR_TID5))
+		if (!(read_sysreg(HCR_EL2) & HCR_EL2_TID5))
 			return false;
 		break;
 	default:
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index c3035aa5a470..d4c332a91803 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -344,14 +344,14 @@ int __pkvm_prot_finalize(void)
 	struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
 	struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
 
-	if (params->hcr_el2 & HCR_VM)
+	if (params->hcr_el2 & HCR_EL2_VM)
 		return -EPERM;
 
 	params->vttbr = kvm_get_vttbr(mmu);
 	params->vtcr = mmu->vtcr;
-	params->hcr_el2 |= HCR_VM;
+	params->hcr_el2 |= HCR_EL2_VM;
 	if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
-		params->hcr_el2 |= HCR_FWB;
+		params->hcr_el2 |= HCR_EL2_FWB;
 
 	/*
 	 * The CMO below not only cleans the updated params to the
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 5168321713d9..fc0475f0877d 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -38,23 +38,26 @@ unsigned int kvm_host_sve_max_vl;
  */
 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu);
 
-/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */
+/*
+ * The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on
+ * each entry, and a protected vCPU's TWI and TWE at vCPU load.
+ */
 static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
 
 	if (has_hvhe())
-		vcpu->arch.hcr_el2 |= HCR_E2H;
+		vcpu->arch.hcr_el2 |= HCR_EL2_E2H;
 
 	if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) {
 		/* route synchronous external abort exceptions to EL2 */
-		vcpu->arch.hcr_el2 |= HCR_TEA;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TEA;
 		/* trap error record accesses */
-		vcpu->arch.hcr_el2 |= HCR_TERR;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TERR;
 	}
 
 	if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
-		vcpu->arch.hcr_el2 |= HCR_FWB;
+		vcpu->arch.hcr_el2 |= HCR_EL2_FWB;
 
 	/*
 	 * Without AArch32 EL1, leave RW set and let the entry fail with an
@@ -66,7 +69,7 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 		vcpu->arch.hcr_el2 &= ~HCR_EL2_RW;
 
 	if (vcpu_has_ptrauth(vcpu))
-		vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
+		vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK);
 }
 
 static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu)
@@ -75,14 +78,14 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu)
 	u64 val = vcpu->arch.hcr_el2;
 
 	/* No support for AArch32. */
-	val |= HCR_RW;
+	val |= HCR_EL2_RW;
 
 	/*
 	 * Always trap:
 	 * - Feature id registers: to control features exposed to guests
 	 * - Implementation-defined features
 	 */
-	val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1;
+	val |= HCR_EL2_TACR | HCR_EL2_TIDCP | HCR_EL2_TID3 | HCR_EL2_TID1;
 
 	if (cpus_have_final_cap(ARM64_HAS_EVT) &&
 	    !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) &&
@@ -92,20 +95,20 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu)
 		val |= HCR_EL2_TID2;
 
 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
-		val |= HCR_TERR | HCR_TEA;
-		val &= ~(HCR_FIEN);
+		val |= HCR_EL2_TERR | HCR_EL2_TEA;
+		val &= ~(HCR_EL2_FIEN);
 	}
 
 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP))
-		val &= ~(HCR_AMVOFFEN);
+		val &= ~(HCR_EL2_AMVOFFEN);
 
 	if (!kvm_has_mte(kvm)) {
-		val |= HCR_TID5;
-		val &= ~(HCR_DCT | HCR_ATA);
+		val |= HCR_EL2_TID5;
+		val &= ~(HCR_EL2_DCT | HCR_EL2_ATA);
 	}
 
 	if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP))
-		val |= HCR_TLOR;
+		val |= HCR_EL2_TLOR;
 
 	vcpu->arch.hcr_el2 = val;
 }
@@ -392,13 +395,19 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc
 {
 	struct kvm *kvm = &hyp_vm->kvm;
 	unsigned long host_arch_flags = READ_ONCE(host_kvm->arch.flags);
+	u32 vgic_model = READ_ONCE(host_kvm->arch.vgic.vgic_model);
 	DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES);
 
 	/* CTR_EL0 is always under host control, even for protected VMs. */
 	hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0;
 
-	/* Preserve the vgic model so that GICv3 emulation works */
-	hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model;
+	/*
+	 * pKVM runs only GICv3. Anything else becomes 0, the model of a VM
+	 * with no in-kernel irqchip, which keeps EL2 off the GICv5 registers.
+	 */
+	if (vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
+		vgic_model = 0;
+	hyp_vm->kvm.arch.vgic.vgic_model = vgic_model;
 
 	/* No restrictions for non-protected VMs. */
 	if (!kvm_vm_is_protected(kvm)) {
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 91e1edf40bcd..89fb9dd351ad 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -110,7 +110,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
 	write_sysreg(__kvm_hyp_host_vector, vbar_el2);
 }
 
-/* Save VGICv3 state on non-VHE systems */
+/* Save vgic state on non-VHE systems */
 static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
 {
 	if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
@@ -125,7 +125,7 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
 	}
 }
 
-/* Restore VGICv3 state on non-VHE systems */
+/* Restore vgic state on non-VHE systems */
 static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
 {
 	if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) {
diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c
index 74d4a509f1de..2dfb5ff9db57 100644
--- a/arch/arm64/kvm/hyp/vgic-v3-sr.c
+++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c
@@ -464,7 +464,7 @@ bool __vgic_v3_get_gic_config(void)
 	if (has_vhe()) {
 		flags = local_daif_save();
 	} else {
-		sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
+		sysreg_clear_set_hcr(0, HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO);
 		isb();
 	}
 
@@ -479,7 +479,7 @@ bool __vgic_v3_get_gic_config(void)
 	if (has_vhe()) {
 		local_daif_restore(flags);
 	} else {
-		sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
+		sysreg_clear_set_hcr(HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO, 0);
 		isb();
 	}
 
diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 4c965f9b1e1d..3f2390a13173 100644
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -47,7 +47,7 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
  * - FIEN: no way we let a guest have access to the RAS "Common Fault
  *   Injection" thing, whatever that does
  */
-#define NV_HCR_GUEST_EXCLUDE	(HCR_TGE | HCR_API | HCR_APK | HCR_FIEN)
+#define NV_HCR_GUEST_EXCLUDE	(HCR_EL2_TGE | HCR_EL2_API | HCR_EL2_APK | HCR_EL2_FIEN)
 
 static u64 __compute_hcr(struct kvm_vcpu *vcpu)
 {
@@ -65,10 +65,10 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
 	if (is_hyp_ctxt(vcpu)) {
 		host_data_set_flag(VCPU_IN_HYP_CONTEXT);
 
-		hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB;
+		hcr |= HCR_EL2_NV | HCR_EL2_NV2 | HCR_EL2_AT | HCR_EL2_TTLB;
 
 		if (!vcpu_el2_e2h_is_set(vcpu))
-			hcr |= HCR_NV1;
+			hcr |= HCR_EL2_NV1;
 
 		/* Publish the guest's view of HCR_EL2 to the HW */
 		if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu))
@@ -91,7 +91,7 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
 		host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
 
 		guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
-		if (guest_hcr & HCR_NV) {
+		if (guest_hcr & HCR_EL2_NV) {
 			u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));
 
 			/* Inherit the low bits from the actual register */
@@ -99,14 +99,14 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
 			write_sysreg_s(va, SYS_VNCR_EL2);
 
 			/* Force NV2 in case the guest is forgetful... */
-			guest_hcr |= HCR_NV2;
+			guest_hcr |= HCR_EL2_NV2;
 		}
 
 		/*
 		 * Exclude the guest's TWED configuration if it hasn't set TWE
 		 * to avoid potentially delaying traps for the host.
 		 */
-		if (!(guest_hcr & HCR_TWE))
+		if (!(guest_hcr & HCR_EL2_TWE))
 			guest_hcr &= ~(HCR_EL2_TWEDEn | HCR_EL2_TWEDEL);
 	}
 
diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c
index c386d9f1c101..33ac9a8c954e 100644
--- a/arch/arm64/kvm/hyp/vhe/tlb.c
+++ b/arch/arm64/kvm/hyp/vhe/tlb.c
@@ -62,7 +62,7 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu,
 	 */
 	__load_stage2(mmu);
 	val = read_sysreg(hcr_el2);
-	val &= ~HCR_TGE;
+	val &= ~HCR_EL2_TGE;
 	write_sysreg_hcr(val);
 	isb();
 }
diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c
index f4b932b7bf12..c00b0044ef11 100644
--- a/arch/arm64/kvm/hypercalls.c
+++ b/arch/arm64/kvm/hypercalls.c
@@ -198,8 +198,10 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user
 
 	if (!kvm_smccc_filter_configured(kvm)) {
 		r = kvm_smccc_filter_insert_reserved(kvm);
-		if (WARN_ON_ONCE(r))
+		if (r) {
+			WARN_ON_ONCE(r != -ENOMEM);
 			goto out_unlock;
+		}
 	}
 
 	r = mtree_insert_range(&kvm->arch.smccc_filter, start, end,
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 96bf926ca520..aa2011cc0d5a 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -228,7 +228,7 @@ static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
 
 static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
 {
-	if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA))
+	if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_TGE | HCR_EL2_TEA))
 		return true;
 
 	if (!vcpu_mode_priv(vcpu))
@@ -272,7 +272,7 @@ int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr)
 {
 	u64 esr;
 
-	if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM))
+	if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM))
 		return kvm_inject_nested_excl_atomic(vcpu, addr);
 
 	__kvm_inject_sea(vcpu, false, addr);
@@ -387,6 +387,6 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
 	}
 
 	vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
-	*vcpu_hcr(vcpu) |= HCR_VSE;
+	*vcpu_hcr(vcpu) |= HCR_EL2_VSE;
 	return 1;
 }
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 5b9ff769729e..1db10d8d6db7 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1781,6 +1781,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
 						      s2vi.vma_pagesize, write_fault, exec_fault,
 						      false);
 		kfree(mapping);
+		return ret;
 	}
 
 	if (!(s2fd->memslot->flags & KVM_MEM_READONLY))
@@ -2952,18 +2953,18 @@ void kvm_set_way_flush(struct kvm_vcpu *vcpu)
 
 	/*
 	 * If this is the first time we do a S/W operation
-	 * (i.e. HCR_TVM not set) flush the whole memory, and set the
+	 * (i.e. HCR_EL2_TVM not set) flush the whole memory, and set the
 	 * VM trapping.
 	 *
 	 * Otherwise, rely on the VM trapping to wait for the MMU +
 	 * Caches to be turned off. At that point, we'll be able to
 	 * clean the caches again.
 	 */
-	if (!(hcr & HCR_TVM)) {
+	if (!(hcr & HCR_EL2_TVM)) {
 		trace_kvm_set_way_flush(*vcpu_pc(vcpu),
 					vcpu_has_cache_enabled(vcpu));
 		stage2_flush_vm(vcpu->kvm);
-		*vcpu_hcr(vcpu) = hcr | HCR_TVM;
+		*vcpu_hcr(vcpu) = hcr | HCR_EL2_TVM;
 	}
 }
 
@@ -2981,7 +2982,7 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
 
 	/* Caches are now on, stop trapping VM ops (until a S/W op) */
 	if (now_enabled)
-		*vcpu_hcr(vcpu) &= ~HCR_TVM;
+		*vcpu_hcr(vcpu) &= ~HCR_EL2_TVM;
 
 	trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
 }
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index f1c414f1c3fa..4d83281a7a28 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -777,7 +777,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
 	vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
 	hcr = vcpu_read_sys_reg(vcpu, HCR_EL2);
 
-	nested_stage2_enabled = hcr & HCR_VM;
+	nested_stage2_enabled = hcr & HCR_EL2_VM;
 
 	/* Don't consider the CnP bit for the vttbr match */
 	vttbr &= ~VTTBR_CNP_BIT;
@@ -858,7 +858,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
 	 */
 	s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT;
 	s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
-	s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
+	s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM;
 
 out:
 	atomic_inc(&s2_mmu->refcnt);
@@ -979,7 +979,7 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
 				vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
 		}
 
-		if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
+		if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV)
 			kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
 	}
 }
@@ -2181,6 +2181,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
  *
  * What follows is some slop to (1) react to exception routing / masking and (2)
  * preserve the pending SError state across translation regimes.
+ *
+ * Also generate the maintenance interrupt state, and finally check that if
+ * we need to take an interrupt targetting the guest hypervisor.
  */
 void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
 {
@@ -2189,6 +2192,11 @@ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
 
 	if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
 		kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
+
+	vgic_v3_nested_update_mi(vcpu);
+
+	if (vgic_state_is_nested(vcpu) && kvm_vgic_vcpu_pending_irq(vcpu))
+		kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);
 }
 
 void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
@@ -2202,7 +2210,7 @@ void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
 	 * We previously decided that an SError was deliverable to the guest.
 	 * Reap the pending state from HCR_EL2 and...
 	 */
-	if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
+	if (unlikely(__test_and_clear_bit(__ffs(HCR_EL2_VSE), hcr)))
 		vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
 
 	/*
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 380dfb1db0be..016997288fd2 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -333,7 +333,7 @@ static int __init pkvm_drop_host_privileges(void)
 	return ret;
 }
 
-void __init pkvm_selftests(void)
+static void __init pkvm_selftests(void)
 {
 #ifdef CONFIG_NVHE_EL2_DEBUG
 	int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest);
@@ -387,6 +387,8 @@ static int __init finalize_pkvm(void)
 	if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised())
 		return 0;
 
+	pkvm_selftests();
+
 	/*
 	 * Exclude HYP sections from kmemleak so that they don't get peeked
 	 * at, which would end badly once inaccessible.
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index edfc3be1d4b3..850eba167aad 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -586,7 +586,7 @@ static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
 }
 
 /*
- * Generic accessor for VM registers. Only called as long as HCR_TVM
+ * Generic accessor for VM registers. Only called as long as HCR_EL2_TVM
  * is set. If the guest enables the MMU, we stop trapping the VM
  * sys_regs and leave it in complete control of the caches.
  */
@@ -2893,7 +2893,7 @@ static u64 reset_hcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 	u64 val = r->val;
 
 	if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1))
-		val |= HCR_E2H;
+		val |= HCR_EL2_E2H;
 
 	__vcpu_assign_sys_reg(vcpu, r->reg, val);
 
@@ -5798,31 +5798,31 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu)
 	struct kvm *kvm = vcpu->kvm;
 
 	if (has_vhe() || has_hvhe())
-		vcpu->arch.hcr_el2 |= HCR_E2H;
+		vcpu->arch.hcr_el2 |= HCR_EL2_E2H;
 	if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) {
 		/* route synchronous external abort exceptions to EL2 */
-		vcpu->arch.hcr_el2 |= HCR_TEA;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TEA;
 		/* trap error record accesses */
-		vcpu->arch.hcr_el2 |= HCR_TERR;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TERR;
 	}
 
 	if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
-		vcpu->arch.hcr_el2 |= HCR_FWB;
+		vcpu->arch.hcr_el2 |= HCR_EL2_FWB;
 
 	if (cpus_have_final_cap(ARM64_HAS_EVT) &&
 	    !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) &&
 	    kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_sanitised_ftr_reg(SYS_CTR_EL0))
-		vcpu->arch.hcr_el2 |= HCR_TID4;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TID4;
 	else
-		vcpu->arch.hcr_el2 |= HCR_TID2;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TID2;
 
 	if (vcpu_el1_is_32bit(vcpu))
-		vcpu->arch.hcr_el2 &= ~HCR_RW;
+		vcpu->arch.hcr_el2 &= ~HCR_EL2_RW;
 
 	if (kvm_has_mte(vcpu->kvm))
-		vcpu->arch.hcr_el2 |= HCR_ATA;
+		vcpu->arch.hcr_el2 |= HCR_EL2_ATA;
 	else
-		vcpu->arch.hcr_el2 |= HCR_TID5;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TID5;
 
 	/*
 	 * In the absence of FGT, we cannot independently trap TLBI
@@ -5830,7 +5830,7 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu)
 	 * TLBIs would be far worse. Live with it...
 	 */
 	if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
-		vcpu->arch.hcr_el2 |= HCR_TTLBOS;
+		vcpu->arch.hcr_el2 |= HCR_EL2_TTLBOS;
 }
 
 void kvm_calculate_traps(struct kvm_vcpu *vcpu)
@@ -5874,6 +5874,8 @@ static int kvm_vm_finalize_sys_regs(struct kvm *kvm)
 	if (kvm_id_regs_final(kvm))
 		return 0;
 
+	kvm_idregs_enforce_dependencies(kvm);
+
 	/*
 	 * This hacks into the ID registers, so only perform it when the
 	 * first vcpu runs, or the kvm_set_vm_id_reg() helper will scream.
diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c
index 979301c3dbd8..7050e274b742 100644
--- a/arch/arm64/kvm/vgic/vgic-init.c
+++ b/arch/arm64/kvm/vgic/vgic-init.c
@@ -121,6 +121,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 		goto out_unlock;
 	}
 
+	if (kvm->arch.vgic.its_model && kvm->arch.vgic.its_model != type) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
 	if (kvm_id_regs_final(kvm))
 		goto out_unlock;
 	ret = 0;
@@ -433,6 +438,8 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 		return ret;
 
 	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		break;
 	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		/*
 		 * If we are creating a VCPU with a GICv3 we must also register
@@ -451,6 +458,9 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 		 */
 		if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus)
 			return -EINVAL;
+		break;
+	default:
+		unreachable();
 	}
 
 	return ret;
@@ -509,7 +519,7 @@ int vgic_init(struct kvm *kvm)
 		if (vgic_supports_direct_irqs(kvm)) {
 			ret = vgic_v4_init(kvm);
 			if (ret)
-				return ret;
+				goto out_free_spis;
 		}
 	} else {
 		if (!dist->nr_spis)
@@ -524,17 +534,24 @@ int vgic_init(struct kvm *kvm)
 		kvm_vgic_vcpu_reset(vcpu);
 
 	ret = kvm_vgic_setup_default_irq_routing(kvm);
-	if (ret) {
-		if (vgic_is_v5(kvm))
-			vgic_v5_teardown(kvm);
-
-		return ret;
-	}
+	if (ret)
+		goto out_teardown;
 
 	vgic_debug_init(kvm);
 	dist->initialized = true;
 
 	return 0;
+
+out_teardown:
+	if (vgic_is_v5(kvm))
+		vgic_v5_teardown(kvm);
+	else
+		vgic_v4_teardown(kvm);
+out_free_spis:
+	kfree(dist->spis);
+	dist->spis = NULL;
+
+	return ret;
 }
 
 static void kvm_vgic_dist_destroy(struct kvm *kvm)
@@ -550,6 +567,10 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
 	dist->nr_spis = 0;
 	dist->vgic_dist_base = VGIC_ADDR_UNDEF;
 
+	/* A VM may be destroyed without ever creating a VGIC. */
+	if (!dist->vgic_model)
+		goto out;
+
 	switch (dist->vgic_model) {
 	case KVM_DEV_TYPE_ARM_VGIC_V2:
 		dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
@@ -567,8 +588,11 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
 		kfree(dist->vgic_v5_irs_data);
 		dist->vgic_v5_irs_data = NULL;
 		break;
+	default:
+		unreachable();
 	}
 
+out:
 	xa_destroy(&dist->lpi_xa);
 }
 
diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c
index 1874c3c289a6..8d0ad74ebc64 100644
--- a/arch/arm64/kvm/vgic/vgic-irs-v5.c
+++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c
@@ -1134,7 +1134,7 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
 	const struct vgic_register_region *region;
 	struct vgic_reg_attr reg_attr;
 	struct kvm_vcpu *vcpu;
-	gpa_t addr, offset;
+	gpa_t offset;
 	int ret, align;
 
 	ret = vgic_v5_parse_attr(dev, attr, &reg_attr);
@@ -1142,7 +1142,6 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
 		return ret;
 
 	vcpu = reg_attr.vcpu;
-	addr = reg_attr.addr;
 
 	if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS)
 		return vgic_v5_has_cpu_sysregs_attr(vcpu, attr);
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index e5657cb60356..7e9b265b1f52 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -776,6 +776,7 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 				its_unmap_vlpi(ite->irq->host_irq);
 
 			irq->hw = false;
+			irq->mapped = false;
 		}
 
 		vgic_put_irq(kvm, ite->irq);
@@ -784,6 +785,23 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 	kfree(ite);
 }
 
+/* Mapping one LPI from two events is UNPREDICTABLE: the first ITE keeps it. */
+static bool vgic_its_map_lpi(struct kvm *kvm, struct its_ite *ite,
+			     struct vgic_irq *irq, struct kvm_vcpu *vcpu)
+{
+	scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+		if (!irq->mapped) {
+			irq->mapped = true;
+			irq->target_vcpu = vcpu;
+			ite->irq = irq;
+			return true;
+		}
+	}
+
+	vgic_put_irq(kvm, irq);
+	return false;
+}
+
 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 {
 	return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
@@ -815,12 +833,35 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 		struct its_device *device = find_its_device(its, device_id);
 		int ite_esz = vgic_its_get_abi(its)->ite_esz;
 		gpa_t gpa = device->itt_addr + ite->event_id * ite_esz;
+		struct vgic_irq *irq = ite->irq;
+		struct kvm_vcpu *vcpu, *target;
+
+		vgic_its_invalidate_cache(its);
+
 		/*
-		 * Though the spec talks about removing the pending state, we
-		 * don't bother here since we clear the ITTE anyway and the
-		 * pending state is a property of the ITTE struct.
+		 * Only an LPI on a vCPU's AP list can be in its list registers,
+		 * which the exit path folds back, and that exit moves the LPI
+		 * to its target vCPU if the target has changed: pause both.
 		 */
-		vgic_its_invalidate_cache(its);
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			vcpu = irq->vcpu;
+			target = irq->target_vcpu;
+		}
+
+		if (vcpu) {
+			if (target != vcpu)
+				kvm_pause_vcpu(target);
+			kvm_pause_vcpu(vcpu);
+		}
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			irq->pending_latch = false;
+
+		if (vcpu) {
+			kvm_resume_vcpu(vcpu);
+			if (target != vcpu)
+				kvm_resume_vcpu(target);
+		}
 
 		its_free_ite(kvm, ite);
 
@@ -1093,13 +1134,14 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
 		vcpu = collection_to_vcpu(kvm, collection);
 
 	irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
+	if (!IS_ERR(irq) && !vgic_its_map_lpi(kvm, ite, irq, vcpu))
+		irq = ERR_PTR(-EEXIST);
 	if (IS_ERR(irq)) {
 		if (new_coll)
 			vgic_its_free_collection(its, coll_id);
 		its_free_ite(kvm, ite);
 		return PTR_ERR(irq);
 	}
-	ite->irq = irq;
 
 	return 0;
 }
@@ -1382,6 +1424,15 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
 	if (vcpu1 == vcpu2)
 		return 0;
 
+	/*
+	 * Bulk operations such as MOVALL are a pain, as they can clash
+	 * badly locking-wise with other vcpus entering and exiting the
+	 * guest, should they be affected by it. Stopping the guest is a
+	 * safer bet to ensure uncontended access and ultimately forward
+	 * progress. Yeah...
+	 */
+	kvm_arm_halt_guest(kvm);
+
 	xa_for_each(&dist->lpi_xa, intid, irq) {
 		irq = vgic_get_irq(kvm, intid);
 		if (!irq)
@@ -1394,6 +1445,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
 
 	vgic_its_invalidate_cache(its);
 
+	kvm_arm_resume_guest(kvm);
+
 	return 0;
 }
 
@@ -1875,6 +1928,19 @@ static int vgic_its_create(struct kvm_device *dev, u32 type)
 
 	mutex_lock(&dev->kvm->arch.config_lock);
 
+	/*
+	 * Reject creation of a GICv3 ITS if:
+	 * - The VM already has a VGIC other than GICv3.
+	 * - An existing ITS requires a VGIC model other than GICv3.
+	 */
+	if ((irqchip_in_kernel(dev->kvm) && !vgic_is_v3(dev->kvm)) ||
+	    (dev->kvm->arch.vgic.its_model &&
+	     dev->kvm->arch.vgic.its_model != KVM_DEV_TYPE_ARM_VGIC_V3)) {
+		mutex_unlock(&dev->kvm->arch.config_lock);
+		kfree(its);
+		return -ENODEV;
+	}
+
 	if (vgic_initialized(dev->kvm)) {
 		ret = vgic_v4_init(dev->kvm);
 		if (ret < 0) {
@@ -1902,7 +1968,7 @@ static int vgic_its_create(struct kvm_device *dev, u32 type)
 	xa_init(&its->translation_cache);
 
 	dev->kvm->arch.vgic.msis_require_devid = true;
-	dev->kvm->arch.vgic.has_its = true;
+	dev->kvm->arch.vgic.its_model = KVM_DEV_TYPE_ARM_VGIC_V3;
 	its->enabled = false;
 	its->dev = dev;
 
@@ -2208,7 +2274,10 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
 		its_free_ite(kvm, ite);
 		return PTR_ERR(irq);
 	}
-	ite->irq = irq;
+
+	/* Drop an ITE whose LPI another ITE maps, as MAPTI does. */
+	if (!vgic_its_map_lpi(kvm, ite, irq, vcpu))
+		its_free_ite(kvm, ite);
 
 	return offset;
 }
diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c
index 5ab789accc9d..f9ff553bbbf1 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c
+++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c
@@ -42,10 +42,7 @@ bool vgic_has_its(struct kvm *kvm)
 {
 	struct vgic_dist *dist = &kvm->arch.vgic;
 
-	if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
-		return false;
-
-	return dist->has_its;
+	return dist->its_model && dist->its_model == dist->vgic_model;
 }
 
 bool vgic_supports_direct_msis(struct kvm *kvm)
@@ -303,9 +300,19 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
 		if (ctlr != GICR_CTLR_ENABLE_LPIS)
 			return;
 
+		/*
+		 * Yes, disabling LPIs is painful, since it can be done from
+		 * a *remote* vcpu! So let's not take any chance, and make
+		 * sure that the target vCPU has written their LRs back to the
+		 * irq structures, and release any reference they would have.
+		 *
+		 * If it hurts, don't do it.
+		 */
+		kvm_pause_vcpu(vcpu);
 		vgic_flush_pending_lpis(vcpu);
 		vgic_its_invalidate_all_caches(vcpu->kvm);
 		atomic_set_release(&vgic_cpu->ctlr, 0);
+		kvm_resume_vcpu(vcpu);
 	} else {
 		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
 					      GICR_CTLR_ENABLE_LPIS);
diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c
index e3e84d7f5ad5..82bfc4d4f122 100644
--- a/arch/arm64/kvm/vgic/vgic-v3-nested.c
+++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c
@@ -120,8 +120,8 @@ bool vgic_state_is_nested(struct kvm_vcpu *vcpu)
 	u64 xmo;
 
 	if (is_nested_ctxt(vcpu)) {
-		xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO);
-		WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO),
+		xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_IMO | HCR_EL2_FMO);
+		WARN_ONCE(xmo && xmo != (HCR_EL2_IMO | HCR_EL2_FMO),
 			  "Separate virtual IRQ/FIQ settings not supported\n");
 
 		return !!xmo;
diff --git a/arch/arm64/kvm/vgic/vgic-v4.c b/arch/arm64/kvm/vgic/vgic-v4.c
index ed236f083f0d..30e1de3fc7d9 100644
--- a/arch/arm64/kvm/vgic/vgic-v4.c
+++ b/arch/arm64/kvm/vgic/vgic-v4.c
@@ -333,6 +333,9 @@ void vgic_v4_teardown(struct kvm *kvm)
 		free_irq(irq, vcpu);
 	}
 
+	/* Make sure we free all VM-wide, per-CPU resources */
+	its_vm->nr_vpes = atomic_read(&kvm->online_vcpus);
+
 	its_free_vcpu_irqs(its_vm);
 	kfree(its_vm->vpes);
 	its_vm->nr_vpes = 0;
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index c2726d4c06e6..bea848c93a99 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -1071,11 +1071,10 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 static inline bool can_access_vgic_from_kernel(void)
 {
 	/*
-	 * GICv2 can always be accessed from the kernel because it is
-	 * memory-mapped, and VHE systems can access GICv3 EL2 system
-	 * registers.
+	 * GICv2 is memory-mapped. Every other CPU interface uses EL2 system
+	 * registers, which only VHE can access from the kernel.
 	 */
-	return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe();
+	return has_vhe() || kvm_vgic_global_state.type == VGIC_V2;
 }
 
 static inline void vgic_save_state(struct kvm_vcpu *vcpu)
@@ -1167,7 +1166,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 	 *
 	 *   We do that by placing a request to ourselves which will
 	 *   abort the entry procedure and inject the exception at the
-	 *   beginning of the run loop.
+	 *   beginning of the run loop (see kvm_nested_flush_hwstate()).
 	 *
 	 * - Otherwise, do exactly *NOTHING* apart from enabling the virtual
 	 *   CPU interface. The guest state is already loaded, and we can
@@ -1177,16 +1176,10 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 	 * maintenance interrupt state, as it may fire.
 	 */
 	if (vgic_state_is_nested(vcpu)) {
-		if (kvm_vgic_vcpu_pending_irq(vcpu))
-			kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);
-
 		vgic_v3_flush_nested(vcpu);
 		return;
 	}
 
-	if (vcpu_has_nv(vcpu))
-		vgic_v3_nested_update_mi(vcpu);
-
 	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
 	vgic_flush_state(vcpu);
diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c
index 8455b4a5fbb0..f707a3cb281a 100644
--- a/drivers/irqchip/irq-gic-v4.c
+++ b/drivers/irqchip/irq-gic-v4.c
@@ -147,14 +147,20 @@ static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
 	return 0;
 
 err:
-	if (vpe->sgi_domain)
+	if (vpe->sgi_domain) {
 		irq_domain_remove(vpe->sgi_domain);
-	if (vpe->fwnode)
+		vpe->sgi_domain = NULL;
+	}
+	if (vpe->fwnode) {
 		irq_domain_free_fwnode(vpe->fwnode);
+		vpe->fwnode = NULL;
+	}
 	kfree(name);
 	return -ENOMEM;
 }
 
+static void its_free_sgi_irqs(struct its_vm *vm);
+
 int its_alloc_vcpu_irqs(struct its_vm *vm)
 {
 	int vpe_base_irq, i;
@@ -185,16 +191,23 @@ int its_alloc_vcpu_irqs(struct its_vm *vm)
 		vm->vpes[i]->irq = vpe_base_irq + i;
 		ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
 		if (ret)
-			goto err;
+			goto err_free_irqs;
 	}
 
 	return 0;
 
+err_free_irqs:
+	its_free_sgi_irqs(vm);
+	irq_domain_free_irqs(vpe_base_irq, vm->nr_vpes);
 err:
-	if (vm->domain)
+	if (vm->domain) {
 		irq_domain_remove(vm->domain);
-	if (vm->fwnode)
+		vm->domain = NULL;
+	}
+	if (vm->fwnode) {
 		irq_domain_free_fwnode(vm->fwnode);
+		vm->fwnode = NULL;
+	}
 
 	return -ENOMEM;
 }
@@ -207,14 +220,21 @@ static void its_free_sgi_irqs(struct its_vm *vm)
 		return;
 
 	for (i = 0; i < vm->nr_vpes; i++) {
-		unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
+		unsigned int irq;
+
+		/* irq_find_mapping() falls back to the default domain on NULL. */
+		if (!vm->vpes[i]->sgi_domain)
+			continue;
 
+		irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
 		if (WARN_ON(!irq))
 			continue;
 
 		irq_domain_free_irqs(irq, 16);
 		irq_domain_remove(vm->vpes[i]->sgi_domain);
 		irq_domain_free_fwnode(vm->vpes[i]->fwnode);
+		vm->vpes[i]->sgi_domain = NULL;
+		vm->vpes[i]->fwnode = NULL;
 	}
 }
 
@@ -224,6 +244,8 @@ void its_free_vcpu_irqs(struct its_vm *vm)
 	irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
 	irq_domain_remove(vm->domain);
 	irq_domain_free_fwnode(vm->fwnode);
+	vm->domain = NULL;
+	vm->fwnode = NULL;
 }
 
 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 7be81b936bf1..4b2c41917b77 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -292,6 +292,7 @@ struct vgic_irq {
 	bool active:1;
 	bool hw:1;			/* Tied to HW IRQ */
 	bool on_lr:1;			/* Present in a CPU LR */
+	bool mapped:1;			/* LPIs: mapped by an ITE */
 	refcount_t refcount;		/* Used for LPIs */
 	u32 hwintid;			/* HW INTID number */
 	unsigned int host_irq;		/* linux irq corresponding to hwintid */
@@ -502,7 +503,9 @@ struct vgic_dist {
 	bool			initialized;
 
 	/* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
-	u32			vgic_model;
+	enum kvm_device_type	vgic_model;
+	/* VGIC model required by the VM's ITS devices. */
+	enum kvm_device_type	its_model;
 
 	/* Implementation revision as reported in the GICD_IIDR */
 	u32			implementation_rev;
@@ -547,7 +550,6 @@ struct vgic_dist {
 	struct vgic_io_device	dist_iodev;
 	struct vgic_io_device	cpuif_iodev;
 
-	bool			has_its;
 	bool			table_write_in_progress;
 
 	/*
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index c41397eccee5..35d043b19332 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -1337,6 +1337,7 @@ source "kernel/trace/rv/Kconfig"
 
 config TRACE_REMOTE
 	bool
+	select TRACING
 
 config SIMPLE_RING_BUFFER
 	bool
diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c
index 10849d21c0dd..1d2cc25cb3f4 100644
--- a/tools/testing/selftests/kvm/arm64/set_id_regs.c
+++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c
@@ -815,6 +815,10 @@ static void test_idreg_frozen_after_failed_run(void)
 		SYS_REVIDR_EL1,
 		SYS_AIDR_EL1,
 	};
+	u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE];
+	struct reg_mask_range range = {
+		.addr = (__u64)masks,
+	};
 	struct kvm_vcpu_init init;
 	struct kvm_vcpu *vcpu;
 	struct kvm_vm *vm;
@@ -831,6 +835,7 @@ static void test_idreg_frozen_after_failed_run(void)
 
 	vm = vm_create(1);
 	vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0);
+	vm_ioctl(vm, KVM_ARM_GET_REG_WRITABLE_MASKS, &range);
 	kvm_get_default_vcpu_target(vm, &init);
 	init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3);
 	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
@@ -852,7 +857,9 @@ static void test_idreg_frozen_after_failed_run(void)
 	 */
 	for (int i = 0; i < ARRAY_SIZE(test_regs); i++) {
 		const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits;
-		u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg);
+		u32 reg_id = test_regs[i].reg;
+		u64 reg = KVM_ARM64_SYS_REG(reg_id);
+		u64 mask = masks[encoding_to_range_idx(reg_id)];
 		u64 val = vcpu_get_reg(vcpu, reg);
 
 		for (int j = 0; ftr_bits[j].type != FTR_END; j++) {
@@ -860,6 +867,10 @@ static void test_idreg_frozen_after_failed_run(void)
 			u64 safe = get_safe_value(&ftr_bits[j], ftr);
 			u64 new_val;
 
+			/* Only a field userspace could change can be frozen. */
+			if ((mask & ftr_bits[j].mask) != ftr_bits[j].mask)
+				continue;
+
 			if (safe == ftr)
 				continue;
 
diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c
index d5df742b2300..5bf7d99d2914 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_v5.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c
@@ -2003,8 +2003,40 @@ int test_kvm_device(u32 gic_dev_type)
 	return 0;
 }
 
+static void test_vgic_v3_its_with_vgic_v5(void)
+{
+	struct kvm_vm *vm = vm_create_barebones();
+	int gic_fd, its_fd, ret;
+
+	if (__kvm_test_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS)) {
+		pr_info("No GICv3 ITS support; skipping mixed-model test.\n");
+		kvm_vm_free(vm);
+		return;
+	}
+
+	gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5);
+	ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS);
+	TEST_ASSERT(ret < 0 && errno == ENODEV,
+		    "GICv3 ITS created after GICv5 VGIC: ret=%d errno=%d",
+		    ret, errno);
+	close(gic_fd);
+	kvm_vm_free(vm);
+
+	vm = vm_create_barebones();
+	its_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS);
+	ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5);
+	TEST_ASSERT(ret < 0 && errno == ENODEV,
+		    "GICv5 VGIC created after GICv3 ITS: ret=%d errno=%d",
+		    ret, errno);
+	close(its_fd);
+	kvm_vm_free(vm);
+}
+
 void run_tests(u32 gic_dev_type)
 {
+	pr_info("Test VGICv5 and GICv3 ITS model mismatch\n");
+	test_vgic_v3_its_with_vgic_v5();
+
 	pr_info("Test VGICv5 address attrs\n");
 	test_vgic_v5_addr_attrs();