From e6418104fbc661d5c9b280e5b9f3ff83709cf0c9 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:35:17 +0000 Subject: irqchip/gic-v5: Provide OF IRS config frame attrs to KVM KVM needs to interact with the host IRS in order to, for example, make VMs or VPEs valid. There are two potential approaches here. Either the host irqchip driver can provide an interface, or KVM can interact directly with the host IRS. The latter of these two is chosen as the set of MMIO registers that KVM needs to interact with is orthogonal to the set used by the host irqchip driver (with the exception of some of the read-only IRS_IDRx registers). Pass KVM a pointer to an IRS config frame - the config frame belonging to ANY IRS is fine as long as one IRS's config frame is used consistently - in struct gic_kvm_info. Additionally, include a flag telling KVM whether the IRS is coherent or non-coherent in order to make sure that KVM can do the correct cache state management, if required. Only OF (Device Tree) is supported with this change. ACPI is not. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-3-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- include/linux/irqchip/arm-gic-v5.h | 3 +++ include/linux/irqchip/arm-vgic-info.h | 5 +++++ 2 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 2c2fb39f049c..59a587585431 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -332,6 +332,8 @@ struct gicv5_irs_chip_data { raw_spinlock_t spi_config_lock; }; +#define IRS_FLAGS_NON_COHERENT BIT(0) + static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset, const char *reg_s, u32 mask, u32 *val) @@ -379,6 +381,7 @@ void __init gicv5_free_lpi_domain(void); int gicv5_irs_of_probe(struct device_node *parent); int gicv5_irs_acpi_probe(void); +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void); void gicv5_irs_remove(void); int gicv5_irs_enable(void); void gicv5_irs_its_probe(void); diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index 67d9d960273b..f05370e2debf 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h @@ -38,6 +38,11 @@ struct gic_kvm_info { bool has_v4_1; /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; + /* GICv5 IRS base */ + struct { + void __iomem *base; + bool non_coherent; + } gicv5_irs; }; #ifdef CONFIG_KVM -- cgit v1.2.3 From 5ae1ecf679afefadccea64ff393675aebedd3253 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:36:18 +0000 Subject: KVM: arm64: gic-v5: Define remaining IRS MMIO registers Complete the set of defined IRS MMIO registers in the GICv5 header file. Up until now, the set of defined IRS MMIO registers has been driven by code requirements. However, in order to properly emulate the IRS MMIO interface in KVM, the full set of IRS MMIO registers needs to be added. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-5-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- include/linux/irqchip/arm-gic-v5.h | 239 +++++++++++++++++++++++++++++++++++-- 1 file changed, 230 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 59a587585431..8dc5847a646e 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -63,20 +63,34 @@ #define GICV5_OUTER_SHARE 0b10 #define GICV5_INNER_SHARE 0b11 +#define GICV5_AIDR_COMPONENT_IRS 0b00 +#define GICV5_AIDR_COMPONENT_ITS 0b01 +#define GICV5_AIDR_COMPONENT_IWB 0b10 + +#define GICV5_AIDR_ARCH_MAJ_REV_V5 0 +#define GICV5_AIDR_ARCH_MIN_REV_V0 0 + /* * IRS registers and tables structures */ #define GICV5_IRS_IDR0 0x0000 #define GICV5_IRS_IDR1 0x0004 #define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 #define GICV5_IRS_IDR5 0x0014 #define GICV5_IRS_IDR6 0x0018 #define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 #define GICV5_IRS_CR0 0x0080 #define GICV5_IRS_CR1 0x0084 #define GICV5_IRS_SYNCR 0x00c0 #define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 #define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 #define GICV5_IRS_SPI_CFGR 0x0114 #define GICV5_IRS_SPI_STATUSR 0x0118 #define GICV5_IRS_PE_SELR 0x0140 @@ -86,11 +100,51 @@ #define GICV5_IRS_IST_CFGR 0x0190 #define GICV5_IRS_IST_STATUSR 0x0194 #define GICV5_IRS_MAP_L2_ISTR 0x01c0 - +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_IRSID GENMASK(31, 16) +#define GICV5_IRS_IDR0_SWE BIT(12) +#define GICV5_IRS_IDR0_MPAM BIT(11) +#define GICV5_IRS_IDR0_MEC BIT(10) +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_VIRT_ONE_N BIT(8) +#define GICV5_IRS_IDR0_ONE_N BIT(7) #define GICV5_IRS_IDR0_VIRT BIT(6) +#define GICV5_IRS_IDR0_PA_RANGE GENMASK(5, 2) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) + +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 +#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE 0b01 +#define GICV5_IRS_IDR0_INT_DOM_EL3 0b10 +#define GICV5_IRS_IDR0_INT_DOM_REALM 0b11 #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) #define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16) +#define GICV5_IRS_IDR1_PE_CNT GENMASK(15, 0) #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001 @@ -106,13 +160,30 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) + +#define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) +#define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) +#define GICV5_IRS_IDR3_VMD_SZ GENMASK(4, 1) +#define GICV5_IRS_IDR3_VMD BIT(0) + +#define GICV5_IRS_IDR4_VPE_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR4_VPED_SZ GENMASK(5, 0) + #define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IIDR_PRODUCT_ID GENMASK(31, 20) +#define GICV5_IRS_IIDR_VARIANT GENMASK(19, 16) +#define GICV5_IRS_IIDR_REVISION GENMASK(15, 12) +#define GICV5_IRS_IIDR_IMPLEMENTER GENMASK(11, 0) + +#define GICV5_IRS_AIDR_COMPONENT GENMASK(11, 8) +#define GICV5_IRS_AIDR_ARCHMAJORREV GENMASK(7, 4) +#define GICV5_IRS_AIDR_ARCHMINORREV GENMASK(3, 0) #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -135,21 +206,39 @@ #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) -#define GICV5_IRS_SPI_STATUSR_V BIT(1) -#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_VMR_VIRT BIT_ULL(63) +#define GICV5_IRS_SPI_VMR_VM_ID GENMASK_ULL(15, 0) #define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_DOMAINR_DOMAIN GENMASK(1, 0) + +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE 0b00 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE 0b01 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3 0b10 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM 0b11 + +#define GICV5_IRS_SPI_RESAMPLER_ID GENMASK(23, 0) + #define GICV5_IRS_SPI_CFGR_TM BIT(0) +#define GICV5_IRS_SPI_CFGR_TM_EDGE 0b0 +#define GICV5_IRS_SPI_CFGR_TM_LEVEL 0b1 + +#define GICV5_IRS_SPI_STATUSR_V BIT(1) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) + #define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0) +#define GICV5_IRS_PE_STATUSR_ONLINE BIT(2) #define GICV5_IRS_PE_STATUSR_V BIT(1) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_CR0_DPS BIT(0) -#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL #define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) #define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) @@ -167,15 +256,111 @@ #define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01 #define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10 -#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) -#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) #define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0) +#define GICV5_IRS_VMT_BASER_ADDR GENMASK_ULL(55, 3) +#define GICV5_IRS_VMT_BASER_ADDR_SHIFT 3ULL +#define GICV5_IRS_VMT_BASER_VALID BIT_ULL(0) + +#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL 0b1 +#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR 0b0 + +#define GICV5_IRS_VMT_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_VMT_CFGR_VM_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_VMT_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VPE_SELR_S BIT_ULL(63) +#define GICV5_IRS_VPE_SELR_VPE_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VPE_SELR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VPE_DBR_DBV BIT_ULL(63) +#define GICV5_IRS_VPE_DBR_REQ_DB BIT_ULL(62) +#define GICV5_IRS_VPE_DBR_DBPM GENMASK_ULL(36, 32) +#define GICV5_IRS_VPE_DBR_INTID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_HPPIR_HPPIV BIT_ULL(32) +#define GICV5_IRS_VPE_HPPIR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VPE_HPPIR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_CR0_DPS BIT(0) + +#define GICV5_IRS_VPE_STATUSR_V BIT(1) +#define GICV5_IRS_VPE_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VM_DBR_EN BIT_ULL(63) +#define GICV5_IRS_VM_DBR_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VM_SELR_VM_ID GENMASK(15, 0) + +#define GICV5_IRS_VM_STATUSR_V BIT(1) +#define GICV5_IRS_VM_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VMAP_L2_VMTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VMTR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VMR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VMR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VMR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VISTR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VISTR_TYPE GENMASK_ULL(31, 29) + +#define GICV5_IRS_VMAP_L2_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_L2_VISTR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VMAP_L2_VISTR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VMAP_VPER_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0) +#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62) +#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63) + +#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1) + +#define GICV5_IRS_MEC_IDR_MECIDSIZE GENMASK(3, 0) + +#define GICV5_IRS_MEC_MECID_R_MECID GENMASK(15, 0) + +#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP BIT(24) +#define GICV5_IRS_MPAM_IDR_PMG_MAX GENMASK(23, 16) +#define GICV5_IRS_MPAM_IDR_PARTID_MAX GENMASK(15, 0) + +#define GICV5_IRS_MPAM_PARTID_R_IDLE BIT(31) +#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP GENMASK(25, 24) +#define GICV5_IRS_MPAM_PARTID_R_PMG GENMASK(23, 16) +#define GICV5_IRS_MPAM_PARTID_R_PARTID GENMASK(15, 0) + +#define GICV5_IRS_SWERR_STATUSR_IMP_EC GENMASK_ULL(31, 24) +#define GICV5_IRS_SWERR_STATUSR_EC GENMASK_ULL(23, 16) +#define GICV5_IRS_SWERR_STATUSR_OF BIT_ULL(3) +#define GICV5_IRS_SWERR_STATUSR_S1V BIT_ULL(2) +#define GICV5_IRS_SWERR_STATUSR_S0V BIT_ULL(1) +#define GICV5_IRS_SWERR_STATUSR_V BIT_ULL(0) + +#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL BIT_ULL(63) +#define GICV5_IRS_SWERR_SYNDROMER0_TYPE GENMASK_ULL(62, 60) +#define GICV5_IRS_SWERR_SYNDROMER0_ID GENMASK_ULL(55, 32) +#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SWERR_SYNDROMER1_ADDR GENMASK_ULL(55, 3) + #define GICV5_ISTL1E_VALID BIT_ULL(0) +#define GICV5_IRS_ISTL1E_SIZE 8UL #define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12) +#define GICV5_IRS_SETLPIR 0x0000 +#define GICV5_IRS_SETLPIR_ID GENMASK(23, 0) + /* * ITS registers and tables structures */ @@ -297,6 +482,42 @@ #define GICV5_GSI_IWB_FRAME_ID GENMASK(28, 16) #define GICV5_GSI_IWB_WIRE GENMASK(15, 0) +/* + * CoreSight identification registers - ordered by increasing offset. + */ +#define GICV5_CORESIGHT_DEVARCH 0xffbc +#define GICV5_CORESIGHT_PIDR4 0xffd0 +#define GICV5_CORESIGHT_PIDR5 0xffd4 +#define GICV5_CORESIGHT_PIDR6 0xffd8 +#define GICV5_CORESIGHT_PIDR7 0xffdc +#define GICV5_CORESIGHT_PIDR0 0xffe0 +#define GICV5_CORESIGHT_PIDR1 0xffe4 +#define GICV5_CORESIGHT_PIDR2 0xffe8 +#define GICV5_CORESIGHT_PIDR3 0xffec +#define GICV5_CORESIGHT_CIDR0 0xfff0 +#define GICV5_CORESIGHT_CIDR1 0xfff4 +#define GICV5_CORESIGHT_CIDR2 0xfff8 +#define GICV5_CORESIGHT_CIDR3 0xfffc + +#define GICV5_CORESIGHT_DEVARCH_VAL \ + (FIELD_PREP(GENMASK(31, 21), 0x23b) | \ + BIT(20) | \ + 0x5a19) + +#define GICV5_CORESIGHT_PIDR4_JEP106_CONT 0x04 +#define GICV5_CORESIGHT_PIDR5_RES0 0x00 +#define GICV5_CORESIGHT_PIDR6_RES0 0x00 +#define GICV5_CORESIGHT_PIDR7_RES0 0x00 +#define GICV5_CORESIGHT_PIDR0_PART_0 0x4b +#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1 0xb0 +#define GICV5_CORESIGHT_PIDR2_DES_1 0x0b +#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD 0x00 + +#define GICV5_CORESIGHT_CIDR0_VAL 0x0d +#define GICV5_CORESIGHT_CIDR1_VAL 0xf0 +#define GICV5_CORESIGHT_CIDR2_VAL 0x05 +#define GICV5_CORESIGHT_CIDR3_VAL 0xb1 + /* * Global Data structures and functions */ -- cgit v1.2.3 From b3a40e812322daaca26d40fde681dc3079b4f3b4 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:38:20 +0000 Subject: KVM: arm64: gic-v5: Add VPE doorbell domain VPE doorbells allow the GICv5 hardware to notify KVM when an SPI or LPI can be signalled to a non-resident VPE. This provides the mechanism used to wake blocked vcpus once the hardware determines that the interrupt is eligible to be delivered. Add support for a per-VM VPE doorbell irq domain. The domain is created under the GICv5 LPI domain, with one doorbell allocated per VPE. Store the allocated doorbell base in the VM's GICv5 state so that later patches can request per-vcpu doorbell IRQs and use them for IRS commands and wakeups. Add the per-VPE doorbell state to the GICv5 CPU interface state. The doorbell IRQ number is populated when the IRQs are requested, and the db_fired state is used by later patches once doorbell delivery is wired up. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-9-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 24 +++++-- arch/arm64/kvm/vgic/vgic-v5.c | 142 +++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.h | 1 + include/kvm/arm_vgic.h | 4 ++ include/linux/irqchip/arm-gic-v5.h | 2 + 5 files changed, 166 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..052e680b8b60 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -474,8 +474,12 @@ int vgic_init(struct kvm *kvm) kvm_vgic_vcpu_reset(vcpu); ret = kvm_vgic_setup_default_irq_routing(kvm); - if (ret) + if (ret) { + if (vgic_is_v5(kvm)) + vgic_v5_teardown(kvm); + return ret; + } vgic_debug_init(kvm); dist->initialized = true; @@ -496,16 +500,22 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + dist->vgic_cpu_base = VGIC_ADDR_UNDEF; + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) vgic_v3_free_redist_region(kvm, rdreg); INIT_LIST_HEAD(&dist->rd_regions); - } else { - dist->vgic_cpu_base = VGIC_ADDR_UNDEF; - } - if (vgic_supports_direct_irqs(kvm)) - vgic_v4_teardown(kvm); + if (vgic_supports_direct_irqs(kvm)) + vgic_v4_teardown(kvm); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + vgic_v5_teardown(kvm); + break; + } xa_destroy(&dist->lpi_xa); } diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index e75a66b3920a..60bb39421ad0 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -7,6 +7,7 @@ #include #include +#include #include "vgic.h" @@ -125,6 +126,137 @@ skip_v5: return 0; } +/* + * This set of irq_chip functions is specific for doorbells. + */ +static const struct irq_chip vgic_v5_db_irq_chip = { + .name = "GICv5-DB", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static void vgic_v5_irq_db_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + const struct irq_chip *chip = &vgic_v5_db_irq_chip; + struct vgic_v5_vm *vm = arg; + struct irq_data *irqd; + int ret; + + if (!vm) { + kvm_err("invalid parameter for doorbell irq allocation\n"); + return -EINVAL; + } + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); + if (ret) + return ret; + + for (int i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip, + domain->host_data); + irqd = irq_desc_get_irq_data(irq_to_desc(virq + i)); + irqd_set_single_target(irqd); + } + + return 0; +} + +static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = { + .alloc = vgic_v5_irq_db_domain_alloc, + .free = vgic_v5_irq_db_domain_free, +}; + +static int vgic_v5_create_per_vm_domain(struct kvm *kvm) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + int nr_vcpus = atomic_read(&kvm->online_vcpus); + int id = task_pid_nr(current); + int ret, db_virq = 0; + + if (!gicv5_global_data.lpi_domain) { + kvm_err("LPI domain uninitialized, can't set up KVM Doorbells\n"); + return -ENODEV; + } + + vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id); + if (!vm->fwnode) + return -ENOMEM; + + /* + * KVM per-VM VPE DB domain; child of LPI domain; only ever handles + * doorbells. We know how many doorbells we have, and therefore we + * create a linear domain. + */ + vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, + 0, nr_vcpus, vm->fwnode, + &vgic_v5_irq_db_domain_ops, vm); + if (!vm->domain) { + ret = -ENOMEM; + goto err; + } + + db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm); + if (db_virq <= 0) { + ret = db_virq; + goto err; + } + + kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq; + + return 0; + +err: + if (vm->domain) + irq_domain_remove(vm->domain); + if (vm->fwnode) + irq_domain_free_fwnode(vm->fwnode); + + kvm->arch.vgic.gicv5_vm.vpe_db_base = 0; + vm->domain = NULL; + vm->fwnode = NULL; + + return ret; +} + +static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm) +{ + if (!vm->domain) + return; + + if (vm->vpe_db_base) { + irq_domain_free_irqs(vm->vpe_db_base, vm->domain->revmap_size); + vm->vpe_db_base = 0; + } + + irq_domain_remove(vm->domain); + irq_domain_free_fwnode(vm->fwnode); + vm->domain = NULL; + vm->fwnode = NULL; +} + void vgic_v5_reset(struct kvm_vcpu *vcpu) { /* @@ -140,10 +272,16 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu) vcpu->arch.vgic_cpu.num_pri_bits = 5; } +void vgic_v5_teardown(struct kvm *kvm) +{ + vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm); +} + int vgic_v5_init(struct kvm *kvm) { struct kvm_vcpu *vcpu; unsigned long idx; + int ret; if (vgic_initialized(kvm)) return 0; @@ -155,6 +293,10 @@ int vgic_v5_init(struct kvm *kvm) } } + ret = vgic_v5_create_per_vm_domain(kvm); + if (ret) + return ret; + /* We only allow userspace to drive the SW_PPI, if it is implemented. */ bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis, VGIC_V5_NR_PRIVATE_IRQS); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b71d486ae514..42cb205554da 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -383,6 +383,7 @@ void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); void vgic_v5_reset(struct kvm_vcpu *vcpu); int vgic_v5_init(struct kvm *kvm); +void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index c82aa7c9e3f3..78a442046ebc 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -379,6 +379,10 @@ struct vgic_v5_vm { * convenient way to do that). */ DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + + struct fwnode_handle *fwnode; + struct irq_domain *domain; + int vpe_db_base; }; struct vgic_dist { diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 8dc5847a646e..6ed886c09bec 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -615,6 +615,8 @@ void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ struct gicv5_vpe { + int db; + bool db_fired; bool resident; }; -- cgit v1.2.3 From f45438c015291f26e99f7fe355c4c191b5750f8d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:38:51 +0000 Subject: KVM: arm64: gic-v5: Create and manage VM and VPE tables GICv5 uses a set of in-memory tables to track and manage VM state. These must be allocated by the hypervisor and provided to the IRS. The VMT (Virtual Machine Table) is a linear or two-level table comprising VMT Entries (VMTEs). Each VMTE describes the state for a single VM. This state includes things such as the SPI and LPI IST configuration (coming in a future commit), an implementation-defined VM Descriptor, and a VPE Table (VPET). The VPET contains one entry per possible VPE ID belonging to a VM. It is used to mark a VPE as valid and provide the address of an implementation-defined VPE Descriptor (VPED), which the hardware uses to track and manage VPE state. Allocate each VM's VPEDs as a single dense array indexed by vcpu_idx, while the VPET remains indexed by the userspace-provided vcpu_id. This keeps VPED storage proportional to the number of vCPUs even when their IDs are sparse. The VMT and VPET are shared with the IRS. On systems with a non-coherent IRS, cache maintenance operates at cache-line granularity, while multiple entries can occupy the same cache line. Use a common lock for CPU accesses to these tables and IRS command processing so that writing back one entry cannot overwrite an IRS update to a neighbouring entry. The implementation-defined VMD and VPED storage is also visible to the IRS. Round these allocations up to whole cache lines to prevent cache maintenance from corrupting unrelated slab objects. Initialise the storage before publishing its addresses to the IRS. This commit adds support for allocating the VMT and its descriptor backing state, and for managing VMTEs. The VMTEs can be initialised or released for reuse. VM IDs are allocated with an IDA, while an XArray tracks the host-side allocations associated with populated VMTEs. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-10-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/Makefile | 2 +- arch/arm64/kvm/vgic/vgic-init.c | 2 + arch/arm64/kvm/vgic/vgic-v5-tables.c | 681 +++++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.h | 100 +++++ arch/arm64/kvm/vgic/vgic-v5.c | 19 + drivers/irqchip/irq-gic-v5-irs.c | 12 +- include/kvm/arm_vgic.h | 4 + include/linux/irqchip/arm-gic-v5.h | 14 +- 8 files changed, 824 insertions(+), 10 deletions(-) create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.c create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.h (limited to 'include/linux') diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 59612d2f277c..431de9b145ca 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,7 +24,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 052e680b8b60..d5e05d979407 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -154,6 +154,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) case KVM_DEV_TYPE_ARM_VGIC_V3: INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions); break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; } /* diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c new file mode 100644 index 000000000000..7252d48431a5 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vgic.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_v5_vmt *vmt_info; + +/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */ +DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock); + +/* Serialises lazy installation of shared second-level VMTs. */ +static DEFINE_MUTEX(vmt_l2_lock); + +static DEFINE_XARRAY(vm_info); + +/* Level 1 Virtual Machine Table Entry */ +#define GICV5_VMTEL1E_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design */ +#define GICV5_VMTEL1E_L2_ADDR GENMASK(51, 12) + +#define GICV5_VMTEL2E_SIZE 32ULL +/* An L2 table (two-level VMT) is ALWAYS 4kB! */ +#define GICV5_VMT_L2_TABLE_SIZE 4096ULL +#define GICV5_VMT_L2_TABLE_ENTRIES (GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE) + +/* + * As the L2 VMTE is a large data structure, we are splitting it into 4 parts. + * We only mask and shift WITHIN each part for simplicity. + */ +/* First 64-bit chunk */ +#define GICV5_VMTEL2E_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_VMD_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VMD_ADDR GENMASK_ULL(55, 3) +/* Second 64-bit chunk */ +#define GICV5_VMTEL2E_VPET_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VPET_ADDR GENMASK_ULL(55, 3) +#define GICV5_VMTEL2E_VPE_ID_BITS GENMASK_ULL(63, 59) +/* Third & fourth 64-bit chunks (the encodings are the same for each) */ +#define GICV5_VMTEL2E_IST_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_IST_L2SZ GENMASK_ULL(2, 1) +#define GICV5_VMTEL2E_IST_ADDR_SHIFT 6ULL +#define GICV5_VMTEL2E_IST_ADDR GENMASK_ULL(55, 6) +#define GICV5_VMTEL2E_IST_ISTSZ GENMASK_ULL(57, 56) +#define GICV5_VMTEL2E_IST_STRUCTURE BIT_ULL(58) +#define GICV5_VMTEL2E_IST_ID_BITS GENMASK_ULL(63, 59) + +/* Virtual PE Table Entry */ +#define GICV5_VPE_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design. */ +#define GICV5_VPED_ADDR_SHIFT 3ULL +#define GICV5_VPED_ADDR GENMASK_ULL(55, 3) + +/* + * Our IRS might be coherent or non-coherent. If coherent, we can just emit a + * DSB to ensure that we're in sync. However, when non-coherent, we need to + * manage our cached data explicitly. + * + * This helper is used to handle both coherent and non-coherent IRSes, and + * handles all combinations of cleaning and invalidating to the PoC. + */ +static void vgic_v5_clean_inval(void *va, size_t size) +{ + unsigned long base = (unsigned long)va; + + dsb(ishst); + + if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent) + dcache_clean_inval_poc(base, base + size); +} + +/* + * Create a linear VM Table. Directly using the number of entries supplied as + * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per + * the GICv5 requirements for the IRS_VMT_BASER. + */ +static int vgic_v5_alloc_vmt_linear(unsigned int num_entries) +{ + vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base, + num_entries); + if (!vmt_info->linear.vmt_base) + return -ENOMEM; + + vgic_v5_clean_inval(vmt_info->linear.vmt_base, + num_entries * sizeof(struct vmtl2_entry)); + + return 0; +} + +/* + * Allocate the first level of a two-level VM table. The second-level VM tables + * are allocated on demand (by vgic_v5_alloc_l2_vmt()). + */ +static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries) +{ + /* + * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is + * mandated by the GICv5 specification (GICv5 EAC0 Specification rule + * D_LSPBK). Hence, round up the number of entries to be at least 128 + * (or the next highest power of two as we give the HW the number of VM + * ID bits). + */ + if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES) + num_entries = GICV5_VMT_L2_TABLE_ENTRIES; + num_entries = roundup_pow_of_two(num_entries); + + vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES); + vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents); + if (!vmt_info->l2.vmt_base) + return -ENOMEM; + + vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs, + vmt_info->l2.num_l1_ents, + GFP_KERNEL); + if (!vmt_info->l2.l2ptrs) { + kfree(vmt_info->l2.vmt_base); + return -ENOMEM; + } + + vgic_v5_clean_inval(vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry)); + + return 0; +} + +/* + * Allocate a second level VMT, if required. This can be called eagerly, and + * will only perform the allocation if required. + */ +static int vgic_v5_alloc_l2_vmt(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd = VMT_L2_MAP; + struct vmtl2_entry *l2_table; + unsigned int l1_index; + int ret; + + /* Nothing to do if we have linear tables! */ + if (!vmt_info->two_level) + return 0; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return -EINVAL; + + /* + * We have 4k-sized L2 tables - this is mandated by the spec for + * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means + * that we have 128 entries per L1 VMTE. + */ + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + + guard(mutex)(&vmt_l2_lock); + + /* Already valid? Great! */ + if (vmt_info->l2.l2ptrs[l1_index]) + return 0; + + l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES); + if (!l2_table) + return -ENOMEM; + + /* The VMT is shared between all VMs. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE); + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], + cpu_to_le64(virt_to_phys(l2_table))); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + } + + /* + * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a + * conduit for interacting with the host's IRS. In the current case, + * this lets us resolve the VM ID to pass to the hardware. + */ + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + /* We've failed to make the L2 VMT valid - things are very broken! */ + if (ret) { + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Remove the pointer from L1 table */ + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + } + + kfree(l2_table); + return ret; + } + + vmt_info->l2.l2ptrs[l1_index] = l2_table; + + return 0; +} + +/* + * Allocate the top-level VMT. This can either be linear or two-level. + */ +int vgic_v5_vmt_allocate(unsigned int max_vpes) +{ + int ret; + + /* Allocate the tracking structure */ + vmt_info = kzalloc_obj(*vmt_info, GFP_KERNEL); + if (!vmt_info) + return -ENOMEM; + + ida_init(&vmt_info->vm_id_ida); + vmt_info->max_vpes = max_vpes; + vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps); + vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps); + vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps); + vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps); + + if (vmt_info->two_level) + ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries); + else + ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries); + + /* If anything failed, free our tracking structure before returning */ + if (ret) { + kfree(vmt_info); + vmt_info = NULL; + } + + return ret; +} + +/* + * Free the VMT and associated tracking structures. This isn't strictly expected + * to be called in general operation, but instead exists for completeness. + */ +int vgic_v5_vmt_free(void) +{ + if (!vmt_info) + return 0; + + if (!vmt_info->two_level) { + kfree(vmt_info->linear.vmt_base); + } else { + /* Free the L2 tables; kfree(NULL) is safe */ + for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i) + kfree(vmt_info->l2.l2ptrs[i]); + kfree(vmt_info->l2.l2ptrs); + + /* And now free the L1 table */ + kfree(vmt_info->l2.vmt_base); + } + + ida_destroy(&vmt_info->vm_id_ida); + kfree(vmt_info); + vmt_info = NULL; + + return 0; +} + +/* + * Look up a VMT Entry by VM ID. + */ +static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id) +{ + unsigned int l1_index, l2_index; + struct vmtl2_entry *l2_table; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return ERR_PTR(-EINVAL); + + if (!vmt_info->two_level) + return &vmt_info->linear.vmt_base[vm_id]; + + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES; + + if (l1_index >= vmt_info->l2.num_l1_ents) + return ERR_PTR(-E2BIG); + + if (!vmt_info->l2.l2ptrs[l1_index]) + return ERR_PTR(-EINVAL); + + l2_table = vmt_info->l2.l2ptrs[l1_index]; + return &l2_table[l2_index]; +} + +/* + * Zero a VMT Entry, and flush & invalidate to the PoC, if required. + */ +static int vgic_v5_reset_vmte(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* + * The VMT is normal memory shared with the IRS. Invalidate before + * rewriting the entry so that cacheline-granular maintenance cannot + * later push stale data for neighbouring IRS-visible state back to + * memory. + */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + + /* + * Prevent the compiler from eliding the individual VMTE + * stores. Ordering and visibility to the IRS are provided by the + * surrounding cache maintenance and command protocol, not by + * WRITE_ONCE(). + * + * The same compiler-access constraint applies to READ_ONCE() users in + * this file: when inspecting IRS-visible table entries, read the field + * exactly once and prevent the compiler from reusing, merging or + * tearing the access. Coherency and freshness for non-coherent IRSes + * still come from the surrounding cache maintenance. + */ + WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL)); + + /* And make our write visible to the IRS (if non-coherent) */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data + * structure. If we're out of VM IDs, the IDA catches that, and we return the + * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and + * return -EBUSY. + */ +int vgic_v5_allocate_vm_id(struct kvm *kvm) +{ + int id; + + if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL) + return -EBUSY; + + id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u, + GFP_KERNEL); + if (id < 0) + return id; + + kvm->arch.vgic.gicv5_vm.vm_id = id; + + return 0; +} + +/* + * Release the VM ID to allow it to be reallocated in the future. + */ +void vgic_v5_release_vm_id(struct kvm *kvm) +{ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id); + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; +} + +/* + * Initialise an entry in the VMT based on the index of the VM. + * + * Note: We don't mark the VMTE as valid as this needs to be done by + * the hardware. + */ +int vgic_v5_vmte_init(struct kvm *kvm) +{ + size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size; + void *vped_base = NULL, *vmd = NULL; + struct vgic_v5_vm_info *vmi = NULL; + u64 tmp, vmte_val0 = 0, vmte_val1; + u32 vm_id = vgic_v5_vm_id(kvm); + int ret, nr_cpus, nr_vcpus; + bool vmi_inserted = false; + struct vmtl2_entry *vmte; + vpe_entry *vpet = NULL; + struct kvm_vcpu *vcpu; + u16 max_vpe_id = 0; + unsigned long i; + + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus > vmt_info->max_vpes) + return -E2BIG; + + /* + * If we're using two-level VMTs, L2 is allocated on demand. For linear + * VMTs, this is a NOP. + */ + ret = vgic_v5_alloc_l2_vmt(kvm); + if (ret) + return ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* If the entry is already valid, something went wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + vmi = kzalloc_obj(*vmi); + if (!vmi) { + ret = -ENOMEM; + goto out_fail; + } + + ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL); + if (ret) + goto out_fail; + vmi_inserted = true; + + /* Allocate and assign the VM Descriptor, if required. */ + if (vmt_info->vmd_size != 0) { + vmd_alloc_size = round_up(vmt_info->vmd_size, + dma_get_cache_alignment()); + vmd = kzalloc(vmd_alloc_size, GFP_KERNEL); + if (!vmd) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vmd_base = vmd; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR, + virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT); + vmte_val0 = tmp; + } + + /* + * Allocate and assign the VPE Table. + * + * First of all, iterate over all vcpus to find the highest VPE ID we + * require - we need to ensure that we have enough storage for all + * vcpu_id values that userspace has picked and not just the total + * number of vcpus. This gives us the number of VPEs required for the + * VM. + * + * Round up the number of VPEs to a whole power of two as we cannot + * describe non-powers-of-two in the VMTE field as it conveys the number + * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is + * encoded as N - 1, so expose at least one VPE ID bit even for a + * single-vCPU VM to keep the views consistent. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + u16 vpe_id = vgic_v5_vpe_id(vcpu); + + if (vpe_id > max_vpe_id) + max_vpe_id = vpe_id; + } + + nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1)); + vmi->vpe_id_bits = fls(nr_cpus) - 1; + + vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet), + dma_get_cache_alignment()); + vpet = kzalloc(vpet_alloc_size, GFP_KERNEL); + if (!vpet) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vpet_base = vpet; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR, + virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits); + vmte_val1 = tmp; + + /* + * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed + * by the potentially sparse vcpu_id, but using that ID here would waste + * memory. Given that this is not userspace visible, we can cheat a bit + * and use the dense index instead. This is the ONLY place that we do + * this. + * + * Round the requested size up to a whole cacheline. kzalloc() gurantees + * natural alignment, so we ensure that the cachelines cannot be shared + * with unrelated slab objects. VPED and cacheline sizes are powers of + * two, so this also preserves the required VPED alignment when a VPED + * is larger than a cacheline. + */ + vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size, + dma_get_cache_alignment()); + vped_base = kzalloc(vped_alloc_size, GFP_KERNEL); + if (!vped_base) { + ret = -ENOMEM; + goto out_fail; + } + vmi->vped_base = vped_base; + + if (vmd) + vgic_v5_clean_inval(vmd, vmd_alloc_size); + vgic_v5_clean_inval(vpet, vpet_alloc_size); + vgic_v5_clean_inval(vped_base, vped_alloc_size); + + /* Publish the VMTE while serialising access to the shared VMT. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + kvm->arch.vgic.gicv5_vm.vmte_allocated = true; + + return 0; + +out_fail: + /* kfree(NULL) is safe so we can just kfree() at leisure */ + kfree(vmd); + kfree(vpet); + kfree(vped_base); + if (vmi_inserted) + xa_erase(&vm_info, vm_id); + kfree(vmi); + + vgic_v5_reset_vmte(kvm); + + return ret; +} + +/* + * Release the VMT Entry, freeing up any allocated data structures before + * zeroing the VMTE. + * + * The VMTE must be marked as invalid before it is released. + */ +int vgic_v5_vmte_release(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + int ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* Reject if the VMTE has not been marked as invalid! */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + goto no_vmi; + + kfree(vmi->vped_base); + kfree(vmi->vpet_base); + kfree(vmi->vmd_base); + + xa_erase(&vm_info, vm_id); + kfree(vmi); + +no_vmi: + /* + * If we didn't get far enough into allocating a VMTE to create the VM + * info structure, then we just zero the VMTE and move on. There's + * nothing else we can realistically do here. + */ + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + kvm->arch.vgic.gicv5_vm.vmte_allocated = false; + + return 0; +} + +/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */ +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + vpe_entry tmp, *vpet_base; + void *vped; + + /* Make sure we're not over what the hardware supports */ + if (vpe_id >= vmt_info->max_vpes) + return -E2BIG; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + + /* If the VPETE for this CPU is already valid we've gone wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base)); + if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID) + return -EBUSY; + } + + vped = (u8 *)vmi->vped_base + + (size_t)vcpu->vcpu_idx * vmt_info->vped_size; + + tmp = FIELD_PREP(GICV5_VPED_ADDR, virt_to_phys(vped) >> GICV5_VPED_ADDR_SHIFT); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +/* Clear the VPE's table entry after the VMTE has been made invalid. */ +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + vpe_entry *vpet_base; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EBUSY; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], 0ULL); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h new file mode 100644 index 000000000000..962be0c7cd3f --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#ifndef __KVM_ARM_VGICV5_TABLES_H__ +#define __KVM_ARM_VGICV5_TABLES_H__ + +#include +#include +#include + +/* Level 1 Virtual Machine Table Entry */ +typedef __le64 vmtl1_entry; + +/* Level 2 Virtual Machine Table Entry */ +struct vmtl2_entry { + __le64 val[4]; +}; + +/* Virtual PE Table Entry */ +typedef __le64 vpe_entry; + +struct vgic_v5_vm_info { + void __iomem *vmd_base; + vpe_entry __iomem *vpet_base; + void *vped_base; + u8 vpe_id_bits; +}; + +struct vgic_v5_vmt { + union { + struct { + struct vmtl2_entry *vmt_base; + unsigned int num_ents; + } linear; + struct { + vmtl1_entry *vmt_base; + struct vmtl2_entry **l2ptrs; + unsigned int num_l1_ents; + } l2; + }; + bool two_level; + unsigned int num_entries; + unsigned int max_vpes; + size_t vmd_size; + size_t vped_size; + struct ida vm_id_ida; +}; + +static inline u32 vgic_v5_vm_id(struct kvm *kvm) +{ + return kvm->arch.vgic.gicv5_vm.vm_id; +} + +/* + * For vGICv5, we need to consolidate two views: + * - The vcpu ID assigned by userspace, which it can assign however it + * sees fit. + * - The index into the VPET, which is also presented to the guest as + * the IAFFID for the VPE in question. + * + * Ideally, we don't want to intercept the guest setting an interrupt's affinity + * - this would somewhat defeat the point of allowing the hardware to manage the + * interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID, + * which ensures that both userspace and the GICv5 hardware have a consistent + * view. + * + * This means two things: + * - Our VPET might be sparse, depending on the IDs picked by userspace. + * - We need to check the vcpu ID provided by userspace, and reject anything + * we cannot handle. + * + * In order to at least ensure consistent usage in the places that we need to + * use the VPE ID, we provide this helper. + */ +static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu) +{ + return vcpu->vcpu_id; +} + +static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db; +} + +extern raw_spinlock_t vgic_v5_irs_lock; + +int vgic_v5_vmt_allocate(unsigned int max_vpes); +int vgic_v5_vmt_free(void); + +int vgic_v5_allocate_vm_id(struct kvm *kvm); +void vgic_v5_release_vm_id(struct kvm *kvm); + +int vgic_v5_vmte_init(struct kvm *kvm); +int vgic_v5_vmte_release(struct kvm *kvm); +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 60bb39421ad0..efe11a5a28f9 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -5,10 +5,12 @@ #include +#include #include #include #include +#include "vgic-v5-tables.h" #include "vgic.h" #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps @@ -126,6 +128,22 @@ skip_v5: return 0; } +static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) +{ + enum gicv5_vcpu_cmd *cmd = vcpu_info; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + switch (*cmd) { + case VMT_L2_MAP: + case VMTE_MAKE_VALID: + case VMTE_MAKE_INVALID: + /* Not yet implemented */ + default: + return -EINVAL; + } +} + /* * This set of irq_chip functions is specific for doorbells. */ @@ -137,6 +155,7 @@ static const struct irq_chip vgic_v5_db_irq_chip = { .irq_set_affinity = irq_chip_set_affinity_parent, .irq_get_irqchip_state = irq_chip_get_parent_state, .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index 01b032a06c2b..abf92b1c619d 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -269,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi) * itself is not supported) again serves to make it easier to find physically * contiguous blocks of memory. */ -static unsigned int gicv5_irs_l2_sz(u32 idr2) +unsigned int gicv5_irs_l2_sz(u32 l2sz) { switch (PAGE_SIZE) { case SZ_64K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_64K; fallthrough; case SZ_4K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; fallthrough; case SZ_16K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_16K; break; } - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; return GICV5_IRS_IST_CFGR_L2SZ_64K; @@ -334,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); if (two_levels) - l2sz = gicv5_irs_l2_sz(idr2); + l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2)); istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 78a442046ebc..3ec3f7e1b88b 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -361,6 +361,8 @@ struct vgic_redist_region { struct list_head list; }; +#define VGIC_V5_VM_ID_INVAL (-1) + struct vgic_v5_vm { /* * We only expose a subset of PPIs to the guest. This subset is a @@ -383,6 +385,8 @@ struct vgic_v5_vm { struct fwnode_handle *fwnode; struct irq_domain *domain; int vpe_db_base; + u32 vm_id; + bool vmte_allocated; }; struct vgic_dist { diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 6ed886c09bec..fae3af230953 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -160,9 +160,9 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(0), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(1), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(2), (r)) #define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) #define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) @@ -611,6 +611,7 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid); struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id); int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); +unsigned int gicv5_irs_l2_sz(u32 l2sz); void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ @@ -655,4 +656,11 @@ void gicv5_deinit_lpis(void); void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); + +enum gicv5_vcpu_cmd { + VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ + VMTE_MAKE_VALID, /* Make the VMTE valid */ + VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ +}; + #endif -- cgit v1.2.3 From 77f7a9282fee67180134e75b107ff09d7e1a5876 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:39:21 +0000 Subject: KVM: arm64: gic-v5: Introduce guest IST alloc and management GICv5 guests use Interrupt State Tables (ISTs) to track and manage the interrupt state for SPIs and LPIs. These ISTs are provided to the host's IRS via the VMTE. On a host GICv5 system, SPIs do not require any up-front memory allocation prior to their use, unlike LPIs which require the OS to allocate an IST. For a GICv5 guest, the same holds from the guest's point of view: SPIs should require no explicit memory allocation by the guest. This means that KVM must provision the memory passed to the IRS for managing a guest's SPI state. Introduce the machinery needed to allocate a linear SPI IST. A later commit will allocate it as part of VM creation, before running the guest for the first time. The host is responsible for freeing this memory on guest teardown. For LPIs, the guest provisions memory for its LPI IST. KVM does not pass that memory directly to the host IRS. Instead, add support for allocating a shadow LPI IST and passing that to the IRS through the VMTE. The LPI IST may be allocated as a two-level structure when supported and required by the configured LPI ID space, as many more LPIs are expected than SPIs. The host frees this memory on guest teardown. Assigning an IST to a VM requires an IRS operation rather than writing the Valid bit directly in the VMTE. Extend the doorbell command interface with the operations needed to make SPI and LPI ISTs valid or invalid. Together with the existing VMT commands, the command set is: VMT_L2_MAP - Make a second level VM table valid VMTE_MAKE_VALID - Make a single VMTE, and hence VM, valid VMTE_MAKE_INVALID - Make a single VMTE, and hence VM, invalid SPI_VIST_MAKE_VALID - Make the SPI IST valid LPI_VIST_MAKE_VALID - Make the LPI IST valid LPI_VIST_MAKE_INVALID - Make the LPI IST invalid Once implemented, irq_set_vcpu_affinity() will act as an ioctl-like interface for issuing commands specific to either a VM or the particular VPE that the doorbell belongs to. None of these commands are plumbed through to the host IRS at this stage. There is intentionally no SPI_VIST_MAKE_INVALID command. The SPI IST will be allocated as part of VM creation and will remain valid for the lifetime of the VM. It can be freed after the VMTE has been made invalid during teardown. The LPI IST, on the other hand, is driven by the guest, which is free to invalidate and free its LPI IST at any point. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-11-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 551 +++++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.h | 22 ++ arch/arm64/kvm/vgic/vgic-v5.c | 3 + include/linux/irqchip/arm-gic-v5.h | 3 + 4 files changed, 579 insertions(+) (limited to 'include/linux') diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index 7252d48431a5..f088006f3967 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -65,6 +65,14 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VPED_ADDR_SHIFT 3ULL #define GICV5_VPED_ADDR GENMASK_ULL(55, 3) +/* + * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of + * the VMTE (0-based). We call this a section here in an attempt to simplify the + * code. + */ +#define GICV5_VMTEL2_LPI_SECTION 2 +#define GICV5_VMTEL2_SPI_SECTION 3 + /* * Our IRS might be coherent or non-coherent. If coherent, we can just emit a * DSB to ensure that we're in sync. However, when non-coherent, we need to @@ -553,6 +561,25 @@ out_fail: return ret; } +/* + * The following set of forward declarations makes the code layout a *little* + * clearer as it lets us keep the IST-related code together. + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + /* * Release the VMT Entry, freeing up any allocated data structures before * zeroing the VMTE. @@ -581,6 +608,22 @@ int vgic_v5_vmte_release(struct kvm *kvm) if (!vmi) goto no_vmi; + /* If we have an LPI IST, free it */ + if (vmi->h_lpi_ist) { + ret = vgic_v5_lpi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_lpi_ist = NULL; + + /* If we have an SPI IST, free it */ + if (vmi->h_spi_ist) { + ret = vgic_v5_spi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_spi_ist = NULL; + kfree(vmi->vped_base); kfree(vmi->vpet_base); kfree(vmi->vmd_base); @@ -679,3 +722,511 @@ int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) return 0; } + +/* + * Assign an already allocated IST to the VM by populating the fields in the + * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even + * if the paths to reach it might be vastly different. + */ +static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, + bool two_level, unsigned int id_bits, + unsigned int l2sz, unsigned int istsz, + bool spi_ist) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd; + struct vmtl2_entry *vmte; + unsigned int section; + u64 tmp; + int ret; + + /* + * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3 + * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI + * sections have the same layout, and as we are either operating on SPIs + * or LPIs we pick a section of the VMTE to modify up-front. + * + * See the GICv5 EAC0 Specification 11.2.2 for more details about the + * VMTE layout. + */ + section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION; + + if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) { + pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n", + task_pid_nr(current), ist_base, + GICV5_VMTEL2E_IST_ADDR); + return -EINVAL; + } + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR, + ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits); + if (two_level) + tmp |= GICV5_VMTEL2E_IST_STRUCTURE; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Bail if already allocated */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[section])) & + GICV5_VMTEL2E_IST_VALID) + return -EINVAL; + + WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + /* Finally, mark the entry as valid */ + cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + return ret; +} + +/* + * Allocate a Linear IST - always used for SPIs and potentially LPIs. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the + * size). + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, unsigned int istsz) +{ + const size_t n = max(5, id_bits + 1 + istsz); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + u32 l1sz; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + /* + * Allocate the IST. We only have one level, so we just use the L2 ISTE. + */ + l1sz = BIT(n + 1); + ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + if (spi_ist) { + vmi->h_spi_ist = ist; + } else { + vmi->h_lpi_ist_structure = false; + vmi->h_lpi_ist = ist; + } + + vgic_v5_clean_inval(ist, l1sz); + + return 0; +} + +/* + * Allocate the first level of a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + u32 vm_id = vgic_v5_vm_id(kvm); + const u32 l1_size = BIT(n + 1); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + vmi->h_lpi_ist_structure = true; + vmi->h_lpi_ist = ist; + + vgic_v5_clean_inval(ist, l1_size); + + return 0; +} + +/* + * Allocate ALL of the second level ISTs for a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. The l2_size calculation is from section 11.2.3 of the same document. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + const size_t l2_size = BIT(11 + (2 * l2sz) + 1); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *l2ist; + __le64 *l1ist; + int index; + u64 val; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + l1ist = vmi->h_lpi_ist; + + /* + * Allocate the storage for the pointers to the L2 ISTs (used when + * freeing later). + */ + vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries, + GFP_KERNEL_ACCOUNT); + if (!vmi->h_lpi_l2_ists) + return -ENOMEM; + + /* Allocate the L2 IST for each L1 IST entry */ + for (index = 0; index < l1_entries; ++index) { + l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT); + if (!l2ist) { + while (--index >= 0) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + + return -ENOMEM; + } + + /* + * We are not doing on-demand allocation of the L2 ISTs, and are + * instead provisioning the whole IST up front. This means that + * we are able to mark the L2 ISTs as valid in the L1 ISTEs as + * the overall IST is not yet valid. + */ + val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) | + GICV5_ISTL1E_VALID; + l1ist[index] = cpu_to_le64(val); + + vmi->h_lpi_l2_ists[index] = l2ist; + + vgic_v5_clean_inval(l2ist, l2_size); + } + + /* Handle CMOs for the whole L1 IST in one go */ + vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist)); + + return 0; +} + +/* Allocate a two-level IST - LPIs, only */ +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + int ret; + + /* + * Allocate the L1 IST first, then all of the L2s. Everything + * is preallocated and we do no on-demand IST allocation. This + * is to avoid needing to track if and when the guest is doing + * on-demand IST allocation. + */ + ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz); + if (ret) + return ret; + + ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz); + if (ret) { + /* Free the L1 IST again */ + vmi = xa_load(&vm_info, vm_id); + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = 0; + + return ret; + } + + return 0; +} + +static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2sz) +{ + if (!vmi->h_lpi_ist_structure) { + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + return; + } + + if (vmi->h_lpi_l2_ists) { + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + int index; + + for (index = 0; index < l1_entries; ++index) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + } + + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; +} + +static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return; + + kfree(vmi->h_spi_ist); + vmi->h_spi_ist = NULL; +} + +/* + * Free a Linear IST. Can only happen once the VM is dead. + */ +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + struct vgic_v5_vm_info *vmi; + int section; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + if (spi) { + section = GICV5_VMTEL2_SPI_SECTION; + vgic_v5_free_allocated_spi_ist(kvm); + } else { + section = GICV5_VMTEL2_LPI_SECTION; + vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0); + } + + /* The VM should be dead here, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Free a Two-Level IST. Can only happen once the VM is dead. + */ +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi) +{ + unsigned int id_bits, istsz, l2sz; + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + u64 tmp; + int section; + + /* We don't create two-level SPI ISTs, so freeing is a bad idea! */ + if (spi) + return -EINVAL; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + section = GICV5_VMTEL2_LPI_SECTION; + + if (!vmi->h_lpi_ist_structure) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + tmp = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp); + istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp); + l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp); + + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + /* The VM must be dead, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* Helper to determine ISTE size based on metadata requirements */ +static unsigned int vgic_v5_ist_istsz(unsigned int id_bits) +{ + if (!vgic_v5_irs_istmd(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +/* + * Allocate an IST for SPIs. + * + * We don't anticipate a large number of SPIs being allocated. Therefore, we + * always allocate a Linear IST for SPIs. This will need to be revisited should + * that assumption no longer hold. + */ +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + phys_addr_t base_addr; + unsigned int istsz; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + istsz = vgic_v5_ist_istsz(id_bits); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_spi_ist) + return -EBUSY; + + ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz); + if (ret) + return ret; + base_addr = virt_to_phys(vmi->h_spi_ist); + + ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, + true); + if (ret) { + vgic_v5_free_allocated_spi_ist(kvm); + return ret; + } + + return 0; +} + +/* + * Free the IST for SPIs. Should only happen once the VM is dead. + */ +static int vgic_v5_spi_ist_free(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->arch.config_lock); + + return vgic_v5_linear_ist_free(kvm, true); +} + +/* + * Allocate an IST for LPIs. + * + * Unlike with SPIs, we anticipate that the guest will allocate a relatively + * large number of LPIs. Therefore, while we support doing a linear LPI IST, it + * is expected that LPI ISTs will be two-level. + */ +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + unsigned int istsz, l2sz; + phys_addr_t phys_addr; + bool two_level; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_lpi_ist) + return -EBUSY; + + istsz = vgic_v5_ist_istsz(id_bits); + l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps)); + + /* + * Determine if we want to create a Linear or a Two-Level IST. + * + * A two-level IST is only required when a single L2 IST cannot cover + * the requested ID space. This depends on the L2 IST size selected for + * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to + * two-level too early when the selected L2 IST is larger than a page, + * and the allocation sizing arithmetic would underflow. + */ + two_level = vgic_v5_irs_ist_levels(&irs_caps) && + id_bits > ((10 - istsz) + (2 * l2sz)); + + if (!two_level) + ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */, + id_bits, istsz); + else + ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz, + l2sz); + + if (ret) + return ret; + + phys_addr = virt_to_phys(vmi->h_lpi_ist); + ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, + istsz, false); + if (ret) + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + return ret; +} + +/* Free the LPI IST again */ +int vgic_v5_lpi_ist_free(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + if (!vmi->h_lpi_ist_structure) + return vgic_v5_linear_ist_free(kvm, false); + else + return vgic_v5_two_level_ist_free(kvm, false); +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index 962be0c7cd3f..a711a247cc6b 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -26,6 +26,24 @@ struct vgic_v5_vm_info { vpe_entry __iomem *vpet_base; void *vped_base; u8 vpe_id_bits; + + /* + * Both the LPI and SPI ISTs are allocated by the hypervisor. While it + * would be possible to track and access them by iterating over the ISTs + * themselves, it makes more sense to store pointers to the ISTs. + * + * The LPI IST can either be two-level or linear. Hence, we keep track + * of the structure. If it is two-level, we retain pointers to the L1 + * IST and to each L2 IST array. If it is linear, we just store the base + * address of the IST array. + * + * The SPI IST is linear, and therefore we just store the base address + * of the SPI IST array. + */ + bool h_lpi_ist_structure; + __le64 *h_lpi_ist; + __le64 **h_lpi_l2_ists; + __le64 *h_spi_ist; }; struct vgic_v5_vmt { @@ -97,4 +115,8 @@ int vgic_v5_vmte_release(struct kvm *kvm); int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_free(struct kvm *kvm); + #endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index efe11a5a28f9..c65355483f64 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -138,6 +138,9 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) case VMT_L2_MAP: case VMTE_MAKE_VALID: case VMTE_MAKE_INVALID: + case SPI_VIST_MAKE_VALID: + case LPI_VIST_MAKE_VALID: + case LPI_VIST_MAKE_INVALID: /* Not yet implemented */ default: return -EINVAL; diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index fae3af230953..24949b604531 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -661,6 +661,9 @@ enum gicv5_vcpu_cmd { VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ VMTE_MAKE_VALID, /* Make the VMTE valid */ VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + SPI_VIST_MAKE_VALID, /* No corresponding invalid */ + LPI_VIST_MAKE_VALID, /* Triggered by a guest */ + LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ }; #endif -- cgit v1.2.3 From d749533c006d4e6bd4413a8c99768cdd4f4e706d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:40:53 +0000 Subject: KVM: arm64: gic-v5: Implement VPE IRS MMIO Ops Introduce interfaces to make VPEs valid, and to configure them, via the host's IRS. As with the other valid bits in the GICv5 VM tables, VPEs cannot be made valid directly, and instead are made valid via an IRS MMIO Op. Additionally, some of the VPE configuration takes place via the IRS MMIO interface too (via the IRS_VPE_CR0, IRS_VPE_DBR). VPE doorbells are, for example, configured via this interface. The existing VPE-doorbell-based commands are extended with: VPE_MAKE_VALID - Make the VPE valid in the VPET Note: There is no VPE_MAKE_INVALID as VPEs are only made invalid on teardown, at which point the whole VMTE is marked as invalid. Hence, it is not required. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-14-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5.c | 84 ++++++++++++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v5.h | 1 + 2 files changed, 85 insertions(+) (limited to 'include/linux') diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index e1571d1788a6..455161e52f81 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -201,6 +201,18 @@ static int vgic_v5_irs_wait_for_vm_op(void) NULL); } +/* + * Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR, + * IRS_VPE_CR0. + */ +static int vgic_v5_irs_wait_for_vpe_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VPE_STATUSR, + GICV5_IRS_VPE_STATUSR_IDLE, + NULL); +} + static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) { int ret; @@ -308,10 +320,73 @@ static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); } +static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, + irq_hw_number_t db_hwirq) +{ + u64 vmap_vper, dbr, selr; + u32 statusr, cr0; + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + /* Mark the VPE as valid */ + vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) | + GICV5_IRS_VMAP_VPER_M; + irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER); + + /* Wait for the VPE to be marked valid in the VPET */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) | + GICV5_IRS_VPE_SELR_S; + irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR); + if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr)) + return -EINVAL; + + /* Set targeted only routing (disable 1ofN vPE selection) */ + cr0 = GICV5_IRS_VPE_CR0_DPS; + irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + /* + * The VPE has not yet run. Therefore, make sure that all interrupts + * will generate a doorbell. + */ + dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) | + GICV5_IRS_VPE_DBR_DBV; + irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + return 0; +} + static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { struct vgic_v5_vm *vm = data->domain->host_data; enum gicv5_vcpu_cmd *cmd = vcpu_info; + /* Our VPE ID is the index within the doorbell domain */ + u16 vpe_id = data->hwirq; guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); @@ -322,6 +397,15 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) return vgic_v5_irs_set_vm_valid(vm->vm_id); case VMTE_MAKE_INVALID: return vgic_v5_irs_set_vm_invalid(vm->vm_id); + case VPE_MAKE_VALID: + /* + * We need the actual LPI ID which lives in the top-most parent + * domain. This hwirq won't include the type (LPI) but that's + * not required for the IRS_VPE_DBR. + */ + while (data->parent_data) + data = data->parent_data; + return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq); case SPI_VIST_MAKE_VALID: return vgic_v5_irs_set_vist_valid(vm->vm_id, true); case LPI_VIST_MAKE_VALID: diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 24949b604531..299e75bc9525 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -661,6 +661,7 @@ enum gicv5_vcpu_cmd { VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ VMTE_MAKE_VALID, /* Make the VMTE valid */ VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + VPE_MAKE_VALID, /* No corresponding invalid */ SPI_VIST_MAKE_VALID, /* No corresponding invalid */ LPI_VIST_MAKE_VALID, /* Triggered by a guest */ LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ -- cgit v1.2.3 From 968c346143b71678a4970d1d0ab3f6505be7dfd9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:16 +0100 Subject: tracing: Include linux/types.h in trace_remote_event.h trace_remote_event.h uses bool without including linux/types.h, so a translation unit that includes it ahead of anything else that pulls types.h in fails to build, as with nvhe/trace.h at EL2. Reviewed-by: Vincent Donnefort Reviewed-by: Marc Zyngier Acked-by: Steven Rostedt Signed-off-by: Fuad Tabba Acked-by: Steven Rostedt Link: https://patch.msgid.link/20260901140326.3812068-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- include/linux/trace_remote_event.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h index c8ae1e1f5e72..e4cc2d4497bc 100644 --- a/include/linux/trace_remote_event.h +++ b/include/linux/trace_remote_event.h @@ -3,6 +3,8 @@ #ifndef _LINUX_TRACE_REMOTE_EVENTS_H #define _LINUX_TRACE_REMOTE_EVENTS_H +#include + struct trace_remote; struct trace_event_fields; struct trace_seq; -- cgit v1.2.3 From caa63620f9e37a5eec4d0af13534ebd6c0d33f97 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:44 +0100 Subject: KVM: Allow architectures to disallow pre-fault All existing architectures which implement KVM pre-fault (x86, s390) have mechanisms for disallowing pre-faulting. Currently these are open coded as part of kvm_arch_vcpu_pre_fault_memory(). Formalise this by moving them into a new kvm_arch_pre_fault_allowed() hook, which every architecture selecting CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY must implement, returning an error code if the operation is disallowed or 0 otherwise. The hook is called early in the generic code, allowing architectures to disallow the operation prior to vCPU load. This is important, as kvm_vcpu_pre_fault_memory() is the only place where generic code can call vcpu_load() on a vCPU that has not yet been initialised. This lays the foundation for a future change which implements pre-faulting for arm64 which needs to disallow the mechanism for uninitialised vCPUs. Suggested-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-1-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/s390/kvm/s390/s390.c | 11 ++++++++--- arch/x86/kvm/mmu/mmu.c | 11 ++++++++--- arch/x86/kvm/vmx/tdx.c | 2 +- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 10 +++++++++- 5 files changed, 27 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index 5c73f43782a7..47fe032444f4 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -5784,6 +5784,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + return 0; +} + /** * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables * @vcpu: the vcpu that shall appear to have generated the fault-in. @@ -5810,9 +5818,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_ gpa_t end; int rc; - if (kvm_is_ucontrol(vcpu->kvm)) - return -EINVAL; - rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); if (rc == PGM_ADDRESSING) return -ENOENT; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..c35fd2868c20 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5086,6 +5086,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, } } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!vcpu->kvm->arch.pre_fault_allowed) + return -EOPNOTSUPP; + + return 0; +} + long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { @@ -5095,9 +5103,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, u64 end; int r; - if (!vcpu->kvm->arch.pre_fault_allowed) - return -EOPNOTSUPP; - if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..a4d4479e2b28 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1708,7 +1708,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); /* - * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() + * Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed() * before kvm_tdx->state. Userspace must not be allowed to pre-fault * arbitrary memory until the initial memory image is finalized. Pairs * with the smp_wmb() in tdx_td_finalize(). diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..39a4d345aeca 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2643,6 +2643,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu); long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..41356577bee7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4353,7 +4353,7 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { - int idx; + int idx, err; long r; u64 full_size; @@ -4365,6 +4365,14 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, range->gpa + range->size <= range->gpa) return -EINVAL; + /* + * Certain architectures (e.g. arm64) need to reject the ioctl 'early' + * before vcpu_load(). + */ + err = kvm_arch_pre_fault_allowed(vcpu); + if (err) + return err; + vcpu_load(vcpu); idx = srcu_read_lock(&vcpu->kvm->srcu); -- cgit v1.2.3 From 8233000034a630432a7da0311d94e8725207a095 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 15 Sep 2026 09:44:39 +0100 Subject: KVM: Fix comments that refer to the non-existent VCPU_RUN ioctl Neither the VCPU_RUN in kvm_main.c nor the VCPU RUN in kvm_host.h names an ioctl that exists. The one that enters the guest is KVM_RUN, so refer to it by that name. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260915084441.3846947-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..acae5a61e7f8 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -562,7 +562,7 @@ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) /* * The memory barrier ensures a previous write to vcpu->requests cannot * be reordered with the read of vcpu->mode. It pairs with the general - * memory barrier following the write of vcpu->mode in VCPU RUN. + * memory barrier following the write of vcpu->mode in KVM_RUN. */ smp_mb__before_atomic(); return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..cc391b0666a6 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -470,7 +470,7 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_dirty_ring_free(&vcpu->dirty_ring); /* - * No need for rcu_read_lock as VCPU_RUN is the only place that changes + * No need for rcu_read_lock as KVM_RUN is the only place that changes * the vcpu->pid pointer, and at destruction time all file descriptors * are already gone. */ -- cgit v1.2.3