diff options
| -rw-r--r-- | arch/x86/include/asm/kvm-x86-ops.h | 1 | ||||
| -rw-r--r-- | arch/x86/include/asm/kvm_host.h | 7 | ||||
| -rw-r--r-- | arch/x86/include/asm/vmx.h | 8 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/common.h | 90 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/main.c | 145 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/nested.c | 36 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/posted_intr.c | 24 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/posted_intr.h | 8 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/tdx.c | 112 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/vmx.c | 205 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/vmx.h | 37 | ||||
| -rw-r--r-- | arch/x86/kvm/vmx/x86_ops.h | 2 | ||||
| -rw-r--r-- | arch/x86/kvm/x86.c | 13 |
13 files changed, 397 insertions, 291 deletions
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 67b43c167045..48fa973b98d5 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed) KVM_X86_OP(nmi_allowed) KVM_X86_OP(get_nmi_mask) KVM_X86_OP(set_nmi_mask) +KVM_X86_OP_OPTIONAL(clear_hlt) KVM_X86_OP(enable_nmi_window) KVM_X86_OP(enable_irq_window) KVM_X86_OP_OPTIONAL(update_cr8_intercept) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 2a0245fa7e26..0378577d9d8b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1621,6 +1621,13 @@ struct kvm_x86_ops { int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); + /* + * Force the vCPU out of any hardware-tracked halted/inactive state so + * that it will fetch and execute instructions on the next VM-Enter. + * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across + * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. + */ + void (*clear_hlt)(struct kvm_vcpu *vcpu); /* Whether or not a virtual NMI is pending in hardware. */ bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); /* diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b3096ff04..e49b836c7313 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -20,6 +20,14 @@ #include <asm/trapnr.h> #include <asm/vmxfeatures.h> +/* + * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason + * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter. + * The value itself is arbitrary, it just needs to not collide with a real exit + * reason. + */ +#define EXIT_REASON_UNDEFINED 0xdead + struct vmcs_hdr { u32 revision_id:31; u32 shadow_vmcs:1; diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 08005676702c..b2905335a2d2 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -2,10 +2,13 @@ #ifndef __KVM_X86_VMX_COMMON_H #define __KVM_X86_VMX_COMMON_H +#include <linux/container_of.h> #include <linux/kvm_host.h> #include <asm/posted_intr.h> +#include <asm/vmx.h> #include "mmu.h" +#include "vmx_ops.h" union vmx_exit_reason { struct { @@ -55,6 +58,8 @@ struct vcpu_vt { #endif }; +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu); + #ifdef CONFIG_KVM_INTEL_TDX static __always_inline bool is_td(struct kvm *kvm) @@ -74,14 +79,62 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + +static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) +{ + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); +} + +static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) +{ + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); +} + +static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu) +{ + return to_vt(vcpu)->exit_reason; +} + +static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + + return vt->exit_qualification; +} + +static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + return vt->exit_intr_info; +} + +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ return !kvm_is_addr_direct(kvm, gpa); } -static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, - unsigned long exit_qualification) +static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long exit_qualification) { u64 error_code; @@ -164,8 +217,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the * interrupt if necessary. */ -static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, - struct pi_desc *pi_desc, int vector) +static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu, + struct pi_desc *pi_desc, int vector) { if (pi_test_and_set_pir(vector, pi_desc)) return; @@ -183,6 +236,33 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); +static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, + unsigned long exit_qual) +{ + bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; + + ++vcpu->stat.notify_window_exits; + + if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || + context_invalid) { + vcpu->run->exit_reason = KVM_EXIT_NOTIFY; + vcpu->run->notify.flags = context_invalid ? + KVM_NOTIFY_CONTEXT_INVALID : 0; + return 0; + } + + return 1; +} + +static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 797206912ae9..d75170a6199e 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/entry-common.h> #include <linux/moduleparam.h> #include "x86_ops.h" @@ -10,8 +11,12 @@ #include "tdx.h" #include "tdx_arch.h" +#pragma GCC poison to_vmx + +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt)); + #ifdef CONFIG_KVM_INTEL_TDX -static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt)); static void vt_disable_virtualization_cpu(void) { @@ -154,15 +159,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) return vmx_vcpu_run(vcpu, run_flags); } -static int vt_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion fastpath) -{ - if (is_td_vcpu(vcpu)) - return tdx_handle_exit(vcpu, fastpath); - - return vmx_handle_exit(vcpu, fastpath); -} - static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) { if (is_td_vcpu(vcpu)) { @@ -601,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) vmx_set_nmi_mask(vcpu, masked); } +static void vt_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* + * TDX doesn't support disabling HLT-exiting, and KVM can't access a + * TD's VMCS, so there is never any hardware halted state to clear. + */ + if (is_td_vcpu(vcpu)) + return; + + vmx_clear_hlt(vcpu); +} + static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) { /* Refer to the comments in tdx_inject_nmi(). */ @@ -807,7 +815,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) vmx_write_tsc_multiplier(vcpu); } -#ifdef CONFIG_X86_64 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -826,7 +833,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) vmx_cancel_hv_timer(vcpu); } -#endif static void vt_setup_mce(struct kvm_vcpu *vcpu) { @@ -876,6 +882,112 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) +{ + if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vt_get_intr_info(vcpu))) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); + x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); + kvm_after_interrupt(vcpu); +} + +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vt_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + +static int vt_handle_exit(struct kvm_vcpu *vcpu, + enum exit_fastpath_completion fastpath) +{ + int ret; + +#ifdef CONFIG_KVM_INTEL_TDX + if (is_td_vcpu(vcpu)) + ret = tdx_handle_exit(vcpu, fastpath); + else +#endif + ret = vmx_handle_exit(vcpu, fastpath); + + /* + * Exit to user space when bus lock detected to inform that there is + * a bus lock in guest. + */ + if (vt_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -949,7 +1061,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), .vcpu_run = vt_op(vcpu_run), - .handle_exit = vt_op(handle_exit), + .handle_exit = vt_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, .update_emulated_instruction = vmx_update_emulated_instruction, .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), @@ -964,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .nmi_allowed = vt_op(nmi_allowed), .get_nmi_mask = vt_op(get_nmi_mask), .set_nmi_mask = vt_op(set_nmi_mask), + .clear_hlt = vt_op(clear_hlt), .enable_nmi_window = vt_op(enable_nmi_window), .enable_irq_window = vt_op(enable_irq_window), .update_cr8_intercept = vt_op(update_cr8_intercept), @@ -1000,12 +1113,12 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .pi_update_irte = vmx_pi_update_irte, - .pi_start_bypass = vmx_pi_start_bypass, + .pi_update_irte = vt_pi_update_irte, + .pi_start_bypass = vt_pi_start_bypass, #ifdef CONFIG_X86_64 .set_hv_timer = vt_op(set_hv_timer), diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 14a0babb3c8d..9b0bfa2f854c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * "NMI unblocking due to IRET", i.e. the bit can be propagated * as-is from the original EXIT_QUALIFICATION. */ - exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; + exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; } else { if (fault->error_code & PFERR_RSVD_MASK) { vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; @@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * and walks the faulting GPA. */ if (from_hardware) - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + exit_qualification |= vt_get_exit_qual(vcpu) & mask; else exit_qualification |= fault->exit_qualification & mask; @@ -4748,7 +4748,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, { /* update exit information fields: */ vmcs12->vm_exit_reason = vm_exit_reason; - if (vmx_get_exit_reason(vcpu).enclave_mode) + if (vt_get_exit_reason(vcpu).enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; @@ -5374,7 +5374,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, struct x86_exception e; int r; - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) { *ret = 1; @@ -5666,7 +5666,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5772,7 +5772,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5963,7 +5963,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); gpa_t current_vmptr; struct x86_exception e; @@ -6032,7 +6032,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6115,7 +6115,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6247,8 +6247,8 @@ fail: * EXIT_REASON_VMFUNC as the exit reason. */ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, - vmx_get_intr_info(vcpu), - vmx_get_exit_qual(vcpu)); + vt_get_intr_info(vcpu), + vt_get_exit_qual(vcpu)); return 1; } @@ -6299,7 +6299,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -6325,7 +6325,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) - msr_index = vmx_get_exit_qual(vcpu); + msr_index = vt_get_exit_qual(vcpu); else msr_index = kvm_ecx_read(vcpu); @@ -6361,7 +6361,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -6495,7 +6495,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6578,7 +6578,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6748,14 +6748,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) * need to be synthesized by querying the in-kernel LAPIC, but external * interrupts are never reflected to L1 so it's a non-issue. */ - exit_intr_info = vmx_get_intr_info(vcpu); + exit_intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(exit_intr_info)) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 4a6d9a17da23..2644add54b76 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -14,6 +14,8 @@ #include "vmx.h" #include "tdx.h" +#pragma GCC poison to_vmx + /* * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when @@ -54,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) return 0; } -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); struct vcpu_vt *vt = to_vt(vcpu); @@ -144,11 +146,11 @@ after_clear_sn: pi_set_on(pi_desc); } -static bool vmx_can_use_vtd_pi(struct kvm *kvm) +static bool vt_can_use_vtd_pi(struct kvm *kvm) { /* * Note, reading the number of possible bypass IRQs can race with a - * bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures + * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. */ return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && @@ -206,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); } -static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) +static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) { /* * The default posted interrupt vector does nothing when @@ -217,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) * back to the pi_wakeup_handler() function. */ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || - vmx_can_use_vtd_pi(vcpu->kvm); + vt_can_use_vtd_pi(vcpu->kvm); } -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - if (!vmx_needs_pi_wakeup(vcpu)) + if (!vt_needs_pi_wakeup(vcpu)) return; /* @@ -292,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) /* * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup - * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put(). + * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). */ -void vmx_pi_start_bypass(struct kvm *kvm) +void vt_pi_start_bypass(struct kvm *kvm) { - if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm))) + if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) return; kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); } -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector) { diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index a4af39948cf0..c5860ea828ae 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h @@ -8,16 +8,16 @@ #include <asm/posted_intr.h> -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector); -void vmx_pi_start_bypass(struct kvm *kvm); +void vt_pi_start_bypass(struct kvm *kvm); static inline int pi_find_highest_vector(struct pi_desc *pi_desc) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3fbe08907afe..271c2f94b8dd 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -714,7 +714,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm))) return; @@ -739,7 +739,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT, * which passes the interrupt blocked flag. */ - return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || !to_tdx(vcpu)->vp_enter_args.r12; } @@ -757,7 +757,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) * otherwise the interrupt would have been serviced at the instruction * boundary. */ - if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || to_tdx(vcpu)->vp_enter_args.r12) return false; @@ -824,7 +824,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) void tdx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); tdx_prepare_switch_to_host(vcpu); } @@ -919,36 +919,51 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu) return EXIT_REASON_TDCALL; } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) { - struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; - - switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { + switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { case TDX_SUCCESS: case TDX_NON_RECOVERABLE_VCPU: case TDX_NON_RECOVERABLE_TD: case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE: case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE: - break; + return true; default: - return -1u; + return false; } +} + +static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +{ + struct vcpu_tdx *tdx = to_tdx(vcpu); + union vmx_exit_reason exit_reason; + + /* + * Return the synthesized invalid Exit Reason, as the TDX module + * never attempted to run the vCPU, i.e. the Exit Reason is undefined, + * but this is NOT a failed VM-Enter. + */ + if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) + return (union vmx_exit_reason) { + .basic = EXIT_REASON_UNDEFINED, + }; - exit_reason = tdx->vp_enter_ret; + exit_reason.full = (u32)tdx->vp_enter_ret; - switch (exit_reason) { + switch (exit_reason.basic) { case EXIT_REASON_TDCALL: if (tdvmcall_exit_type(vcpu)) - return EXIT_REASON_VMCALL; - - return tdcall_to_vmx_exit_reason(vcpu); + exit_reason.basic = EXIT_REASON_VMCALL; + else + exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu); + break; case EXIT_REASON_EPT_MISCONFIG: /* * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return -1u; + exit_reason.basic = EXIT_REASON_UNDEFINED; + break; default: break; } @@ -965,22 +980,21 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args); - vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu); + vt->exit_reason = tdx_to_vmx_exit_reason(vcpu); vt->exit_qualification = tdx->vp_enter_args.rcx; tdx->ext_exit_qualification = tdx->vp_enter_args.rdx; tdx->exit_gpa = tdx->vp_enter_args.r8; vt->exit_intr_info = tdx->vp_enter_args.r9; - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); guest_state_exit_irqoff(); } static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry && - vmx_get_exit_reason(vcpu).full != -1u; + return vt_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) @@ -1069,8 +1083,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * allowing vCPU entry to avoid contention with tdh_vp_enter() and * TDCALLs. */ - if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) + if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) { + /* + * The vCPU never entered the guest, but this looks like a + * handled exit to the caller. Synthesize an invalid exit + * reason so the previous exit's stale value isn't consumed + * a second time. + */ + vt->exit_reason.full = EXIT_REASON_UNDEFINED; return EXIT_FASTPATH_EXIT_HANDLED; + } trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT); @@ -1129,7 +1151,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu) static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu) { - u32 intr_info = vmx_get_intr_info(vcpu); + u32 intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -1890,7 +1912,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, struct vcpu_tdx *tdx = to_tdx(vcpu); /* TDX supports only posted interrupt. No lapic emulation. */ - __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); } @@ -1898,7 +1920,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu) { u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK; - u64 eq = vmx_get_exit_qual(vcpu); + u64 eq = vt_get_exit_qual(vcpu); if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION) return false; @@ -1934,7 +1956,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) /* Only private GPA triggers zero-step mitigation */ local_retry = true; } else { - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); /* * EPT violation due to instruction fetch should never be * triggered from shared memory in TDX guest. If such EPT @@ -1982,7 +2004,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) while (1) { struct kvm_memory_slot *slot; - ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual); + ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual); if (ret != RET_PF_RETRY || !local_retry) break; @@ -2016,7 +2038,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) return 1; } - if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) + if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu)); return 1; @@ -2027,7 +2049,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); if (fastpath != EXIT_FASTPATH_NONE) return 1; @@ -2121,6 +2143,11 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * - If it's not an MSMI, no need to do anything here. */ return 1; + case EXIT_REASON_NOTIFY: + /* NMI blocking state is handled by TDX module */ + return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu)); + case EXIT_REASON_BUS_LOCK: + return vt_handle_bus_lock_vmexit(vcpu); default: break; } @@ -2136,10 +2163,10 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_tdx *tdx = to_tdx(vcpu); *reason = tdx->vt.exit_reason.full; - if (*reason != -1u) { - *info1 = vmx_get_exit_qual(vcpu); + if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { + *info1 = vt_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); } else { *info1 = 0; *info2 = 0; @@ -3152,6 +3179,17 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc)); td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR); + if (kvm_notify_vmexit_enabled(vcpu->kvm)) { + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_NOTIFY_VM_EXITING); + td_vmcs_write32(tdx, NOTIFY_WINDOW, + vcpu->kvm->arch.notify_window); + } + + if (vcpu->kvm->arch.bus_lock_detection_enabled) + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_BUS_LOCK_DETECTION); + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; @@ -3217,6 +3255,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; + gpa_t nr_bytes, end_gpa; long gmem_ret; int ret; @@ -3235,10 +3274,13 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (!PAGE_ALIGNED(region.source_addr) || (!gmem_in_place_conversion && !region.source_addr) || - !PAGE_ALIGNED(region.gpa) || !region.nr_pages || - region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages) + return -EINVAL; + + if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || + check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) + !vt_is_tdx_private_gpa(kvm, end_gpa)) return -EINVAL; ret = 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 686e586f3290..b341af4c6057 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load_vmcs(vcpu, cpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); } void vmx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); vmx_prepare_switch_to_host(to_vmx(vcpu)); } @@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). */ - if (vmx_get_exit_reason(vcpu).enclave_mode) { + if (vt_get_exit_reason(vcpu).enclave_mode) { kvm_queue_exception(vcpu, UD_VECTOR); return X86EMUL_PROPAGATE_FAULT; } @@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); unsigned long rip, orig_rip; u32 instr_len; @@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) return skip_emulated_instruction(vcpu); } -static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +void vmx_clear_hlt(struct kvm_vcpu *vcpu) { /* * Ensure that we clear the HLT state in the VMCS. We don't need to @@ -4436,7 +4436,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (!vcpu->arch.apic->apicv_active) return -1; - __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); return 0; } @@ -5392,15 +5392,9 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { - unsigned long cr2 = vmx_get_exit_qual(vcpu); + unsigned long cr2 = vt_get_exit_qual(vcpu); if (vcpu->arch.apf.host_apf_flags) goto handle_pf; @@ -5448,7 +5442,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -5529,7 +5523,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) switch (ex_no) { case DB_VECTOR: - dr6 = vmx_get_exit_qual(vcpu); + dr6 = vt_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { /* @@ -5605,7 +5599,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -5694,7 +5688,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -5772,7 +5766,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) int dr, dr7, reg; int err = 1; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5868,7 +5862,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5877,7 +5871,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5898,7 +5892,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); @@ -5907,7 +5901,7 @@ static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and @@ -5935,7 +5929,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5981,7 +5975,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) static int handle_ept_violation(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); gpa_t gpa; /* @@ -6009,7 +6003,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); - return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); + return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); } static int handle_ept_misconfig(struct kvm_vcpu *vcpu) @@ -6172,7 +6166,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -6186,7 +6180,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -6277,23 +6271,9 @@ static int handle_encls(struct kvm_vcpu *vcpu) } #endif /* CONFIG_X86_SGX_KVM */ -static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling to - * vmx_handle_exit(). - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - static int handle_notify(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); - bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; - - ++vcpu->stat.notify_window_exits; + unsigned long exit_qual = vt_get_exit_qual(vcpu); /* * Notify VM exit happened while executing iret from NMI, @@ -6303,15 +6283,7 @@ static int handle_notify(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); - if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || - context_invalid) { - vcpu->run->exit_reason = KVM_EXIT_NOTIFY; - vcpu->run->notify.flags = context_invalid ? - KVM_NOTIFY_CONTEXT_INVALID : 0; - return 0; - } - - return 1; + return __vt_handle_notify(vcpu, exit_qual); } static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) @@ -6321,13 +6293,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } @@ -6387,7 +6359,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_VMFUNC] = handle_vmx_instruction, [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, [EXIT_REASON_ENCLS] = handle_encls, - [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, + [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, [EXIT_REASON_NOTIFY] = handle_notify, [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, [EXIT_REASON_TDCALL] = handle_tdx_instruction, @@ -6398,16 +6370,18 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); + void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); *reason = vmx->vt.exit_reason.full; - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); if (!(vmx->vt.exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(*intr_info)) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); else @@ -6692,10 +6666,10 @@ void dump_vmcs(struct kvm_vcpu *vcpu) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); u32 vectoring_info = vmx->idt_vectoring_info; u16 exit_handler_index; @@ -6851,24 +6825,6 @@ unexpected_vmexit: return 0; } -int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) -{ - int ret = __vmx_handle_exit(vcpu, exit_fastpath); - - /* - * Exit to user space when bus lock detected to inform that there is - * a bus lock in guest. - */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - return 0; - } - return ret; -} - void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { int tpr_threshold; @@ -7132,74 +7088,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. @@ -7239,7 +7127,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->nmi_known_unmasked) return; - exit_intr_info = vmx_get_intr_info(&vmx->vcpu); + exit_intr_info = vt_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* @@ -7406,14 +7294,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, * the fastpath even, all other exits must use the slow path. */ if (is_guest_mode(vcpu) && - vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) + vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) return EXIT_FASTPATH_NONE; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_wrmsr(vcpu); case EXIT_REASON_MSR_WRITE_IMM: - return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); case EXIT_REASON_PREEMPTION_TIMER: return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); @@ -7426,17 +7314,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, } } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) -{ - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); - x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); - kvm_after_interrupt(vcpu); -} - static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned int flags) { @@ -7461,15 +7338,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_enable_fb_clear(vmx); if (unlikely(vmx->fail)) { - vmx->vt.exit_reason.full = 0xdead; + vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; goto out; } vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); - if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) + if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); out: guest_state_exit_irqoff(); @@ -7606,7 +7483,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * checking. */ if (vcpu->arch.nested_run_pending && - !vmx_get_exit_reason(vcpu).failed_vmentry) + !vt_get_exit_reason(vcpu).failed_vmentry) ++vcpu->stat.nested_run; vcpu->arch.nested_run_pending = 0; @@ -7617,7 +7494,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) trace_kvm_exit(vcpu, KVM_ISA_VMX); - if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) + if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 5f505ad14dde..6c9dd5b2365d 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -297,43 +297,6 @@ struct kvm_vmx { u64 *pid_table; }; -static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) -{ - return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt); -} - -static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) -{ - return &(container_of(vt, struct vcpu_vmx, vt)->vcpu); -} - -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -{ - return to_vt(vcpu)->exit_reason; -} - -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - - return vt->exit_qualification; -} - -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - return vt->exit_intr_info; -} - void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 3351b91e924e..cd72684d075d 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); @@ -95,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection); int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); +void vmx_clear_hlt(struct kvm_vcpu *vcpu); void vmx_enable_nmi_window(struct kvm_vcpu *vcpu); void vmx_enable_irq_window(struct kvm_vcpu *vcpu); void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 390cf3512f42..32563a91fba9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9136,6 +9136,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, } kvm_set_mp_state(vcpu, mp_state->mp_state); + + /* + * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's + * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state + * other than HALTED. The hardware state is sticky across VM-Exit and + * VM-Enter and is not touched by any other ioctl, so a vCPU that halted + * with HLT-exiting disabled stays wedged even after userspace rewrites + * its registers, e.g. when a VMM emulates a machine reset. + */ + if (kvm_hlt_in_guest(vcpu->kvm) && + mp_state->mp_state != KVM_MP_STATE_HALTED) + kvm_x86_call(clear_hlt)(vcpu); + kvm_make_request(KVM_REQ_EVENT, vcpu); ret = 0; |