static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);

#endif

#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)

#define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE

ex->payload = payload;

}

{

u32 prev_nr;

int class1, class2;

kvm_make_request(KVM_REQ_EVENT, vcpu);

/*

*/

kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {

kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,

has_payload, payload);

if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {

queue:

vcpu->arch.exception.has_error_code = has_error;

vcpu->arch.exception.vector = nr;

vcpu->arch.exception.error_code = error_code;

void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)

{

}

EXPORT_SYMBOL_GPL(kvm_queue_exception);

void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,

unsigned long payload)

{

}

EXPORT_SYMBOL_GPL(kvm_queue_exception_p);

static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,

u32 error_code, unsigned long payload)

{

}

int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)

{

if (err)

void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)

{

}

EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

/*

* Checks if cpl <= required_cpl; if true, return true. Otherwise queue

* a #GP and return false.

if (vcpu->arch.xcr0 != kvm_host.xcr0)

xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

vcpu->arch.ia32_xss != kvm_host.xss)

wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);

}

vcpu->arch.pkru != vcpu->arch.host_pkru &&

((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||

kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))

}

EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);

kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {

vcpu->arch.pkru = rdpkru();

if (vcpu->arch.pkru != vcpu->arch.host_pkru)

}

if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {

if (vcpu->arch.xcr0 != kvm_host.xcr0)

xsetbv(XCR_XFEATURE_ENABLED_MASK, kvm_host.xcr0);

vcpu->arch.ia32_xss != kvm_host.xss)

wrmsrl(MSR_IA32_XSS, kvm_host.xss);

}

vcpu->arch.xcr0 = xcr0;

if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)

return 0;

}

}

EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);

static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)

{

return __kvm_is_valid_cr4(vcpu, cr4) &&

{

u64 fixed = DR6_FIXED_1;

fixed |= DR6_RTM;

fixed |= DR6_BUS_LOCK;

return fixed;

}

static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)

{

return false;

return false;

return false;

if (efer & (EFER_LME | EFER_LMA) &&

return false;

return false;

return true;

return 1;

if (!host_initiated &&

return 1;

/*

return 1;

if (!host_initiated &&

return 1;

break;

}

static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)

{

return kvm_handle_invalid_op(vcpu);

pr_warn_once("%s instruction emulated as NOP!\n", insn);

return kvm_emulate_as_nop(vcpu);

}

static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)

{

trace_kvm_write_tsc_offset(vcpu->vcpu_id,

vcpu->arch.l1_tsc_offset,

l1_offset);

* participates in.

*/

static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,

{

struct kvm *kvm = vcpu->kvm;

lockdep_assert_held(&kvm->arch.tsc_write_lock);

/*

* We also track th most recent recorded KHZ, write and time to

* allow the matching interval to be extended at each write.

}

}

/*

* For a reliable TSC, we can match TSC offsets, and for an unstable

matched = true;

}

raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);

}

return data.clock;

}

struct gfn_to_pfn_cache *gpc,

{

struct pvclock_vcpu_time_info *guest_hv_clock;

unsigned long flags;

read_lock_irqsave(&gpc->lock, flags);

while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) {

read_unlock_irqrestore(&gpc->lock, flags);

* it is consistent.

*/

smp_wmb();

/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */

smp_wmb();

kvm_gpc_mark_dirty_in_slot(gpc);

read_unlock_irqrestore(&gpc->lock, flags);

}

{

unsigned long flags, tgt_tsc_khz;

unsigned seq;

struct kvm_vcpu_arch *vcpu = &v->arch;

struct kvm_arch *ka = &v->kvm->arch;

s64 kernel_ns;

u64 tsc_timestamp, host_tsc;

bool use_master_clock;

kernel_ns = 0;

host_tsc = 0;

if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {

kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,

vcpu->hw_tsc_khz = tgt_tsc_khz;

}

vcpu->last_guest_tsc = tsc_timestamp;

/* If the host uses TSC clocksource, then it is stable */

if (use_master_clock)

#ifdef CONFIG_KVM_XEN

if (vcpu->xen.vcpu_info_cache.active)

if (vcpu->xen.vcpu_time_info_cache.active)

#endif

return 0;

}

sizeof(u64)))

return 1;

vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;

kvm_async_pf_wakeup_all(vcpu);

u32 msr = msr_info->index;

u64 data = msr_info->data;

return kvm_xen_write_hypercall_page(vcpu, data);

switch (msr) {

break;

case MSR_IA32_ARCH_CAPABILITIES:

if (!msr_info->host_initiated ||

return KVM_MSR_RET_UNSUPPORTED;

vcpu->arch.arch_capabilities = data;

break;

case MSR_IA32_PERF_CAPABILITIES:

if (!msr_info->host_initiated ||

return KVM_MSR_RET_UNSUPPORTED;

if (data & ~kvm_caps.supported_perf_cap)

if ((!guest_has_pred_cmd_msr(vcpu)))

return 1;

reserved_bits |= PRED_CMD_IBPB;

reserved_bits |= PRED_CMD_SBPB;

}

}

case MSR_IA32_FLUSH_CMD:

if (!msr_info->host_initiated &&

return 1;

if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH))

kvm_set_lapic_tscdeadline_msr(vcpu, data);

break;

case MSR_IA32_TSC_ADJUST:

if (!msr_info->host_initiated) {

s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;

adjust_tsc_offset_guest(vcpu, adj);

if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&

((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {

return 1;

vcpu->arch.ia32_misc_enable_msr = data;

} else {

vcpu->arch.ia32_misc_enable_msr = data;

}

case MSR_IA32_TSC:

if (msr_info->host_initiated) {

kvm_synchronize_tsc(vcpu, &data);

u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;

adjust_tsc_offset_guest(vcpu, adj);

vcpu->arch.ia32_tsc_adjust_msr += adj;

if (data & ~kvm_caps.supported_xss)

return 1;

vcpu->arch.ia32_xss = data;

break;

case MSR_SMI_COUNT:

if (!msr_info->host_initiated)

kvm_pr_unimpl_wrmsr(vcpu, msr, data);

break;

case MSR_AMD64_OSVW_ID_LENGTH:

return 1;

vcpu->arch.osvw.length = data;

break;

case MSR_AMD64_OSVW_STATUS:

return 1;

vcpu->arch.osvw.status = data;

break;

#ifdef CONFIG_X86_64

case MSR_IA32_XFD:

if (!msr_info->host_initiated &&

return 1;

if (data & ~kvm_guest_supported_xfd(vcpu))

break;

case MSR_IA32_XFD_ERR:

if (!msr_info->host_initiated &&

return 1;

if (data & ~kvm_guest_supported_xfd(vcpu))

msr_info->data = vcpu->arch.microcode_version;

break;

case MSR_IA32_ARCH_CAPABILITIES:

return KVM_MSR_RET_UNSUPPORTED;

msr_info->data = vcpu->arch.arch_capabilities;

break;

case MSR_IA32_PERF_CAPABILITIES:

return KVM_MSR_RET_UNSUPPORTED;

msr_info->data = vcpu->arch.perf_capabilities;

break;

msr_info->data = 0xbe702111;

break;

case MSR_AMD64_OSVW_ID_LENGTH:

return 1;

msr_info->data = vcpu->arch.osvw.length;

break;

case MSR_AMD64_OSVW_STATUS:

return 1;

msr_info->data = vcpu->arch.osvw.status;

break;

#ifdef CONFIG_X86_64

case MSR_IA32_XFD:

if (!msr_info->host_initiated &&

return 1;

msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;

break;

case MSR_IA32_XFD_ERR:

if (!msr_info->host_initiated &&

return 1;

msr_info->data = vcpu->arch.guest_fpu.xfd_err;

boot_cpu_has(X86_FEATURE_ARAT);

}

#ifdef CONFIG_KVM_HYPERV

static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,

struct kvm_cpuid2 __user *cpuid_arg)

return type < 32 && (kvm_caps.supported_vm_types & BIT(type));

}

int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)

{

int r = 0;

break;

#endif

case KVM_CAP_SYNC_REGS:

break;

case KVM_CAP_ADJUST_CLOCK:

r = KVM_CLOCK_VALID_FLAGS;

break;

case KVM_CAP_X86_DISABLE_EXITS:

break;

case KVM_CAP_X86_SMM:

if (!IS_ENABLED(CONFIG_KVM_SMM))

u64 offset = kvm_compute_l1_tsc_offset(vcpu,

vcpu->arch.last_guest_tsc);

kvm_vcpu_write_tsc_offset(vcpu, offset);

}

if (kvm_lapic_hv_timer_in_use(vcpu))

tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;

ns = get_kvmclock_base_ns();

raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);

r = 0;

case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:

vcpu->arch.pv_cpuid.enforce = cap->args[0];

return 0;

default:

return -EINVAL;

break;

case KVM_CAP_X86_DISABLE_EXITS:

r = -EINVAL;

break;

#define SMT_RSB_MSG "This processor is affected by the Cross-Thread Return Predictions vulnerability. " \

"KVM_CAP_X86_DISABLE_EXITS should only be used with SMT disabled or trusted guests."

r = 0;

break;

case KVM_CAP_MSR_PLATFORM_INFO:

kvm->arch.guest_can_read_msr_platform_info = cap->args[0];

{

struct kvm_vcpu *vcpu;

unsigned long i;

}

int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)

static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)

{

}

static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)

{

}

static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)

{

}

static bool emulator_guest_cpuid_is_intel_compatible(struct x86_emulate_ctxt *ctxt)

}

EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit);

static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)

{

struct kvm *kvm = vcpu->kvm;

if (r == X86EMUL_RETRY_INSTR || r == X86EMUL_PROPAGATE_FAULT)

return 1;

WARN_ON_ONCE(r != X86EMUL_UNHANDLEABLE);

return handle_emulation_failure(vcpu, emulation_type);

}

if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))

kvm_caps.supported_xss = 0;

if (kvm_caps.has_tsc_control) {

/*

* Make sure the user can only configure tsc_khz values that

if (!is_64_bit_hypercall(vcpu))

ret = (u32)ret;

kvm_rax_write(vcpu, ret);

return kvm_skip_emulated_instruction(vcpu);

}

{

unsigned long ret;

trace_kvm_hypercall(nr, a0, a1, a2, a3);

if (!op_64_bit) {

u64 gpa = a0, npages = a1, attrs = a2;

ret = -KVM_ENOSYS;

break;

if (!PAGE_ALIGNED(gpa) || !npages ||

vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;

vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;

vcpu->run->hypercall.args[0] = gpa;

vcpu->run->hypercall.args[1] = npages;

vcpu->run->hypercall.args[2] = attrs;

vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;

WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);

return 0;

}

default:

}

out:

}

int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)

{

if (kvm_xen_hypercall_enabled(vcpu->kvm))

return kvm_xen_hypercall(vcpu);

if (kvm_hv_hypercall_enabled(vcpu))

return kvm_hv_hypercall(vcpu);

}

EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

set_debugreg(vcpu->arch.eff_db[1], 1);

set_debugreg(vcpu->arch.eff_db[2], 2);

set_debugreg(vcpu->arch.eff_db[3], 3);

} else if (unlikely(hw_breakpoint_active())) {

set_debugreg(0, 7);

}

guest_timing_enter_irqoff();

for (;;) {

/*

* Profile KVM exit RIPs:

*/

unsigned long rip = kvm_rip_read(vcpu);

profile_hit(KVM_PROFILING, (void *)rip);

}

switch(vcpu->arch.mp_state) {

case KVM_MP_STATE_HALTED:

case KVM_MP_STATE_AP_RESET_HOLD:

fallthrough;

case KVM_MP_STATE_RUNNABLE:

vcpu->arch.apf.halted = false;

++vcpu->stat.halt_exits;

if (lapic_in_kernel(vcpu)) {

if (kvm_vcpu_has_events(vcpu))

return 1;

} else {

vcpu->run->exit_reason = reason;

{

struct kvm_queued_exception *ex = &vcpu->arch.exception;

struct kvm_run *kvm_run = vcpu->run;

int r;

vcpu_load(vcpu);

kvm_sigset_activate(vcpu);

kvm_run->flags = 0;

goto out;

}

r = -EINVAL;

goto out;

}

out:

kvm_put_guest_fpu(vcpu);

store_regs(vcpu);

post_kvm_run_save(vcpu);

kvm_vcpu_srcu_read_unlock(vcpu);

if (kvm_mpx_supported())

kvm_load_guest_fpu(vcpu);

r = kvm_apic_accept_events(vcpu);

if (r < 0)

goto out;

mp_state->mp_state = vcpu->arch.mp_state;

out:

if (kvm_mpx_supported())

kvm_put_guest_fpu(vcpu);

vcpu_put(vcpu);

goto out;

if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {

set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);

} else

kvm_make_request(KVM_REQ_EVENT, vcpu);

ret = 0;

if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&

sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&

!is_protmode(vcpu))

return 0;

}

kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm);

if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))

else

r = kvm_mmu_create(vcpu);

if (r < 0)

goto free_emulate_ctxt;

}

kvm_async_pf_hash_reset(vcpu);

if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) {

kvm_xen_init_vcpu(vcpu);

vcpu_load(vcpu);

kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);

kvm_vcpu_reset(vcpu, false);

kvm_init_mmu(vcpu);

{

int idx;

kvmclock_reset(vcpu);

kvm_x86_call(vcpu_free)(vcpu);

"does not run without ignore_msrs=1, please report it to kvm@vger.kernel.org.\n");

}

return 0;

out_uninit_mmu:

return ret;

}

/**

* __x86_set_memory_region: Setup KVM internal memory slot

*

struct kvm_memslots *slots = kvm_memslots(kvm);

struct kvm_memory_slot *slot;

if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))

return ERR_PTR_USR(-EINVAL);

m.guest_phys_addr = gpa;

m.userspace_addr = hva;

m.memory_size = size;

if (r < 0)

return ERR_PTR_USR(r);

}

void kvm_arch_pre_destroy_vm(struct kvm *kvm)

{

kvm_mmu_pre_destroy_vm(kvm);

}

__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);

mutex_unlock(&kvm->slots_lock);

}

kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));

kvm_pic_destroy(kvm);

kvm_ioapic_destroy(kvm);

kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));

kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));

kvm_mmu_uninit_vm(kvm);

kvm_page_track_cleanup(kvm);

kvm_xen_destroy_vm(kvm);

kvm_hv_destroy_vm(kvm);

}

static void memslot_rmap_free(struct kvm_memory_slot *slot)

/*

* Clear out the previous array pointers for the KVM_MR_MOVE case. The

* the new memslot is successful.

*/

memset(&slot->arch, 0, sizeof(slot->arch));

if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn())

return -EINVAL;

return kvm_alloc_memslot_metadata(kvm, new);

}

if (!kvm_pv_async_pf_enabled(vcpu))

return false;

return false;

if (is_guest_mode(vcpu)) {

}

vcpu->arch.apf.halted = false;

}

void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)

}

EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,

struct irq_bypass_producer *prod)

{

struct kvm_kernel_irqfd *irqfd =

container_of(cons, struct kvm_kernel_irqfd, consumer);

int ret;

kvm_arch_start_assignment(irqfd->kvm);

ret = kvm_x86_call(pi_update_irte)(irqfd->kvm,

prod->irq, irqfd->gsi, 1);

if (ret)

kvm_arch_end_assignment(irqfd->kvm);

return ret;

}

int ret;

struct kvm_kernel_irqfd *irqfd =

container_of(cons, struct kvm_kernel_irqfd, consumer);

WARN_ON(irqfd->producer != prod);

/*

* When producer of consumer is unregistered, we change back to

* when the irq is masked/disabled or the consumer side (KVM

* int this case doesn't want to receive the interrupts.

*/

ret = kvm_x86_call(pi_update_irte)(irqfd->kvm,

prod->irq, irqfd->gsi, 0);

if (ret)

printk(KERN_INFO "irq bypass consumer (token %p) unregistration"

" fails: %d\n", irqfd->consumer.token, ret);

kvm_arch_end_assignment(irqfd->kvm);

}

bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,

struct kvm_kernel_irq_routing_entry *new)

{

return true;

return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));