From a2237ae761d9baa0e814e61140ca4524e31eb92b Mon Sep 17 00:00:00 2001 From: Sergey Dyasli Date: Wed, 7 Jun 2017 08:20:12 +0100 Subject: xen: fix HYPERVISOR_dm_op() prototype Change the third parameter to be the required struct xen_dm_op_buf * instead of a generic void * (which blindly accepts any pointer). Signed-off-by: Sergey Dyasli Reviewed-by: Juergen Gross Reviewed-by: Stefano Stabellini Signed-off-by: Juergen Gross --- include/xen/arm/hypercall.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/xen/arm/hypercall.h b/include/xen/arm/hypercall.h index 73db4b2eeb89..b40485e54d80 100644 --- a/include/xen/arm/hypercall.h +++ b/include/xen/arm/hypercall.h @@ -39,6 +39,8 @@ #include #include +struct xen_dm_op_buf; + long privcmd_call(unsigned call, unsigned long a1, unsigned long a2, unsigned long a3, unsigned long a4, unsigned long a5); @@ -53,7 +55,8 @@ int HYPERVISOR_physdev_op(int cmd, void *arg); int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args); int HYPERVISOR_tmem_op(void *arg); int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type); -int HYPERVISOR_dm_op(domid_t domid, unsigned int nr_bufs, void *bufs); +int HYPERVISOR_dm_op(domid_t domid, unsigned int nr_bufs, + struct xen_dm_op_buf *bufs); int HYPERVISOR_platform_op_raw(void *arg); static inline int HYPERVISOR_platform_op(struct xen_platform_op *op) { -- cgit v1.2.3 From c48f64ab472389df6f48171899c9d337adfadc5b Mon Sep 17 00:00:00 2001 From: Anoob Soman Date: Wed, 7 Jun 2017 12:46:56 +0100 Subject: xen-evtchn: Bind dyn evtchn:qemu-dm interrupt to next online VCPU A HVM domian booting generates around 200K (evtchn:qemu-dm xen-dyn) interrupts,in a short period of time. All these evtchn:qemu-dm are bound to VCPU 0, until irqbalance sees these IRQ and moves it to a different VCPU. In one configuration, irqbalance runs every 10 seconds, which means irqbalance doesn't get to see these burst of interrupts and doesn't re-balance interrupts most of the time, making all evtchn:qemu-dm to be processed by VCPU0. This cause VCPU0 to spend most of time processing hardirq and very little time on softirq. Moreover, if dom0 kernel PREEMPTION is disabled, VCPU0 never runs watchdog (process context), triggering a softlockup detection code to panic. Binding evtchn:qemu-dm to next online VCPU, will spread hardirq processing evenly across different CPU. Later, irqbalance will try to balance evtchn:qemu-dm, if required. Signed-off-by: Anoob Soman Reviewed-by: Boris Ostrovsky Signed-off-by: Juergen Gross --- drivers/xen/events/events_base.c | 6 +++--- drivers/xen/evtchn.c | 34 +++++++++++++++++++++++++++++++++- include/xen/events.h | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index b52852f38cff..813f1e86a599 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -1303,10 +1303,9 @@ void rebind_evtchn_irq(int evtchn, int irq) } /* Rebind an evtchn so that it gets delivered to a specific cpu */ -static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu) +int xen_rebind_evtchn_to_cpu(int evtchn, unsigned tcpu) { struct evtchn_bind_vcpu bind_vcpu; - int evtchn = evtchn_from_irq(irq); int masked; if (!VALID_EVTCHN(evtchn)) @@ -1338,13 +1337,14 @@ static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu) return 0; } +EXPORT_SYMBOL_GPL(xen_rebind_evtchn_to_cpu); static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest, bool force) { unsigned tcpu = cpumask_first_and(dest, cpu_online_mask); - return rebind_irq_to_cpu(data->irq, tcpu); + return xen_rebind_evtchn_to_cpu(evtchn_from_irq(data->irq), tcpu); } static void enable_dynirq(struct irq_data *data) diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c index 10f1ef582659..9729a64ea1a9 100644 --- a/drivers/xen/evtchn.c +++ b/drivers/xen/evtchn.c @@ -421,6 +421,36 @@ static void evtchn_unbind_from_user(struct per_user_data *u, del_evtchn(u, evtchn); } +static DEFINE_PER_CPU(int, bind_last_selected_cpu); + +static void evtchn_bind_interdom_next_vcpu(int evtchn) +{ + unsigned int selected_cpu, irq; + struct irq_desc *desc; + unsigned long flags; + + irq = irq_from_evtchn(evtchn); + desc = irq_to_desc(irq); + + if (!desc) + return; + + raw_spin_lock_irqsave(&desc->lock, flags); + selected_cpu = this_cpu_read(bind_last_selected_cpu); + selected_cpu = cpumask_next_and(selected_cpu, + desc->irq_common_data.affinity, cpu_online_mask); + + if (unlikely(selected_cpu >= nr_cpu_ids)) + selected_cpu = cpumask_first_and(desc->irq_common_data.affinity, + cpu_online_mask); + + this_cpu_write(bind_last_selected_cpu, selected_cpu); + + /* unmask expects irqs to be disabled */ + xen_rebind_evtchn_to_cpu(evtchn, selected_cpu); + raw_spin_unlock_irqrestore(&desc->lock, flags); +} + static long evtchn_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -478,8 +508,10 @@ static long evtchn_ioctl(struct file *file, break; rc = evtchn_bind_to_user(u, bind_interdomain.local_port); - if (rc == 0) + if (rc == 0) { rc = bind_interdomain.local_port; + evtchn_bind_interdom_next_vcpu(rc); + } break; } diff --git a/include/xen/events.h b/include/xen/events.h index 88da2abaf535..f442ca5fcd82 100644 --- a/include/xen/events.h +++ b/include/xen/events.h @@ -58,6 +58,7 @@ void evtchn_put(unsigned int evtchn); void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector); void rebind_evtchn_irq(int evtchn, int irq); +int xen_rebind_evtchn_to_cpu(int evtchn, unsigned tcpu); static inline void notify_remote_via_evtchn(int port) { -- cgit v1.2.3 From 0b64ffb8db4e310f77a01079ca752d946a8526b5 Mon Sep 17 00:00:00 2001 From: Ankur Arora Date: Fri, 2 Jun 2017 17:05:59 -0700 Subject: xen/pvh*: Support > 32 VCPUs at domain restore When Xen restores a PVHVM or PVH guest, its shared_info only holds up to 32 CPUs. The hypercall VCPUOP_register_vcpu_info allows us to setup per-page areas for VCPUs. This means we can boot PVH* guests with more than 32 VCPUs. During restore the per-cpu structure is allocated freshly by the hypervisor (vcpu_info_mfn is set to INVALID_MFN) so that the newly restored guest can make a VCPUOP_register_vcpu_info hypercall. However, we end up triggering this condition in Xen: /* Run this command on yourself or on other offline VCPUS. */ if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) ) which means we are unable to setup the per-cpu VCPU structures for running VCPUS. The Linux PV code paths makes this work by iterating over cpu_possible in xen_vcpu_restore() with: 1) is target CPU up (VCPUOP_is_up hypercall?) 2) if yes, then VCPUOP_down to pause it 3) VCPUOP_register_vcpu_info 4) if it was down, then VCPUOP_up to bring it back up With Xen commit 192df6f9122d ("xen/x86: allow HVM guests to use hypercalls to bring up vCPUs") this is available for non-PV guests. As such first check if VCPUOP_is_up is actually possible before trying this dance. As most of this dance code is done already in xen_vcpu_restore() let's make it callable on PV, PVH and PVHVM. Based-on-patch-by: Konrad Wilk Reviewed-by: Boris Ostrovsky Signed-off-by: Ankur Arora Signed-off-by: Juergen Gross --- arch/x86/xen/enlighten.c | 45 +++++++++++++++++++++++++++++++------------- arch/x86/xen/enlighten_hvm.c | 20 +++++++------------- arch/x86/xen/smp_hvm.c | 10 ++++++++++ arch/x86/xen/suspend_hvm.c | 11 +++-------- include/xen/xen-ops.h | 2 ++ 5 files changed, 54 insertions(+), 34 deletions(-) (limited to 'include') diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 96b745e3f56c..276cc21619ec 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -106,6 +106,21 @@ int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), return rc >= 0 ? 0 : rc; } +static void xen_vcpu_setup_restore(int cpu) +{ + /* Any per_cpu(xen_vcpu) is stale, so reset it */ + xen_vcpu_info_reset(cpu); + + /* + * For PVH and PVHVM, setup online VCPUs only. The rest will + * be handled by hotplug. + */ + if (xen_pv_domain() || + (xen_hvm_domain() && cpu_online(cpu))) { + xen_vcpu_setup(cpu); + } +} + /* * On restore, set the vcpu placement up again. * If it fails, then we're in a bad state, since @@ -117,17 +132,23 @@ void xen_vcpu_restore(void) for_each_possible_cpu(cpu) { bool other_cpu = (cpu != smp_processor_id()); - bool is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, xen_vcpu_nr(cpu), - NULL); + bool is_up; + + if (xen_vcpu_nr(cpu) == XEN_VCPU_ID_INVALID) + continue; + + /* Only Xen 4.5 and higher support this. */ + is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, + xen_vcpu_nr(cpu), NULL) > 0; if (other_cpu && is_up && HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL)) BUG(); - xen_setup_runstate_info(cpu); + if (xen_pv_domain() || xen_feature(XENFEAT_hvm_safe_pvclock)) + xen_setup_runstate_info(cpu); - if (xen_have_vcpu_info_placement) - xen_vcpu_setup(cpu); + xen_vcpu_setup_restore(cpu); if (other_cpu && is_up && HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL)) @@ -163,11 +184,11 @@ void xen_vcpu_setup(int cpu) BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); /* - * This path is called twice on PVHVM - first during bootup via - * smp_init -> xen_hvm_cpu_notify, and then if the VCPU is being - * hotplugged: cpu_up -> xen_hvm_cpu_notify. - * As we can only do the VCPUOP_register_vcpu_info once lets - * not over-write its result. + * This path is called on PVHVM at bootup (xen_hvm_smp_prepare_boot_cpu) + * and at restore (xen_vcpu_restore). Also called for hotplugged + * VCPUs (cpu_init -> xen_hvm_cpu_prepare_hvm). + * However, the hypercall can only be done once (see below) so if a VCPU + * is offlined and comes back online then let's not redo the hypercall. * * For PV it is called during restore (xen_vcpu_restore) and bootup * (xen_setup_vcpu_info_placement). The hotplug mechanism does not @@ -178,8 +199,6 @@ void xen_vcpu_setup(int cpu) return; } - xen_vcpu_info_reset(cpu); - if (xen_have_vcpu_info_placement) { vcpup = &per_cpu(xen_vcpu_info, cpu); info.mfn = arbitrary_virt_to_mfn(vcpup); @@ -214,7 +233,7 @@ void xen_vcpu_setup(int cpu) if (!xen_have_vcpu_info_placement) { if (cpu >= MAX_VIRT_CPUS) clamp_max_cpus(); - return; + xen_vcpu_info_reset(cpu); } } diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index eb53da6547ee..ba1afadb2512 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -20,7 +20,6 @@ void __ref xen_hvm_init_shared_info(void) { - int cpu; struct xen_add_to_physmap xatp; static struct shared_info *shared_info_page; @@ -35,18 +34,6 @@ void __ref xen_hvm_init_shared_info(void) BUG(); HYPERVISOR_shared_info = (struct shared_info *)shared_info_page; - - /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info - * page, we use it in the event channel upcall and in some pvclock - * related functions. We don't need the vcpu_info placement - * optimizations because we don't use any pv_mmu or pv_irq op on - * HVM. - * When xen_hvm_init_shared_info is run at boot time only vcpu 0 is - * online but xen_hvm_init_shared_info is run at resume time too and - * in that case multiple vcpus might be online. */ - for_each_online_cpu(cpu) { - xen_vcpu_info_reset(cpu); - } } static void __init init_hvm_pv_info(void) @@ -150,6 +137,13 @@ static void __init xen_hvm_guest_init(void) xen_hvm_init_shared_info(); + /* + * xen_vcpu is a pointer to the vcpu_info struct in the shared_info + * page, we use it in the event channel upcall and in some pvclock + * related functions. + */ + xen_vcpu_info_reset(0); + xen_panic_handler_init(); if (xen_feature(XENFEAT_hvm_callback_vector)) diff --git a/arch/x86/xen/smp_hvm.c b/arch/x86/xen/smp_hvm.c index 9e0fb9a015d4..6c8a805819ff 100644 --- a/arch/x86/xen/smp_hvm.c +++ b/arch/x86/xen/smp_hvm.c @@ -28,10 +28,20 @@ static void __init xen_hvm_smp_prepare_boot_cpu(void) static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus) { + int cpu; + native_smp_prepare_cpus(max_cpus); WARN_ON(xen_smp_intr_init(0)); xen_init_lock_cpu(0); + + for_each_possible_cpu(cpu) { + if (cpu == 0) + continue; + + /* Set default vcpu_id to make sure that we don't use cpu-0's */ + per_cpu(xen_vcpu_id, cpu) = XEN_VCPU_ID_INVALID; + } } #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/x86/xen/suspend_hvm.c b/arch/x86/xen/suspend_hvm.c index 01afcadde50a..484999416d8b 100644 --- a/arch/x86/xen/suspend_hvm.c +++ b/arch/x86/xen/suspend_hvm.c @@ -8,15 +8,10 @@ void xen_hvm_post_suspend(int suspend_cancelled) { - int cpu; - - if (!suspend_cancelled) + if (!suspend_cancelled) { xen_hvm_init_shared_info(); + xen_vcpu_restore(); + } xen_callback_vector(); xen_unplug_emulated_devices(); - if (xen_feature(XENFEAT_hvm_safe_pvclock)) { - for_each_online_cpu(cpu) { - xen_setup_runstate_info(cpu); - } - } } diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index c44a2ee8c8f8..218e6aae5433 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -15,6 +15,8 @@ static inline uint32_t xen_vcpu_nr(int cpu) return per_cpu(xen_vcpu_id, cpu); } +#define XEN_VCPU_ID_INVALID U32_MAX + void xen_arch_pre_suspend(void); void xen_arch_post_suspend(int suspend_cancelled); -- cgit v1.2.3 From a714c2865f154a353f5cc3c81aeb8d8065934157 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Wed, 14 Jun 2017 19:23:51 +0200 Subject: xen: sync include/xen/interface/version.h Sync include/xen/interface/version.h with the Xen source. Signed-off-by: Juergen Gross Reviewed-by: Boris Ostrovsky Reviewed-by: Stefano Stabellini Signed-off-by: Juergen Gross --- include/xen/interface/version.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/xen/interface/version.h b/include/xen/interface/version.h index 7ff6498679a3..145f12f9ecec 100644 --- a/include/xen/interface/version.h +++ b/include/xen/interface/version.h @@ -63,4 +63,19 @@ struct xen_feature_info { /* arg == xen_domain_handle_t. */ #define XENVER_guest_handle 8 +#define XENVER_commandline 9 +struct xen_commandline { + char buf[1024]; +}; + +/* + * Return value is the number of bytes written, or XEN_Exx on error. + * Calling with empty parameter returns the size of build_id. + */ +#define XENVER_build_id 10 +struct xen_build_id { + uint32_t len; /* IN: size of buf[]. */ + unsigned char buf[]; +}; + #endif /* __XEN_PUBLIC_VERSION_H__ */ -- cgit v1.2.3