x86/xen: add xen hypercall preemption
Subsystems (6)
v5 cover x86/xen: add xen hypercall preemption
From: "Luis R. Rodriguez" <[email protected]>
This v5 nukes tracing as David said it was useless, it also only adds support for 64-bit as its the only thing I can test, and slightly modifies the documentation in code as to why we want this. The no krobe thing is left in place as I haven't heard confirmation its kosher to remove it.
Luis R. Rodriguez (2):
x86/xen: add xen_is_preemptible_hypercall() x86/xen: allow privcmd hypercalls to be preempted on 64-bit
arch/arm/include/asm/xen/hypercall.h | 5 +++++ arch/x86/include/asm/xen/hypercall.h | 20 ++++++++++++++++++++ arch/x86/kernel/entry_64.S | 2 ++ arch/x86/xen/enlighten.c | 7 +++++++ arch/x86/xen/xen-head.S | 18 +++++++++++++++++- drivers/xen/events/events_base.c | 14 ++++++++++++++ include/xen/events.h | 1 + 7 files changed, 66 insertions(+), 1 deletion(-)
Signature · 1 line
2.1.1
PATCH v5 1/2 xen: add xen_is_preemptible_hypercall()
From: "Luis R. Rodriguez" <[email protected]>
On kernels with voluntary or no preemption we can run into situations where a hypercall issued through userspace will linger around as it addresses sub-operatiosn in kernel context (multicalls). Such operations can trigger soft lockup detection.
We want to address a way to let the kernel voluntarily preempt such calls even on non preempt kernels, to address this we first need to distinguish which hypercalls fall under this category. This implements xen_is_preemptible_hypercall() which lets us do just that by adding a secondary hypercall page, calls made via the new page may be preempted.
This will only be used on x86 for now, on arm we just have a stub to always return false for now.
Andrew had originally submitted a version of this work [0].
[0] http://lists.xen.org/archives/html/xen-devel/2014-02/msg01056.html
Based on original work by: Andrew Cooper <[email protected]>
--- arch/arm/include/asm/xen/hypercall.h | 5 +++++ arch/x86/include/asm/xen/hypercall.h | 20 ++++++++++++++++++++ arch/x86/xen/enlighten.c | 7 +++++++ arch/x86/xen/xen-head.S | 18 +++++++++++++++++- 4 files changed, 49 insertions(+), 1 deletion(-)
arch/arm/include/asm/xen/hypercall.h +5 -0
@@ -74,4 +74,9 @@ MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req, BUG(); } +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return false; +} +#endif /* _ASM_ARM_XEN_HYPERCALL_H */
arch/x86/include/asm/xen/hypercall.h +20 -0
@@ -84,6 +84,22 @@extern struct {char _entry[32 ]; } hypercall_page[]; +#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ; +} +#else +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return false; +} +#endif +#define __HYPERCALL "call hypercall_page+%c[offset]"#define __HYPERCALL_ENTRY (x) \ [offset] "i" (__HYPERVISOR_##x * sizeof(hypercall_page[0]))
@@ -215,7 +231,11 @@ privcmd_call(unsigned call, asmvolatile ("call *%[call]" : __HYPERCALL_5PARAM +#ifndef CONFIG_PREEMPT + : [call]"a" (&preemptible_hypercall_page[call]) +#else : [call] "a" (&hypercall_page[call]) +#endif : __HYPERCALL_CLOBBER5);return (long )__res;
arch/x86/xen/enlighten.c +7 -0
@@ -84,6 +84,9 @@#include "multicalls.h" EXPORT_SYMBOL_GPL (hypercall_page); +#ifndef CONFIG_PREEMPT +EXPORT_SYMBOL_GPL (preemptible_hypercall_page); +#endif /* * Pointer to the xen_vcpu_info structure or
@@ -1531,6 +1534,10 @@ asmlinkage __visible void __init xen_start_kernel(void)#endif xen_setup_machphys_mapping (); +#ifndef CONFIG_PREEMPT +copy_page (preemptible_hypercall_page, hypercall_page); +#endif +/* Install Xen paravirt ops */ pv_info = xen_info; pv_init_ops = xen_init_ops;
arch/x86/xen/xen-head.S +17 -1
@@ -85,9 +85,18 @@ ENTRY(xen_pvh_early_cpu_init) .pushsection .text .balign PAGE_SIZE ENTRY(hypercall_page) + +#ifdef CONFIG_PREEMPT +# define PREEMPT_HYPERCALL_ENTRY(x) +#else +# define PREEMPT_HYPERCALL_ENTRY(x) \ + .global xen_hypercall_##x ## _p ASM_NL \ + .set preemptible_xen_hypercall_##x, xen_hypercall_##x + PAGE_SIZE ASM_NL +#endif #define NEXT_HYPERCALL(x) \ ENTRY(xen_hypercall_##x) \ - .skip 32 + .skip 32 ASM_NL \ + PREEMPT_HYPERCALL_ENTRY(x) NEXT_HYPERCALL(set_trap_table) NEXT_HYPERCALL(mmu_update)
@@ -138,6 +147,13 @@ NEXT_HYPERCALL(arch_4) NEXT_HYPERCALL(arch_5) NEXT_HYPERCALL(arch_6) .balign PAGE_SIZE + +#ifndef CONFIG_PREEMPT +ENTRY(preemptible_hypercall_page) + .skip PAGE_SIZE +#endif /* CONFIG_PREEMPT */ + +#undef NEXT_HYPERCALL .popsection ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux")
Signature · 1 line
2.1.1
Re: [Xen-devel] [PATCH v5 1/2] xen: add xen_is_preemptible_hypercall()
arch/x86/include/asm/xen/hypercall.h line 94
Quoted 9 lines · Luis R. Rodriguez
arch/x86/include/asm/xen/hypercall.h +8
-0+#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ;
I asked for this test to be optimized.
return (regs->ip >> PAGE_SHIFT)
== ((unsigned long)preemptible_hypercall_page >> PAGE_SHIFT)
&& !user_mode_vm(regs);David
Re: [Xen-devel] [PATCH v5 1/2] xen: add xen_is_preemptible_hypercall()
Quoted 13 lines · David Vrabel
arch/x86/include/asm/xen/hypercall.h line 94
Quoted 9 lines · Luis R. Rodriguez
arch/x86/include/asm/xen/hypercall.h +8
-0+#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ;I asked for this test to be optimized.
Quoted 3 lines
return (regs->ip >> PAGE_SHIFT)
== ((unsigned long)preemptible_hypercall_page >> PAGE_SHIFT)
&& !user_mode_vm(regs);It didn't seem you were so sure about it and I asked some clarifications about it but you had not replied:
http://www.gossamer-threads.com/lists/xen/devel/363565?page=last
Luis
Re: [PATCH v5 1/2] xen: add xen_is_preemptible_hypercall()
In reply to David VrabelQuoted 13 lines · David Vrabel
arch/x86/include/asm/xen/hypercall.h line 94
Quoted 9 lines · Luis R. Rodriguez
arch/x86/include/asm/xen/hypercall.h +8
-0+#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ;I asked for this test to be optimized.
Quoted 3 lines
return (regs->ip >> PAGE_SHIFT)
== ((unsigned long)preemptible_hypercall_page >> PAGE_SHIFT)
&& !user_mode_vm(regs);It didn't seem you were so sure about it and I asked some clarifications about it but you had not replied:
http://www.gossamer-threads.com/lists/xen/devel/363565?page=last
Luis
Re: [PATCH v5 1/2] xen: add xen_is_preemptible_hypercall()
In reply to Luis R. Rodriguezarch/x86/include/asm/xen/hypercall.h line 94
Quoted 9 lines · Luis R. Rodriguez
arch/x86/include/asm/xen/hypercall.h +8
-0+#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ;
I asked for this test to be optimized.
return (regs->ip >> PAGE_SHIFT)
== ((unsigned long)preemptible_hypercall_page >> PAGE_SHIFT)
&& !user_mode_vm(regs);David
PATCH v5 1/2 xen: add xen_is_preemptible_hypercall()
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
On kernels with voluntary or no preemption we can run into situations where a hypercall issued through userspace will linger around as it addresses sub-operatiosn in kernel context (multicalls). Such operations can trigger soft lockup detection.
We want to address a way to let the kernel voluntarily preempt such calls even on non preempt kernels, to address this we first need to distinguish which hypercalls fall under this category. This implements xen_is_preemptible_hypercall() which lets us do just that by adding a secondary hypercall page, calls made via the new page may be preempted.
This will only be used on x86 for now, on arm we just have a stub to always return false for now.
Andrew had originally submitted a version of this work [0].
[0] http://lists.xen.org/archives/html/xen-devel/2014-02/msg01056.html
Based on original work by: Andrew Cooper <[email protected]>
--- arch/arm/include/asm/xen/hypercall.h | 5 +++++ arch/x86/include/asm/xen/hypercall.h | 20 ++++++++++++++++++++ arch/x86/xen/enlighten.c | 7 +++++++ arch/x86/xen/xen-head.S | 18 +++++++++++++++++- 4 files changed, 49 insertions(+), 1 deletion(-)
arch/arm/include/asm/xen/hypercall.h +5 -0
@@ -74,4 +74,9 @@ MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req, BUG(); } +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return false; +} +#endif /* _ASM_ARM_XEN_HYPERCALL_H */
arch/x86/include/asm/xen/hypercall.h +20 -0
@@ -84,6 +84,22 @@extern struct {char _entry[32 ]; } hypercall_page[]; +#ifndef CONFIG_PREEMPT +extern struct {char _entry[32 ]; } preemptible_hypercall_page[]; + +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return !user_mode_vm (regs) && + regs->ip >= (unsigned long )preemptible_hypercall_page && + regs->ip < (unsigned long )preemptible_hypercall_page +PAGE_SIZE ; +} +#else +static inline bool xen_is_preemptible_hypercall (struct pt_regs *regs) +{ +return false; +} +#endif +#define __HYPERCALL "call hypercall_page+%c[offset]"#define __HYPERCALL_ENTRY (x) \ [offset] "i" (__HYPERVISOR_##x * sizeof(hypercall_page[0]))
@@ -215,7 +231,11 @@ privcmd_call(unsigned call, asmvolatile ("call *%[call]" : __HYPERCALL_5PARAM +#ifndef CONFIG_PREEMPT + : [call]"a" (&preemptible_hypercall_page[call]) +#else : [call] "a" (&hypercall_page[call]) +#endif : __HYPERCALL_CLOBBER5);return (long )__res;
arch/x86/xen/enlighten.c +7 -0
@@ -84,6 +84,9 @@#include "multicalls.h" EXPORT_SYMBOL_GPL (hypercall_page); +#ifndef CONFIG_PREEMPT +EXPORT_SYMBOL_GPL (preemptible_hypercall_page); +#endif /* * Pointer to the xen_vcpu_info structure or
@@ -1531,6 +1534,10 @@ asmlinkage __visible void __init xen_start_kernel(void)#endif xen_setup_machphys_mapping (); +#ifndef CONFIG_PREEMPT +copy_page (preemptible_hypercall_page, hypercall_page); +#endif +/* Install Xen paravirt ops */ pv_info = xen_info; pv_init_ops = xen_init_ops;
arch/x86/xen/xen-head.S +17 -1
@@ -85,9 +85,18 @@ ENTRY(xen_pvh_early_cpu_init) .pushsection .text .balign PAGE_SIZE ENTRY(hypercall_page) + +#ifdef CONFIG_PREEMPT +# define PREEMPT_HYPERCALL_ENTRY(x) +#else +# define PREEMPT_HYPERCALL_ENTRY(x) \ + .global xen_hypercall_##x ## _p ASM_NL \ + .set preemptible_xen_hypercall_##x, xen_hypercall_##x + PAGE_SIZE ASM_NL +#endif #define NEXT_HYPERCALL(x) \ ENTRY(xen_hypercall_##x) \ - .skip 32 + .skip 32 ASM_NL \ + PREEMPT_HYPERCALL_ENTRY(x) NEXT_HYPERCALL(set_trap_table) NEXT_HYPERCALL(mmu_update)
@@ -138,6 +147,13 @@ NEXT_HYPERCALL(arch_4) NEXT_HYPERCALL(arch_5) NEXT_HYPERCALL(arch_6) .balign PAGE_SIZE + +#ifndef CONFIG_PREEMPT +ENTRY(preemptible_hypercall_page) + .skip PAGE_SIZE +#endif /* CONFIG_PREEMPT */ + +#undef NEXT_HYPERCALL .popsection ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux")
Signature · 1 line
2.1.1
PATCH v5 2/2 x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
Xen has support for splitting heavy work work into a series of hypercalls, called multicalls, and preempting them through what Xen calls continuation [0]. Despite this though without CONFIG_PREEMPT preemption won't happen, without preemption a system can become pretty useless on heavy handed hypercalls. Such is the case for example when creating a > 50 GiB HVM guest, we can get softlockups [1] with:.
kernel: [ 802.084335] BUG: soft lockup - CPU#1 stuck for 22s! [xend:31351]
The softlock up triggers on the TASK_UNINTERRUPTIBLE hanger check (default 120 seconds), on the Xen side in this particular case this happens when the following Xen hypervisor code is used:
xc_domain_set_pod_target() -->
do_memory_op() -->
arch_memory_op() -->
p2m_pod_set_mem_target()
-- long delay (real or emulated) --This happens on arch_memory_op() on the XENMEM_set_pod_target memory op even though arch_memory_op() can handle continuation via hypercall_create_continuation() for example.
Machines over 50 GiB of memory are on high demand and hard to come by so to help replicate this sort of issue long delays on select hypercalls have been emulated in order to be able to test this on smaller machines [2].
On one hand this issue can be considered as expected given that CONFIG_PREEMPT=n is used however we have forced voluntary preemption precedent practices in the kernel even for CONFIG_PREEMPT=n through the usage of cond_resched() sprinkled in many places. To address this issue with Xen hypercalls though we need to find a way to aid to the schedular in the middle of hypercalls. We are motivated to address this issue on CONFIG_PREEMPT=n as otherwise the system becomes rather unresponsive for long periods of time; in the worst case, at least only currently by emulating long delays on select io disk bound hypercalls, this can lead to filesystem corruption if the delay happens for example on SCHEDOP_remote_shutdown (when we call 'xl <domain> shutdown').
We can address this problem by trying to check if we should schedule on the xen timer in the middle of a hypercall on the return from the timer interrupt. We want to be careful to not always force voluntary preemption though so to do this we only selectively enable preemption on very specific xen hypercalls.
This enables hypercall preemption by selectively forcing checks for voluntary preempting only on ioctl initiated private hypercalls where we know some folks have run into reported issues [1].
[0] http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=42217cbc5b3e84b8c145d8cfb62dd5de0134b9e8;hp=3a0b9c57d5c9e82c55dd967c84dd06cb43c49ee9
[1] https://bugzilla.novell.com/show_bug.cgi?id=861093
[2] http://ftp.suse.com/pub/people/mcgrof/xen/emulate-long-xen-hypercalls.patch
Based on original work by: David Vrabel <[email protected]>
--- arch/x86/kernel/entry_64.S | 2 ++ drivers/xen/events/events_base.c | 14 ++++++++++++++ include/xen/events.h | 1 + 3 files changed, 17 insertions(+)
arch/x86/kernel/entry_64.S +2 -0
@@ -1198,6 +1198,8 @@ ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs) popq %rsp CFI_DEF_CFA_REGISTER rsp decl PER_CPU_VAR(irq_count) + movq %rsp, %rdi /* pass pt_regs as first argument */ + call xen_end_upcall jmp error_exit CFI_ENDPROC END(xen_do_hypervisor_callback)
drivers/xen/events/events_base.c +14 -0
@@ -32,6 +32,8 @@#include <linux/slab.h> #include <linux/irqnr.h> #include <linux/pci.h> +#include <linux/sched.h> +#include <linux/kprobes.h> #ifdef CONFIG_X86 #include <asm/desc.h>
@@ -1243,6 +1245,18 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)set_irq_regs (old_regs); } +/* +* Some hypercalls issued by the toolstack can take many 10s of +* seconds. Allow tasks running hypercalls via the privcmd driver to be +* voluntarily preempted even if full kernel preemption is disabled. +*/ +void xen_end_upcall (struct pt_regs *regs) +{ +if (xen_is_preemptible_hypercall (regs)) +_cond_resched (); +} +NOKPROBE_SYMBOL (xen_end_upcall); +void xen_hvm_evtchn_do_upcall (void ) {__xen_evtchn_do_upcall ();
include/xen/events.h +1 -0
@@ -95,6 +95,7 @@ void xen_hvm_callback_vector(void);extern int xen_have_vector_callback;int xen_set_callback_via (uint64_t via);void xen_evtchn_do_upcall (struct pt_regs *regs); +void xen_end_upcall (struct pt_regs *regs);void xen_hvm_evtchn_do_upcall (void );/* Bind a pirq for a physical interrupt to an irq. */
Signature · 1 line
2.1.1
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Jan
Re: [Xen-devel] [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
Quoted 6 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
There are still legitimate reasons to prefer 32bit PV guests over 64bit ones. Previous versions of this patch had 32bit support as well. Why did you drop it?
~Andrew
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Jan BeulichQuoted 6 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
There are still legitimate reasons to prefer 32bit PV guests over 64bit ones. Previous versions of this patch had 32bit support as well. Why did you drop it?
~Andrew
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Jan BeulichQuoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
David
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
Quoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
The hunk for 32-bit should indeed only go in once we get a Tested-by. Please let me know if there is anything else needed for this patch.
Luis
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to David VrabelQuoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
The hunk for 32-bit should indeed only go in once we get a Tested-by. Please let me know if there is anything else needed for this patch.
Luis
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to David VrabelQuoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
Hey folks, what is the status of these patches? Any more feedback?
Luis
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
Quoted 10 lines · Luis R. Rodriguez
Quoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 8 lines
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
Hey folks, what is the status of these patches? Any more feedback?
As I waited I decided to finally install a 32-bit OS but I found now that as of Xen commit 5d1181a5 which went upstream as of xen 4.3 Xen no longer supports 32-bit for x86. So -- Xen no loner even builds on 32-bit systems. The latest version I could use then would be Xen 4.2 but the even the stable-4.2 branch fails to build for me even after fixing quite a bit of compile failures, I'm just going to give up now. I want to build Xen as I'm relying on some customized emulated delays on the hypervisor in order to test the issue, otherwise I'd have to dedicate a huge system to reproduce this issue.
While it is possible folks on old xen may use newer kernels this issue is likely not a high priority on those 32-bit systems and likely hard to reproduce there. When and if someone is able to test this on a 32-bit kernel / hypervisor then they can test the small patch below.
Can we move forward with the 64-bit part then?
arch/x86/kernel/entry_32.S +2 -0
@@ -982,6 +982,8 @@ ENTRY(xen_hypervisor_callback) ENTRY(xen_do_upcall) 1: mov %esp, %eax call xen_evtchn_do_upcall + movl %esp,%eax + call xen_end_upcall jmp ret_from_intr CFI_ENDPROC ENDPROC(xen_hypervisor_callback)
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
Quoted 4 lines · Luis R. Rodriguez
As I waited I decided to finally install a 32-bit OS but I found now that as of Xen commit
5d1181a5which went upstream as of xen 4.3 Xen no longer supports 32-bit for x86.
You don't need a 32-bit hypervisor to run 32-bit dom0 or domUs.
David
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Luis R. RodriguezQuoted 4 lines · Luis R. Rodriguez
As I waited I decided to finally install a 32-bit OS but I found now that as of Xen commit
5d1181a5which went upstream as of xen 4.3 Xen no longer supports 32-bit for x86.
You don't need a 32-bit hypervisor to run 32-bit dom0 or domUs.
David
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Luis R. RodriguezQuoted 10 lines · Luis R. Rodriguez
Quoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 8 lines
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
Hey folks, what is the status of these patches? Any more feedback?
As I waited I decided to finally install a 32-bit OS but I found now that as of Xen commit 5d1181a5 which went upstream as of xen 4.3 Xen no longer supports 32-bit for x86. So -- Xen no loner even builds on 32-bit systems. The latest version I could use then would be Xen 4.2 but the even the stable-4.2 branch fails to build for me even after fixing quite a bit of compile failures, I'm just going to give up now. I want to build Xen as I'm relying on some customized emulated delays on the hypervisor in order to test the issue, otherwise I'd have to dedicate a huge system to reproduce this issue.
While it is possible folks on old xen may use newer kernels this issue is likely not a high priority on those 32-bit systems and likely hard to reproduce there. When and if someone is able to test this on a 32-bit kernel / hypervisor then they can test the small patch below.
Can we move forward with the 64-bit part then?
arch/x86/kernel/entry_32.S +2 -0
@@ -982,6 +982,8 @@ ENTRY(xen_hypervisor_callback) ENTRY(xen_do_upcall) 1: mov %esp, %eax call xen_evtchn_do_upcall + movl %esp,%eax + call xen_end_upcall jmp ret_from_intr CFI_ENDPROC ENDPROC(xen_hypervisor_callback)
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to David VrabelQuoted 9 lines · David Vrabel
Quoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Quoted 6 lines
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
Hey folks, what is the status of these patches? Any more feedback?
Luis
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Jan BeulichQuoted 7 lines · Jan Beulich
Quoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
I'd rather have something omitted (keeping the current behaviour) than something that has not been tested at all.
Obviously it would be preferable to to fix both 32-bit and 64-bit x86 (and ARM as well) but I'm not going to block an important bug fix for the majority use case (64-bit x86).
David
Re: [PATCH v5 2/2] x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Luis R. RodriguezQuoted 1 line
Quoted 1 line
Quoted 1 line · On 27.01.15 at 02:51, <[email protected]> wrote:
Even if David told you this would be acceptable, I have to question an abstract model of fixing issues on only 64-bit kernels - this may be acceptable for distro purposes, but seems hardly the right approach for upstream. If 32-bit ones are to become deliberately broken, the XEN config option should become dependent on !X86_32.
Jan
PATCH v5 2/2 x86/xen: allow privcmd hypercalls to be preempted on 64-bit
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
Xen has support for splitting heavy work work into a series of hypercalls, called multicalls, and preempting them through what Xen calls continuation [0]. Despite this though without CONFIG_PREEMPT preemption won't happen, without preemption a system can become pretty useless on heavy handed hypercalls. Such is the case for example when creating a > 50 GiB HVM guest, we can get softlockups [1] with:.
kernel: [ 802.084335] BUG: soft lockup - CPU#1 stuck for 22s! [xend:31351]
The softlock up triggers on the TASK_UNINTERRUPTIBLE hanger check (default 120 seconds), on the Xen side in this particular case this happens when the following Xen hypervisor code is used:
xc_domain_set_pod_target() -->
do_memory_op() -->
arch_memory_op() -->
p2m_pod_set_mem_target()
-- long delay (real or emulated) --This happens on arch_memory_op() on the XENMEM_set_pod_target memory op even though arch_memory_op() can handle continuation via hypercall_create_continuation() for example.
Machines over 50 GiB of memory are on high demand and hard to come by so to help replicate this sort of issue long delays on select hypercalls have been emulated in order to be able to test this on smaller machines [2].
On one hand this issue can be considered as expected given that CONFIG_PREEMPT=n is used however we have forced voluntary preemption precedent practices in the kernel even for CONFIG_PREEMPT=n through the usage of cond_resched() sprinkled in many places. To address this issue with Xen hypercalls though we need to find a way to aid to the schedular in the middle of hypercalls. We are motivated to address this issue on CONFIG_PREEMPT=n as otherwise the system becomes rather unresponsive for long periods of time; in the worst case, at least only currently by emulating long delays on select io disk bound hypercalls, this can lead to filesystem corruption if the delay happens for example on SCHEDOP_remote_shutdown (when we call 'xl <domain> shutdown').
We can address this problem by trying to check if we should schedule on the xen timer in the middle of a hypercall on the return from the timer interrupt. We want to be careful to not always force voluntary preemption though so to do this we only selectively enable preemption on very specific xen hypercalls.
This enables hypercall preemption by selectively forcing checks for voluntary preempting only on ioctl initiated private hypercalls where we know some folks have run into reported issues [1].
[0] http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=42217cbc5b3e84b8c145d8cfb62dd5de0134b9e8;hp=3a0b9c57d5c9e82c55dd967c84dd06cb43c49ee9
[1] https://bugzilla.novell.com/show_bug.cgi?id=861093
[2] http://ftp.suse.com/pub/people/mcgrof/xen/emulate-long-xen-hypercalls.patch
Based on original work by: David Vrabel <[email protected]>
--- arch/x86/kernel/entry_64.S | 2 ++ drivers/xen/events/events_base.c | 14 ++++++++++++++ include/xen/events.h | 1 + 3 files changed, 17 insertions(+)
arch/x86/kernel/entry_64.S +2 -0
@@ -1198,6 +1198,8 @@ ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs) popq %rsp CFI_DEF_CFA_REGISTER rsp decl PER_CPU_VAR(irq_count) + movq %rsp, %rdi /* pass pt_regs as first argument */ + call xen_end_upcall jmp error_exit CFI_ENDPROC END(xen_do_hypervisor_callback)
drivers/xen/events/events_base.c +14 -0
@@ -32,6 +32,8 @@#include <linux/slab.h> #include <linux/irqnr.h> #include <linux/pci.h> +#include <linux/sched.h> +#include <linux/kprobes.h> #ifdef CONFIG_X86 #include <asm/desc.h>
@@ -1243,6 +1245,18 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)set_irq_regs (old_regs); } +/* +* Some hypercalls issued by the toolstack can take many 10s of +* seconds. Allow tasks running hypercalls via the privcmd driver to be +* voluntarily preempted even if full kernel preemption is disabled. +*/ +void xen_end_upcall (struct pt_regs *regs) +{ +if (xen_is_preemptible_hypercall (regs)) +_cond_resched (); +} +NOKPROBE_SYMBOL (xen_end_upcall); +void xen_hvm_evtchn_do_upcall (void ) {__xen_evtchn_do_upcall ();
include/xen/events.h +1 -0
@@ -95,6 +95,7 @@ void xen_hvm_callback_vector(void);extern int xen_have_vector_callback;int xen_set_callback_via (uint64_t via);void xen_evtchn_do_upcall (struct pt_regs *regs); +void xen_end_upcall (struct pt_regs *regs);void xen_hvm_evtchn_do_upcall (void );/* Bind a pirq for a physical interrupt to an irq. */
Signature · 1 line
2.1.1
Quoted 7 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This v5 nukes tracing as David said it was useless, it also only adds support for 64-bit as its the only thing I can test, and slightly modifies the documentation in code as to why we want this. The no krobe thing is left in place as I haven't heard confirmation its kosher to remove it.
FYI, I went back an reviewed my last v4 patch and folded the xen_end_upcall() change into it instead.
http://lists.xen.org/archives/html/xen-devel/2015-02/msg00682.html
This will be easier for arm/arm64 to make use of in the future since there are fewer arch-specific changes.
David
Quoted 9 lines · David Vrabel
Quoted 7 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This v5 nukes tracing as David said it was useless, it also only adds support for 64-bit as its the only thing I can test, and slightly modifies the documentation in code as to why we want this. The no krobe thing is left in place as I haven't heard confirmation its kosher to remove it.
Quoted 7 lines
FYI, I went back an reviewed my last v4 patch and folded the
xen_end_upcall()change into it instead.http://lists.xen.org/archives/html/xen-devel/2015-02/msg00682.html
This will be easier for arm/arm64 to make use of in the future since there are fewer arch-specific changes.
I had taken similar approach first and determined using pt_regs was safer after advice from Andy. I'll provide feedback on your new patchset.
Luis
Quoted 9 lines · David Vrabel
Quoted 7 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This v5 nukes tracing as David said it was useless, it also only adds support for 64-bit as its the only thing I can test, and slightly modifies the documentation in code as to why we want this. The no krobe thing is left in place as I haven't heard confirmation its kosher to remove it.
Quoted 7 lines
FYI, I went back an reviewed my last v4 patch and folded the
xen_end_upcall()change into it instead.http://lists.xen.org/archives/html/xen-devel/2015-02/msg00682.html
This will be easier for arm/arm64 to make use of in the future since there are fewer arch-specific changes.
I had taken similar approach first and determined using pt_regs was safer after advice from Andy. I'll provide feedback on your new patchset.
Luis
Quoted 7 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This v5 nukes tracing as David said it was useless, it also only adds support for 64-bit as its the only thing I can test, and slightly modifies the documentation in code as to why we want this. The no krobe thing is left in place as I haven't heard confirmation its kosher to remove it.
FYI, I went back an reviewed my last v4 patch and folded the xen_end_upcall() change into it instead.
http://lists.xen.org/archives/html/xen-devel/2015-02/msg00682.html
This will be easier for arm/arm64 to make use of in the future since there are fewer arch-specific changes.
David