Skip to contentSkip to conversation

xen/pvhvm: add soft reset on kexec/kdump support

PATCH In review Vitaly Kuznetsov 4 participants 23 messages Sep 25, 2015 – Sep 29, 2015

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xen/pvhvm: add soft reset on kexec/kdump support

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

---
 arch/x86/xen/enlighten.c      | 23 +++++++++++++++++++++++
 include/xen/interface/sched.h |  8 ++++++++
 2 files changed, 31 insertions(+)
arch/x86/xen/enlighten.c +23 -0
index 30d12af..e4be074 100644
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1807,6 +1811,21 @@ static struct notifier_block xen_hvm_cpu_notifier = {
 	.notifier_call	= xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC_CORE
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_pvhvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
 	if (xen_pv_domain())
@@ -1826,6 +1845,10 @@ static void __init xen_hvm_guest_init(void)
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
 	xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+	machine_ops.shutdown = xen_pvhvm_kexec_shutdown;
+	machine_ops.crash_shutdown = xen_pvhvm_crash_shutdown;
+#endif
 }
 #endif
 
include/xen/interface/sched.h +8 -0
index 9ce0839..f184909 100644
@@ -107,5 +107,13 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time expired.     */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5
 
 #endif /* __XEN_PUBLIC_SCHED_H__ */
Signature · 1 line
2.4.3
Reviewed-by on v1 1/1
#1 lore raw
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 74 lines
---
 arch/x86/xen/enlighten.c      | 23 +++++++++++++++++++++++
 include/xen/interface/sched.h |  8 ++++++++
 2 files changed, 31 insertions(+)
arch/x86/xen/enlighten.c +23 -0
index 30d12af..e4be074 100644
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1807,6 +1811,21 @@ static struct notifier_block xen_hvm_cpu_notifier = {
 	.notifier_call	= xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC_CORE
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_pvhvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
 	if (xen_pv_domain())
@@ -1826,6 +1845,10 @@ static void __init xen_hvm_guest_init(void)
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
 	xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+	machine_ops.shutdown = xen_pvhvm_kexec_shutdown;
+	machine_ops.crash_shutdown = xen_pvhvm_crash_shutdown;
+#endif
 }
 #endif
 
include/xen/interface/sched.h +8 -0
index 9ce0839..f184909 100644
@@ -107,5 +107,13 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time expired.     */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5
 
 #endif /* __XEN_PUBLIC_SCHED_H__ */
Signature · 1 line
2.4.3
Reviewed-by on v1 1/1
#2 lore raw
In reply to Vitaly Kuznetsov
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 74 lines
---
 arch/x86/xen/enlighten.c      | 23 +++++++++++++++++++++++
 include/xen/interface/sched.h |  8 ++++++++
 2 files changed, 31 insertions(+)
arch/x86/xen/enlighten.c +23 -0
index 30d12af..e4be074 100644
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1807,6 +1811,21 @@ static struct notifier_block xen_hvm_cpu_notifier = {
 	.notifier_call	= xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC_CORE
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_pvhvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
 	if (xen_pv_domain())
@@ -1826,6 +1845,10 @@ static void __init xen_hvm_guest_init(void)
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
 	xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+	machine_ops.shutdown = xen_pvhvm_kexec_shutdown;
+	machine_ops.crash_shutdown = xen_pvhvm_crash_shutdown;
+#endif
 }
 #endif
 
include/xen/interface/sched.h +8 -0
index 9ce0839..f184909 100644
@@ -107,5 +107,13 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time expired.     */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5
 
 #endif /* __XEN_PUBLIC_SCHED_H__ */
Signature · 1 line
2.4.3
#3 lore raw
Quoted 33 lines · Konrad Rzeszutek Wilk
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 1 line
Quoted 7 lines

arch/x86/xen/enlighten.c static struct notifier_block xen_hvm_cpu_notifier = { · line 1820

Quoted 7 lines
arch/x86/xen/enlighten.c +7 -0
+#ifdef CONFIG_KEXEC_CORE
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}

This routine is not kexec-specific so "kexec_" should be dropped from the name.

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

-boris

#4 lore raw
In reply to Konrad Rzeszutek Wilk
Quoted 33 lines · Konrad Rzeszutek Wilk
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 1 line
Quoted 7 lines

arch/x86/xen/enlighten.c static struct notifier_block xen_hvm_cpu_notifier = { · line 1820

Quoted 7 lines
arch/x86/xen/enlighten.c +7 -0
+#ifdef CONFIG_KEXEC_CORE
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}

This routine is not kexec-specific so "kexec_" should be dropped from the name.

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

-boris

#5 lore raw
Quoted 33 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 10 lines
Quoted 7 lines

arch/x86/xen/enlighten.c static struct notifier_block xen_hvm_cpu_notifier = { · line 1820

Quoted 7 lines

+#ifdef CONFIG_KEXEC_CORE

arch/x86/xen/enlighten.c +6 -0
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}

This routine is not kexec-specific so "kexec_" should be dropped from the name.

Ok,

Quoted 5 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

Signature · 1 line
  Vitaly
#6 lore raw
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }

Is there another path to handle this op?

Thanks.
-boris

#7 lore raw
In reply to Vitaly Kuznetsov
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }

Is there another path to handle this op?

Thanks.
-boris

#8 lore raw
Quoted 27 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

#9 lore raw
In reply to Boris Ostrovsky
Quoted 27 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

#10 lore raw
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 6 lines
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

-boris

#11 lore raw
In reply to Konrad Rzeszutek Wilk
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 6 lines
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

-boris

#12 lore raw
Quoted 36 lines · Boris Ostrovsky
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }
Quoted 6 lines
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 2 lines

-boris

#13 lore raw
In reply to Boris Ostrovsky
Quoted 36 lines · Boris Ostrovsky
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

    domain_shutdown()
    {
        ...
        if ( is_hardware_domain(d) )
            hwdom_shutdown(reason);
        ...
    }
Quoted 6 lines
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 2 lines

-boris

#14 lore raw
Quoted 35 lines · Konrad Rzeszutek Wilk
Quoted 34 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 25 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 6 lines
Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

-boris

#15 lore raw
In reply to Konrad Rzeszutek Wilk
Quoted 35 lines · Konrad Rzeszutek Wilk
Quoted 34 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 25 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 6 lines
Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

-boris

#16 lore raw
Quoted 43 lines · Boris Ostrovsky
Quoted 35 lines · Konrad Rzeszutek Wilk
Quoted 34 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 25 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 6 lines
Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 5 lines

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

Xen hypervisor doesn't analyzie the reason, it is being analyzed by the toolstack (XEN_DOMCTL_getdomaininfo returns this info encoded in flags with XEN_DOMINF_shutdownshift shift).

Signature · 1 line
  Vitaly
#17 lore raw
In reply to Boris Ostrovsky
Quoted 43 lines · Boris Ostrovsky
Quoted 35 lines · Konrad Rzeszutek Wilk
Quoted 34 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 25 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

     domain_shutdown()
     {
         ...
         if ( is_hardware_domain(d) )
             hwdom_shutdown(reason);
         ...
     }
Quoted 1 line

Is there another path to handle this op?

Yes:
e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation
and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

Quoted 6 lines
Quoted 3 lines

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 5 lines

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

Xen hypervisor doesn't analyzie the reason, it is being analyzed by the toolstack (XEN_DOMCTL_getdomaininfo returns this info encoded in flags with XEN_DOMINF_shutdownshift shift).

Signature · 1 line
  Vitaly
#18 lore raw
Quoted 43 lines · Vitaly Kuznetsov
Quoted 41 lines · Boris Ostrovsky
Quoted 40 lines · Konrad Rzeszutek Wilk
Quoted 28 lines · Boris Ostrovsky
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

      domain_shutdown()
      {
          ...
          if ( is_hardware_domain(d) )
              hwdom_shutdown(reason);
          ...
      }

Is there another path to handle this op?

Quoted 8 lines
Quoted 5 lines

Yes:

   e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation

and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 8 lines
Quoted 5 lines

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

Xen hypervisor doesn't analyzie the reason, it is being analyzed by the toolstack (XEN_DOMCTL_getdomaininfo returns this info encoded in flags with XEN_DOMINF_shutdownshift shift).

Thanks.

My problem was that I didn't realize that toolstack generates macros for shutdown reasons from libxl_types.idl and I couldn't see who looks for SHUTDOWN_soft_reset.

-boris

#19 lore raw
In reply to Vitaly Kuznetsov
Quoted 43 lines · Vitaly Kuznetsov
Quoted 41 lines · Boris Ostrovsky
Quoted 40 lines · Konrad Rzeszutek Wilk
Quoted 28 lines · Boris Ostrovsky
Quoted 27 lines · Konrad Rzeszutek Wilk
Quoted 26 lines · Boris Ostrovsky
Quoted 4 lines · Vitaly Kuznetsov
Quoted 4 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Quoted 8 lines

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

OK, so I was looking at the right tree. Then I don't understand how SHUTDOWN_soft_reset would be reached for a non-privileged domain. The only path that I see is

      domain_shutdown()
      {
          ...
          if ( is_hardware_domain(d) )
              hwdom_shutdown(reason);
          ...
      }

Is there another path to handle this op?

Quoted 8 lines
Quoted 5 lines

Yes:

   e1bd9812966de9a16f30a58e7162b80bd6af361b libxc: support XEN_DOMCTL_soft_reset operation

and
c57e6ebd8c3e490e353e68d96abec1dad01e72f5 (lib)xl: soft reset support

That's toolstack issuing hypercalls from dom0.

I am asking about (non-privileged) guest itself calling SCHEDOP_shutdown.

The hypervisor ends up calling:
__domain_finalise_shutdown which sends an VIRQ_DOM_EXC to dom0 which makes the toolstack do all of those operations.

Quoted 8 lines
Quoted 5 lines

OK. But the I don't see anyone checking that 'reason' (or 'shutdown_code') is SHUTDOWN_soft_reset. In other words, the guest can do 'xen_reboot(23)' or 'xen_reboot(154)'. Or, it seems, even 'xen_reboot(SHUTDOWN_reboot)'? The only value it shouldn't be is SHUTDOWN_suspend.

Xen hypervisor doesn't analyzie the reason, it is being analyzed by the toolstack (XEN_DOMCTL_getdomaininfo returns this info encoded in flags with XEN_DOMINF_shutdownshift shift).

Thanks.

My problem was that I didn't realize that toolstack generates macros for shutdown reasons from libxl_types.idl and I couldn't see who looks for SHUTDOWN_soft_reset.

-boris

#20 lore raw
In reply to Boris Ostrovsky
Quoted 33 lines · Boris Ostrovsky
Quoted 32 lines · Konrad Rzeszutek Wilk
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 8 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

with s/pvhvm/hvm/

to match with the rest of the file.

Thanks.

Quoted 10 lines
Quoted 7 lines

arch/x86/xen/enlighten.c static struct notifier_block xen_hvm_cpu_notifier = { · line 1820

Quoted 7 lines

+#ifdef CONFIG_KEXEC_CORE

arch/x86/xen/enlighten.c +6 -0
+static void xen_pvhvm_kexec_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}

This routine is not kexec-specific so "kexec_" should be dropped from the name.

Ok,

Quoted 5 lines

Also, I am not sure I see how this new op will be used in the hypervisor --- currently AFAICS it is only processed under is_hardware_domain(). Are there other patches that will support HVM guests?

Please see my Xen series:
http://lists.xenproject.org/archives/html/xen-devel/2015-09/msg00547.html
(last 'full' submission).

All patches from my 'toolstack-assisted approach to PVHVM guest kexec' are already merged to xen.git (first 10 are already in 'master' and the last one is in 'staging').

Signature · 1 line
  Vitaly
#21 lore raw
In reply to Vitaly Kuznetsov
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 6 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

I've fixed up the naming (see below) and applied to for-linus-4.3b, thanks.

I've also tagged it for stable since this looks like something that distros would want and I think this is useful to have in place when soft reset support in Xen becomes available.

I can drop the stable tag if others disagree.

David

8<--------------------------------
x86/xen: Support kexec/kdump in HVM guests by doing a soft
reset

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:

- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.

Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log: 'Unknown shutdown reason code
5. Destroying domain.' which gives a clue to what the problem is and eliminates false expectations.

---
 arch/x86/xen/enlighten.c      | 23 +++++++++++++++++++++++
 include/xen/interface/sched.h |  8 ++++++++
 2 files changed, 31 insertions(+)
arch/x86/xen/enlighten.c +23 -0
index 1f1dbd2..993b7a7 100644
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1808,6 +1812,21 @@ static struct notifier_block xen_hvm_cpu_notifier = {
 	.notifier_call	= xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC_CORE
+static void xen_hvm_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_hvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
 	if (xen_pv_domain())
@@ -1827,6 +1846,10 @@ static void __init xen_hvm_guest_init(void)
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
 	xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+	machine_ops.shutdown = xen_hvm_shutdown;
+	machine_ops.crash_shutdown = xen_hvm_crash_shutdown;
+#endif
 }
 #endif
 
include/xen/interface/sched.h +0 -0
index 9ce0839..f184909 100644
@@ -107,5 +107,13 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.
      */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.
      */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time

expired. */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5

#endif /* __XEN_PUBLIC_SCHED_H__ */

Signature · 1 line
2.1.4
#22 lore raw
In reply to Vitaly Kuznetsov
Quoted 16 lines · Vitaly Kuznetsov

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:
- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.
Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Quoted 6 lines

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log:
'Unknown shutdown reason code 5. Destroying domain.'
which gives a clue to what the problem is and eliminates false expectations.

I've fixed up the naming (see below) and applied to for-linus-4.3b, thanks.

I've also tagged it for stable since this looks like something that distros would want and I think this is useful to have in place when soft reset support in Xen becomes available.

I can drop the stable tag if others disagree.

David

8<--------------------------------
x86/xen: Support kexec/kdump in HVM guests by doing a soft
reset

Currently there is a number of issues preventing PVHVM Xen guests from doing successful kexec/kdump:

- Bound event channels.
- Registered vcpu_info.
- PIRQ/emuirq mappings.
- shared_info frame after XENMAPSPACE_shared_info operation.
- Active grant mappings.

Basically, newly booted kernel stumbles upon already set up Xen interfaces and there is no way to reestablish them. In Xen-4.7 a new feature called 'soft reset' is coming. A guest performing kexec/kdump operation is supposed to call SCHEDOP_shutdown hypercall with SHUTDOWN_soft_reset reason before jumping to new kernel. Hypervisor (with some help from toolstack) will do full domain cleanup (but keeping its memory and vCPU contexts intact) returning the guest to the state it had when it was first booted and thus allowing it to start over.

Doing SHUTDOWN_soft_reset on Xen hypervisors which don't support it is probably OK as by default all unknown shutdown reasons cause domain destroy with a message in toolstack log: 'Unknown shutdown reason code
5. Destroying domain.' which gives a clue to what the problem is and eliminates false expectations.

---
 arch/x86/xen/enlighten.c      | 23 +++++++++++++++++++++++
 include/xen/interface/sched.h |  8 ++++++++
 2 files changed, 31 insertions(+)
arch/x86/xen/enlighten.c +23 -0
index 1f1dbd2..993b7a7 100644
@@ -33,6 +33,10 @@
 #include <linux/memblock.h>
 #include <linux/edd.h>
 
+#ifdef CONFIG_KEXEC_CORE
+#include <linux/kexec.h>
+#endif
+
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/xen.h>
@@ -1808,6 +1812,21 @@ static struct notifier_block xen_hvm_cpu_notifier = {
 	.notifier_call	= xen_hvm_cpu_notify,
 };
 
+#ifdef CONFIG_KEXEC_CORE
+static void xen_hvm_shutdown(void)
+{
+	native_machine_shutdown();
+	if (kexec_in_progress)
+		xen_reboot(SHUTDOWN_soft_reset);
+}
+
+static void xen_hvm_crash_shutdown(struct pt_regs *regs)
+{
+	native_machine_crash_shutdown(regs);
+	xen_reboot(SHUTDOWN_soft_reset);
+}
+#endif
+
 static void __init xen_hvm_guest_init(void)
 {
 	if (xen_pv_domain())
@@ -1827,6 +1846,10 @@ static void __init xen_hvm_guest_init(void)
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
 	xen_hvm_init_mmu_ops();
+#ifdef CONFIG_KEXEC_CORE
+	machine_ops.shutdown = xen_hvm_shutdown;
+	machine_ops.crash_shutdown = xen_hvm_crash_shutdown;
+#endif
 }
 #endif
 
include/xen/interface/sched.h +0 -0
index 9ce0839..f184909 100644
@@ -107,5 +107,13 @@ struct sched_watchdog {
 #define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.
      */
 #define SHUTDOWN_crash      3  /* Tell controller we've crashed.
      */
 #define SHUTDOWN_watchdog   4  /* Restart because watchdog time

expired. */
+/*
+ * Domain asked to perform 'soft reset' for it. The expected behavior is to
+ * reset internal Xen state for the domain returning it to the point where it
+ * was created but leaving the domain's memory contents and vCPU contexts
+ * intact. This will allow the domain to start over and set up all Xen specific
+ * interfaces again.
+ */
+#define SHUTDOWN_soft_reset 5

#endif /* __XEN_PUBLIC_SCHED_H__ */

Signature · 1 line
2.1.4