Drivers: hv: add kexec support
Subsystems (2)
cover Drivers: hv: add kexec support
To make general-purpose kexec (not just kdump) possible for Hyper-V guests we need to perform some additional cleanup before starting new kernel (see [PATCH 2/3] for the detailed description).
Know limitations: kexec with balloned out memory is not possible as the newly loaded kernel doesn't know about this memory and there is no way to ask the host to bring all the memory back on cleanup (or at least I'm not aware of such a way). Kexec with hotplugged memory leads to reboot (not exactly sure why).
This series is supposed to be applied on top of K.Y.'s "[PATCH V2 0/5]
Drivers: hv: vmbus: Cleanup the vmbus unload path"
Vitaly Kuznetsov (3):
Drivers: hv: vmbus: remove hv_synic_free_cpu() call from
hv_synic_cleanup()
Drivers: hv: vmbus: add special kexec handler
Drivers: hv: don't do hypercalls when hypercall_page is NULL
arch/x86/include/asm/mshyperv.h | 2 ++ arch/x86/kernel/cpu/mshyperv.c | 30 ++++++++++++++++++++++++++++++ drivers/hv/hv.c | 15 ++++++++------- drivers/hv/vmbus_drv.c | 15 +++++++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-)
Signature · 1 line
1.9.3
PATCH 1/3 Drivers: hv: vmbus: remove hv_synic_free_cpu() call from hv_synic_cleanup()
We already have hv_synic_free() which frees all per-cpu pages for all CPUs, let's remove the hv_synic_free_cpu() call from hv_synic_cleanup() so it will be possible to do separate cleanup (writing to MSRs) and final freeing. This is going to be used to assist kexec.
--- drivers/hv/hv.c | 2 -- drivers/hv/vmbus_drv.c | 1 + 2 files changed, 1 insertion(+), 2 deletions(-)
drivers/hv/hv.c +0 -2
@@ -530,6 +530,4 @@ void hv_synic_cleanup(void *arg)rdmsrl (HV_X64_MSR_SCONTROL , sctrl.as_uint64); sctrl.enable =0 ;wrmsrl (HV_X64_MSR_SCONTROL , sctrl.as_uint64); - -hv_synic_free_cpu (cpu); }
drivers/hv/vmbus_drv.c +1 -0
@@ -1117,6 +1117,7 @@ static void __exit vmbus_exit(void)hv_cleanup ();for_each_online_cpu (cpu)smp_call_function_single (cpu, hv_synic_cleanup,NULL ,1 ); +hv_synic_free ();acpi_bus_unregister_driver (&vmbus_acpi_driver);hv_cpu_hotplug_quirk (false); }
Signature · 1 line
1.9.3
PATCH 2/3 Drivers: hv: vmbus: add special kexec handler
In reply to Vitaly KuznetsovWhen general-purpose kexec (not kdump) is being performed in Hyper-V guest the newly booted kernel fails with an MCE error coming from the host. It is the same error which was fixed in the "Drivers: hv: vmbus: Implement the protocol for tearing down vmbus state" commit - monitor pages remain special and when they're being written to (as the new kernel doesn't know these pages are special) bad things happen. We need to perform some minimalistic cleanup before booting a new kernel on kexec. To do so we need to register a special machine_ops.shutdown handler to be executed before the native_machine_shutdown(). Registering a shutdown notification handler via the register_reboot_notifier() call is not sufficient as it happens to early for our purposes. machine_ops is not being exported to modules (and I don't think we want to export it) so let's do this in mshyperv.c
The minimalistic cleanup consists of cleaning up clockevents, synic MSRs, guest os id MSR, and hypercall MSR.
Kdump doesn't require all this stuff as it lives in a separate memory space.
--- arch/x86/include/asm/mshyperv.h | 2 ++ arch/x86/kernel/cpu/mshyperv.c | 30 ++++++++++++++++++++++++++++++ drivers/hv/vmbus_drv.c | 14 ++++++++++++++ 3 files changed, 46 insertions(+)
arch/x86/include/asm/mshyperv.h +2 -0
@@ -20,4 +20,6 @@ void hyperv_vector_handler(struct pt_regs *regs);void hv_setup_vmbus_irq (void (*handler)(void ));void hv_remove_vmbus_irq (void ); +void hv_setup_kexec_handler (void (*handler)(void )); +void hv_remove_kexec_handler (void );#endif
arch/x86/kernel/cpu/mshyperv.c +30 -0
@@ -18,6 +18,9 @@#include <linux/efi.h> #include <linux/interrupt.h> #include <linux/irq.h> +#ifdef CONFIG_KEXEC +#include <linux/kexec.h> +#endif #include <asm/processor.h> #include <asm/hypervisor.h> #include <asm/hyperv.h>
@@ -28,12 +31,14 @@#include <asm/i8259.h> #include <asm/apic.h> #include <asm/timer.h> +#include <asm/reboot.h> struct ms_hyperv_info ms_hyperv;EXPORT_SYMBOL_GPL (ms_hyperv);#if IS_ENABLED (CONFIG_HYPERV )static void (*vmbus_handler)(void ); +static void (*hv_kexec_handler)(void );void hyperv_vector_handler (struct pt_regs *regs) {
@@ -69,6 +74,28 @@ void hv_remove_vmbus_irq(void) }EXPORT_SYMBOL_GPL (hv_setup_vmbus_irq);EXPORT_SYMBOL_GPL (hv_remove_vmbus_irq); + +void hv_setup_kexec_handler (void (*handler)(void )) +{ + hv_kexec_handler = handler; +} +EXPORT_SYMBOL_GPL (hv_setup_kexec_handler); + +void hv_remove_kexec_handler (void ) +{ +/* We have no way to deallocate the interrupt gate */ + hv_kexec_handler =NULL ; +} +EXPORT_SYMBOL_GPL (hv_remove_kexec_handler); + +static void hv_machine_shutdown (void ) +{ +#ifdef CONFIG_KEXEC +if (kexec_in_progress && hv_kexec_handler) +hv_kexec_handler (); +#endif +native_machine_shutdown (); +}#endif static uint32_t __init ms_hyperv_platform (void )
@@ -143,6 +170,9 @@ static void __init ms_hyperv_init_platform(void) no_timer_check =1 ;#endif +#if IS_ENABLED (CONFIG_HYPERV ) + machine_ops.shutdown = hv_machine_shutdown; +#endif }const __refconst struct hypervisor_x86 x86_hyper_ms_hyperv = {
drivers/hv/vmbus_drv.c +14 -0
@@ -1060,6 +1060,17 @@ static struct acpi_driver vmbus_acpi_driver = { }, }; +static void hv_kexec_handler (void ) +{ +int cpu; + +hv_synic_clockevents_cleanup (); +vmbus_initiate_unload (); +for_each_online_cpu (cpu) +smp_call_function_single (cpu, hv_synic_cleanup,NULL ,1 ); +hv_cleanup (); +}; +static int __init hv_acpi_init (void ) {int ret, t;
@@ -1092,6 +1103,8 @@ static int __init hv_acpi_init(void)if (ret) goto cleanup; +hv_setup_kexec_handler (hv_kexec_handler); +return 0 ; cleanup:
@@ -1104,6 +1117,7 @@ static void __exit vmbus_exit(void) {int cpu; +hv_remove_kexec_handler (); vmbus_connection.conn_state =DISCONNECTED ;hv_synic_clockevents_cleanup ();vmbus_disconnect ();
Signature · 1 line
1.9.3
PATCH 3/3 Drivers: hv: don't do hypercalls when hypercall_page is NULL
In reply to Vitaly KuznetsovAt the very late stage of kexec a driver (which are not being unloaded) can try to post a message or signal an event. This will crash the kernel as we already did hv_cleanup() and the hypercall page is NULL.
Move all common (between 32 and 64 bit code) declarations to the beginning of the do_hypercall() function. Unfortunately we have to write the !hypercall_page check twice to not mix declarations and code.
--- drivers/hv/hv.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
drivers/hv/hv.c +8 -5
@@ -93,11 +93,14 @@ static int query_hypervisor_info(void)*/ static u64 do_hypercall (u64 control,void *input,void *output) { -#ifdef CONFIG_X86_64 -u64 hv_status =0 ;u64 input_address = (input) ?virt_to_phys (input) :0 ;u64 output_address = (output) ?virt_to_phys (output) :0 ;void *hypercall_page = hv_context.hypercall_page; +#ifdef CONFIG_X86_64 +u64 hv_status =0 ; + +if (!hypercall_page) +return (u64 )ULLONG_MAX ; __asm__ __volatile__("mov %0, %%r8" : :"r" (output_address) :"r8" ); __asm__ __volatile__("call *%3" : "=a" (hv_status) :
@@ -112,13 +115,13 @@ static u64 do_hypercall(u64 control, void *input, void *output)u32 control_lo = control &0xFFFFFFFF ;u32 hv_status_hi =1 ;u32 hv_status_lo =1 ; -u64 input_address = (input) ?virt_to_phys (input) :0 ;u32 input_address_hi = input_address >>32 ;u32 input_address_lo = input_address &0xFFFFFFFF ; -u64 output_address = (output) ?virt_to_phys (output) :0 ;u32 output_address_hi = output_address >>32 ;u32 output_address_lo = output_address &0xFFFFFFFF ; -void *hypercall_page = hv_context.hypercall_page; + +if (!hypercall_page) +return (u64 )ULLONG_MAX ; __asm__ __volatile__ ("call *%8" : "=d"(hv_status_hi), "=a"(hv_status_lo) : "d" (control_hi),
Signature · 1 line
1.9.3
Quoted 16 lines
-----Original Message-----
From: Vitaly Kuznetsov [mailto:[email protected]]
Sent: Wednesday, May 20, 2015 9:46 AM
To: [email protected]
Cc: KY Srinivasan; Haiyang Zhang; [email protected]; Dexuan Cui; Ingo Molnar; H. Peter Anvin; Thomas Gleixner; [email protected]
Subject: [PATCH 0/3] Drivers: hv: add kexec supportTo make general-purpose kexec (not just kdump) possible for Hyper-V guests
we need to perform some additional cleanup before starting new kernel (see
[PATCH 2/3] for the detailed description).
Quoted 8 lines
Know limitations: kexec with balloned out memory is not possible as the newly loaded kernel doesn't know about this memory and there is no way to ask the host to bring all the memory back on cleanup (or at least I'm not aware of such a way). Kexec with hotplugged memory leads to reboot (not exactly sure why).
This series is supposed to be applied on top of K.Y.'s "[PATCH V2 0/5]
Drivers: hv: vmbus: Cleanup the vmbus unload path"
I was currently working on this exact issue. After I checked in the clockevents device based on the Hyper-V timers, kexec has been broken since the hypervisor state with regards to the synthetic interrupt controller is not correctly cleaned up. Thank you! Greg, will you be able to take these patches through your tree as well.
Regards,
K. Y
Quoted 16 lines
Vitaly Kuznetsov (3):
Drivers: hv: vmbus: removehv_synic_free_cpu()call from
hv_synic_cleanup()
Drivers: hv: vmbus: add special kexec handler
Drivers: hv: don't do hypercalls whenhypercall_pageis NULLarch/x86/include/asm/mshyperv.h | 2 ++ arch/x86/kernel/cpu/mshyperv.c | 30 ++++++++++++++++++++++++++++++ drivers/hv/hv.c | 15 ++++++++------- drivers/hv/vmbus_drv.c | 15 +++++++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-)--
1.9.3