Skip to contentSkip to conversation

patch for review: kvm: implement guest rebooting

PATCH Leonard Norrgard 2 participants 2 messages Jan 22, 2007 1 message not in the archive KVM

lore
Share & tools
lore
#0 lore raw

I've been looking at the guest reboot problem, which currently reboots the host on svm when a Linux guest issues "# shutdown -r now".

(By the way, with a Linux guest, the rebooting can be worked around by adding the kernel parameter reboot=b to the guest, this makes Linux use an alternative reboot method which doesn't cause the host to reboot under kvm-11).

The patch below stops the rebooting of the host and handles the request to reboot the guest in an orderly fashion, so it's a big step forward, but it doesn't yet succeed in rebooting of the guest, instead the VM exits.

The basic approach is to intercept the shutdown. The patch includes some possible alternatives to reboot, while searching for the best approach:

1) Call kvm_qemu_destroy(); and immediately after kvm_qemu_create_context();
2) New ioctl to reset a vcpu
3) I also tried calling init_vmcb from shutdown_intercept, but that had no noticable effect

Method 1 seems promising, but currently results in kvm exiting due to "do_interrupt: unexpect" errors.

Method 2 currently results in vmrun returning KVM_EXIT_TYPE_FAIL_ENTRY.

- Both of methods 1 and 2 need an API version bump
- This code is safe to run without having to fear a host reboot
- SVM only, at the moment
- svm_reset_vcpu() was initially just the one-line call to init_vmcb(), there's some leftover cruft now...
- A printf has been added to cpu_reset(), if this prints, qemu has called all registered reset handlers.

Please comment. The Patch is relative to kvm-11.

kvm_main.c +24 -0
Index: kernel/kvm_main.c
===================================================================
@@ -1764,6 +1764,23 @@
 	return r;
 }
 
+static int kvm_dev_ioctl_reset_vcpu(struct kvm *kvm, struct kvm_reset_vcpu *slot)
+{
+	struct kvm_vcpu *vcpu;
+
+	if (!valid_vcpu(slot->vcpu))
+		return -EINVAL;
+	vcpu = vcpu_load(kvm, slot->vcpu);
+	if (!vcpu)
+		return -ENOENT;
+
+	kvm_arch_ops->reset_vcpu(vcpu);
+
+	vcpu_put(vcpu);
+
+	return 0;
+}
+
 static long kvm_dev_ioctl(struct file *filp,
 			  unsigned int ioctl, unsigned long arg)
 {
@@ -1780,6 +1797,13 @@
 			goto out;
 		break;
 	}
+	case KVM_RESET_VCPU: {
+		struct kvm_reset_vcpu kvm_reset_vcpu;
+		if (copy_from_user(&kvm_reset_vcpu, (void *)arg, sizeof kvm_reset_vcpu))
+			goto out;
+		r = kvm_dev_ioctl_reset_vcpu(kvm, &kvm_reset_vcpu);
+		break;
+	}
 	case KVM_RUN: {
 		struct kvm_run kvm_run;
 
include/linux/kvm.h +9 -1
Index: kernel/include/linux/kvm.h
===================================================================
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 2
+#define KVM_API_VERSION 3
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -46,6 +46,7 @@
 	KVM_EXIT_HLT              = 5,
 	KVM_EXIT_MMIO             = 6,
 	KVM_EXIT_IRQ_WINDOW_OPEN  = 7,
+	KVM_EXIT_SHUTDOWN         = 8,
 };
 
 /* for KVM_RUN */
@@ -218,6 +219,12 @@
 	};
 };
 
+/* for KVM_RESET_VCPU */
+struct kvm_reset_vcpu {
+	/* in */
+	__u32 vcpu;
+};
+
 #define KVMIO 0xAE
 
 #define KVM_GET_API_VERSION       _IO(KVMIO, 1)
@@ -235,5 +242,6 @@
 #define KVM_GET_MSRS              _IOWR(KVMIO, 13, struct kvm_msrs)
 #define KVM_SET_MSRS              _IOWR(KVMIO, 14, struct kvm_msrs)
 #define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 15, struct kvm_msr_list)
+#define KVM_RESET_VCPU            _IO(KVMIO, 16)
 
 #endif
kvm.h +1 -0
Index: kernel/kvm.h
===================================================================
@@ -379,6 +379,7 @@
 	int (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run);
 	int (*vcpu_setup)(struct kvm_vcpu *vcpu);
 	void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
+	void (*reset_vcpu)(struct kvm_vcpu *vcpu);
 };
 
 extern struct kvm_stat kvm_stat;
svm.c +28 -1
Index: kernel/svm.c
===================================================================
@@ -508,7 +508,8 @@
 				(1ULL << INTERCEPT_VMSAVE) |
 				(1ULL << INTERCEPT_STGI) |
 				(1ULL << INTERCEPT_CLGI) |
-				(1ULL << INTERCEPT_SKINIT);
+				(1ULL << INTERCEPT_SKINIT) |
+				(1ULL << INTERCEPT_SHUTDOWN);
 
 	control->iopm_base_pa = iopm_base;
 	control->msrpm_base_pa = msrpm_base;
@@ -582,6 +583,21 @@
 	return r;
 }
 
+static void svm_reset_vcpu(struct kvm_vcpu *vcpu)
+{
+	printk("....svm_reset_vcpu\n");
+	memset(vcpu->svm->vmcb, 0, PAGE_SIZE);
+	//vcpu->svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
+	vcpu->svm->cr0 = 0x00000010;
+	vcpu->svm->asid_generation = 0;
+	memset(vcpu->svm->db_regs, 0, sizeof(vcpu->svm->db_regs));
+	init_vmcb(vcpu->svm->vmcb);
+
+	fx_init(vcpu);
+
+	init_vmcb(vcpu->svm->vmcb);
+}
+
 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
 {
 	if (!vcpu->svm)
@@ -1216,6 +1232,14 @@
 	return 1;
 }
 
+static int shutdown_interception(struct kvm_vcpu *vcpu, struct kvm_run  *kvm_run)
+{
+	/* Note: After an intercepted shutdown, the state saved in the VMCB is undefined.  */
+	printk("....shutdown_interception\n");
+	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
+	return 0;
+}
+
 static int (*svm_exit_handlers[])(struct kvm_vcpu *vcpu,
 				      struct kvm_run *kvm_run) = {
 	[SVM_EXIT_READ_CR0]           		= emulate_on_interception,
@@ -1256,6 +1280,7 @@
 	[SVM_EXIT_STGI]				= invalid_op_interception,
 	[SVM_EXIT_CLGI]				= invalid_op_interception,
 	[SVM_EXIT_SKINIT]			= invalid_op_interception,
+	[SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
 };
 
 
@@ -1690,6 +1715,8 @@
 	.run = svm_vcpu_run,
 	.skip_emulated_instruction = skip_emulated_instruction,
 	.vcpu_setup = svm_vcpu_setup,
+
+	.reset_vcpu = svm_reset_vcpu,
 };
 
 static int __init svm_init(void)
cpu-exec.c +14 -2
Index: qemu/cpu-exec.c
===================================================================
@@ -456,8 +456,20 @@
 
 #ifdef USE_KVM
             if (kvm_allowed) {
-                kvm_cpu_exec(env);
-                longjmp(env->jmp_env, 1);
+	        int ret;
+                ret = kvm_cpu_exec(env);
+		if (ret == 1) {
+		    longjmp(env->jmp_env, 1);
+		} /* else if (ret == 2) {
+		     unused code
+		} */ else {
+		    if (env->interrupt_request != 0) {
+		        /* hardware interrupt will be executed just after */
+		    } else {
+		        /* otherwise, we restart */
+		        longjmp(env->jmp_env, 1);
+		    }
+		}
             }
 #endif
             T0 = 0; /* force lookup of first TB */
target-i386/helper2.c +1 -1
Index: qemu/target-i386/helper2.c
===================================================================
@@ -151,7 +151,7 @@
 void cpu_reset(CPUX86State *env)
 {
     int i;
-
+    printf("....cpu_reset (env)\n");
     memset(env, 0, offsetof(CPUX86State, breakpoints));
 
     tlb_flush(env, 1);
qemu-kvm.c +19 -3
Index: qemu/qemu-kvm.c
===================================================================
@@ -413,9 +413,7 @@
     if (!saved_env[0])
 	saved_env[0] = env;
 
-    kvm_run(kvm_context, 0);
-
-    return 0;
+    return kvm_run(kvm_context, 0);
 }
 
 
@@ -578,6 +576,23 @@
 
     return 1;
 }
+
+static int kvm_guest_reboot(void *opaque, int vcpu)
+{
+	/* Note: After an intercepted shutdown, the state saved in the VMCB is undefined.  */
+	CPUState **envs = opaque, *env;
+	env = envs[0];
+
+	printf ("----kvm_guest_reboot\n");
+
+	qemu_system_reset_request();
+	//env->exception_index = EXCP_INTERRUPT;
+	//cpu_loop_exit ();
+
+	///* Not reached.  */
+
+	return 3;
+}
  
 static struct kvm_callbacks qemu_kvm_ops = {
     .cpuid = kvm_cpuid,
@@ -600,6 +615,7 @@
     .io_window = kvm_io_window,
     .try_push_interrupts = try_push_interrupts,
     .post_kvm_run = post_kvm_run,
+    .shutdown = kvm_guest_reboot,
 };
 
 int kvm_qemu_init()
vl.c +1 -1
Index: qemu/vl.c
===================================================================
@@ -93,7 +93,7 @@
 #include "qemu-kvm.h"
 #endif
 
-#define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
+#define DEFAULT_NETWORK_SCRIPT "/etc/kvm/kvm-ifup"
 
 //#define DEBUG_UNUSED_IOPORT
 //#define DEBUG_IOPORT
kvmctl.c +40 -1

65 lines not on this page · raw message

kvmctl.h +4 -0
Index: user/kvmctl.h
===================================================================
@@ -62,6 +62,8 @@
     int (*io_window)(void *opaque);
     int (*try_push_interrupts)(void *opaque);
     void (*post_kvm_run)(void *opaque, struct kvm_run *kvm_run);
+	/// Guest is being rebooted
+    int (*shutdown)(void *opaque, int vcpu);
 };
 
 /*!
@@ -248,4 +250,6 @@
 			  unsigned long len);
 void kvm_get_dirty_pages(kvm_context_t, int slot, void *buf);
 
+int kvm_reset_vcpu(kvm_context_t kvm, int vcpu);
+
 #endif

-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your opinions on IT & business topics through brief surveys - and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV

_______________________________________________
kvm-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/kvm-devel

A message that is not in the archive (sent off-list or not yet received): 45B456E9.3000709-g2GXA8XeJSExHbG02/[email protected]

#2 lore raw
Quoted 3 lines · Leonard Norrgard

I've been looking at the guest reboot problem, which currently reboots the host on svm when a Linux guest issues "# shutdown -r now".

Obviously a major issue, which we've neglected. Thanks for addressing it.

Quoted 9 lines

The patch below stops the rebooting of the host and handles the request to reboot the guest in an orderly fashion, so it's a big step forward, but it doesn't yet succeed in rebooting of the guest, instead the VM exits.

The basic approach is to intercept the shutdown. The patch includes some possible alternatives to reboot, while searching for the best approach:

1) Call kvm_qemu_destroy(); and immediately after kvm_qemu_create_context();

I don't like this approach. It means memory needs to be deallocated and reallocated. There's some likelyhood of the reallocation failing, so a reboot can turn into a dead VM. Memory contents is not preserved, while a real reset does preserve memory.

More fundamentally, a real cpu can undergo a reset, and so should a virtual cpu.

Quoted 2 lines

2) New ioctl to reset a vcpu

Definitely the way to go. It'll possibly be needed for guest smp too.

Quoted 3 lines

3) I also tried calling init_vmcb from shutdown_intercept, but that had no noticable effect

I think "shutdown" in this context is processor shutdown, not a vcpu reset. IIRC a processor shutdown can be caused by invalid state or hardware errors.

We should intercept it, but it's not that important IMO. (We should also find out what it means exactly)

Quoted 6 lines

Method 1 seems promising, but currently results in kvm exiting due to "do_interrupt: unexpect" errors.

Method 2 currently results in vmrun returning KVM_EXIT_TYPE_FAIL_ENTRY.

Looks like not everything was initialized correctly...

Quoted 43 lines

- Both of methods 1 and 2 need an API version bump
- This code is safe to run without having to fear a host reboot
- SVM only, at the moment
- svm_reset_vcpu() was initially just the one-line call to init_vmcb(), there's some leftover cruft now...
- A printf has been added to cpu_reset(), if this prints, qemu has called all registered reset handlers.

Please comment. The Patch is relative to kvm-11.

------------------------------------------------------------------------

Index: kernel/kvm_main.c
===================================================================

kvm_main.c +17 -0
@@ -1764,6 +1764,23 @@
 	return r;
 }
 
+static int kvm_dev_ioctl_reset_vcpu(struct kvm *kvm, struct kvm_reset_vcpu *slot)
+{
+	struct kvm_vcpu *vcpu;
+
+	if (!valid_vcpu(slot->vcpu))
+		return -EINVAL;
+	vcpu = vcpu_load(kvm, slot->vcpu);
+	if (!vcpu)
+		return -ENOENT;
+
+	kvm_arch_ops->reset_vcpu(vcpu);
+
+	vcpu_put(vcpu);
+
+	return 0;
+}
+
 static long kvm_dev_ioctl(struct file *filp,
 			  unsigned int ioctl, unsigned long arg)
 {

kvm_main.c line 1802

Quoted 6 lines
kvm_main.c +2 -0
@@ -1780,6 +1797,13 @@
 			goto out;
 		break;
 	}
+	case KVM_RESET_VCPU: {
+		struct kvm_reset_vcpu kvm_reset_vcpu;
 

Empty line here.

Quoted 30 lines
kvm_main.c +5 -0
+		if (copy_from_user(&kvm_reset_vcpu, (void *)arg, sizeof kvm_reset_vcpu))
+			goto out;
+		r = kvm_dev_ioctl_reset_vcpu(kvm, &kvm_reset_vcpu);
+		break;
+	}
 	case KVM_RUN: {
 		struct kvm_run kvm_run;

Index: kernel/include/linux/kvm.h
===================================================================

include/linux/kvm.h +2 -1
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 2
+#define KVM_API_VERSION 3
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -46,6 +46,7 @@
 	KVM_EXIT_HLT              = 5,
 	KVM_EXIT_MMIO             = 6,
 	KVM_EXIT_IRQ_WINDOW_OPEN  = 7,
+	KVM_EXIT_SHUTDOWN         = 8,
 };
 
 /* for KVM_RUN */

include/linux/kvm.h line 228

Quoted 10 lines
include/linux/kvm.h +6 -0
@@ -218,6 +219,12 @@
 	};
 };
 
+/* for KVM_RESET_VCPU */
+struct kvm_reset_vcpu {
+	/* in */
+	__u32 vcpu;
+};
+
 

Why define the structure? Since ioctl() passes one argument, it can be the vcpu number.

include/linux/kvm.h line 246

Quoted 9 lines
include/linux/kvm.h +0 -0
 #define KVMIO 0xAE
 

#define KVM_GET_API_VERSION _IO(KVMIO, 1)

include/linux/kvm.h +1 -0
@@ -235,5 +242,6 @@
 #define KVM_GET_MSRS              _IOWR(KVMIO, 13, struct kvm_msrs)
 #define KVM_SET_MSRS              _IOWR(KVMIO, 14, struct kvm_msrs)
 #define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 15, struct kvm_msr_list)
+#define KVM_RESET_VCPU            _IO(KVMIO, 16)
 

If you're passing a structure, this needs to be _IOR(KVMIO, 16, struct kvm_reset_vcpu).

Quoted 9 lines
include/linux/kvm.h +5 -0
@@ -582,6 +583,21 @@
 	return r;
 }
 
+static void svm_reset_vcpu(struct kvm_vcpu *vcpu)
+{
+	printk("....svm_reset_vcpu\n");
+	memset(vcpu->svm->vmcb, 0, PAGE_SIZE);
+	//vcpu->svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;

include/linux/kvm.h line 600

Quoted 10 lines
include/linux/kvm.h +9 -0
+	vcpu->svm->cr0 = 0x00000010;
+	vcpu->svm->asid_generation = 0;
+	memset(vcpu->svm->db_regs, 0, sizeof(vcpu->svm->db_regs));
+	init_vmcb(vcpu->svm->vmcb);
+
+	fx_init(vcpu);
+
+	init_vmcb(vcpu->svm->vmcb);
+}
 

You will need to reset the mmu context as well, as we're transitioning from protected mode to real mode. Look at the set_sregs ioctl for an example. This should be done from common code.

Perhaps init_vmcb relies on the save area being cleared. Try zeroing vmcb->save in init_vmcb().

include/linux/kvm.h line 1721

Quoted 9 lines
include/linux/kvm.h +2 -0
@@ -1690,6 +1715,8 @@
 	.run = svm_vcpu_run,
 	.skip_emulated_instruction = skip_emulated_instruction,
 	.vcpu_setup = svm_vcpu_setup,
+
+	.reset_vcpu = svm_reset_vcpu,
 };
 

It would be nice to call reset_vcpu() as part of the normal vcpu creating sequence, to avoid having a rarely-used code path.

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys - and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV