Skip to contentSkip to conversation

Activate Virtualization On Demand v2

PATCH In review Alexander Graf 3 participants 10 messages Jun 15, 2009 – Jun 18, 2009

Subsystems (7)
x86 KVM PowerPC s390 +3 more subsystems
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Activate Virtualization On Demand v2

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.

So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.

---

v2 uses kvm_lock and traces failures atomically

---
 arch/ia64/kvm/kvm-ia64.c        |    8 ++-
 arch/powerpc/kvm/powerpc.c      |    2 +-
 arch/s390/kvm/kvm-s390.c        |    2 +-
 arch/x86/include/asm/kvm_host.h |    2 +-
 arch/x86/kvm/svm.c              |   13 ++++--
 arch/x86/kvm/vmx.c              |    7 +++-
 arch/x86/kvm/x86.c              |    4 +-
 include/linux/kvm_host.h        |    2 +-
 virt/kvm/kvm_main.c             |   82 +++++++++++++++++++++++++++++++++------
 9 files changed, 96 insertions(+), 26 deletions(-)
arch/ia64/kvm/kvm-ia64.c +5 -3
index 906d597..3141a92 100644
@@ -124,7 +124,7 @@ long ia64_pal_vp_create(u64 *vpd, u64 *host_iva, u64 *opt_handler)
 
 static  DEFINE_SPINLOCK(vp_lock);
 
-void kvm_arch_hardware_enable(void *garbage)
+int kvm_arch_hardware_enable(void *garbage)
 {
 	long  status;
 	long  tmp_base;
@@ -137,7 +137,7 @@ void kvm_arch_hardware_enable(void *garbage)
 	slot = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	local_irq_restore(saved_psr);
 	if (slot < 0)
-		return;
+		return -EINVAL;
 
 	spin_lock(&vp_lock);
 	status = ia64_pal_vp_init_env(kvm_vsa_base ?
@@ -145,7 +145,7 @@ void kvm_arch_hardware_enable(void *garbage)
 			__pa(kvm_vm_buffer), KVM_VM_BUFFER_BASE, &tmp_base);
 	if (status != 0) {
 		printk(KERN_WARNING"kvm: Failed to Enable VT Support!!!!\n");
-		return ;
+		return -EINVAL;
 	}
 
 	if (!kvm_vsa_base) {
@@ -154,6 +154,8 @@ void kvm_arch_hardware_enable(void *garbage)
 	}
 	spin_unlock(&vp_lock);
 	ia64_ptr_entry(0x3, slot);
+
+	return 0;
 }
 
 void kvm_arch_hardware_disable(void *garbage)
arch/powerpc/kvm/powerpc.c +2 -1
index 9057335..6558ab7 100644
@@ -80,7 +80,8 @@ int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu)
 	return r;
 }
 
-void kvm_arch_hardware_enable(void *garbage)
+int kvm_arch_hardware_enable(void *garbage)
 {
+	return 0;
 }
 
arch/s390/kvm/kvm-s390.c +2 -1
index cbfe91e..a14e676 100644
@@ -70,7 +70,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 
 
 /* Section: not file related */
-void kvm_arch_hardware_enable(void *garbage)
+int kvm_arch_hardware_enable(void *garbage)
 {
 	/* every s390 is virtualization enabled ;-) */
+	return 0;
 }
arch/x86/include/asm/kvm_host.h +1 -1
index 4627627..72d5075 100644
@@ -463,7 +463,7 @@ struct descriptor_table {
 struct kvm_x86_ops {
 	int (*cpu_has_kvm_support)(void);          /* __init */
 	int (*disabled_by_bios)(void);             /* __init */
-	void (*hardware_enable)(void *dummy);      /* __init */
+	int (*hardware_enable)(void *dummy);
 	void (*hardware_disable)(void *dummy);
 	void (*check_processor_compatibility)(void *rtn);
 	int (*hardware_setup)(void);               /* __init */
arch/x86/kvm/svm.c +9 -4
index 04ee964..47a8b94 100644
@@ -245,7 +245,7 @@ static void svm_hardware_disable(void *garbage)
 	cpu_svm_disable();
 }
 
-static void svm_hardware_enable(void *garbage)
+static int svm_hardware_enable(void *garbage)
 {
 
 	struct svm_cpu_data *svm_data;
@@ -254,16 +254,20 @@ static void svm_hardware_enable(void *garbage)
 	struct desc_struct *gdt;
 	int me = raw_smp_processor_id();
 
+	rdmsrl(MSR_EFER, efer);
+	if (efer & EFER_SVME)
+		return -EBUSY;
+
 	if (!has_svm()) {
 		printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me);
-		return;
+		return -EINVAL;
 	}
 	svm_data = per_cpu(svm_data, me);
 
 	if (!svm_data) {
 		printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n",
 		       me);
-		return;
+		return -EINVAL;
 	}
 
 	svm_data->asid_generation = 1;
@@ -274,11 +278,12 @@ static void svm_hardware_enable(void *garbage)
 	gdt = (struct desc_struct *)gdt_descr.address;
 	svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 
-	rdmsrl(MSR_EFER, efer);
 	wrmsrl(MSR_EFER, efer | EFER_SVME);
 
 	wrmsrl(MSR_VM_HSAVE_PA,
 	       page_to_pfn(svm_data->save_area) << PAGE_SHIFT);
+
+	return 0;
 }
 
 static void svm_cpu_uninit(int cpu)
arch/x86/kvm/vmx.c +6 -1
index d3919ac..3df3b0a 100644
@@ -1068,12 +1068,15 @@ static __init int vmx_disabled_by_bios(void)
 	/* locked but not enabled */
 }
 
-static void hardware_enable(void *garbage)
+static int hardware_enable(void *garbage)
 {
 	int cpu = raw_smp_processor_id();
 	u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
 	u64 old;
 
+	if (read_cr4() & X86_CR4_VMXE)
+		return -EBUSY;
+
 	INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
 	rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
 	if ((old & (FEATURE_CONTROL_LOCKED |
@@ -1088,6 +1091,8 @@ static void hardware_enable(void *garbage)
 	asm volatile (ASM_VMX_VMXON_RAX
 		      : : "a"(&phys_addr), "m"(phys_addr)
 		      : "memory", "cc");
+
+	return 0;
 }
 
 static void vmclear_local_vcpus(void)
arch/x86/kvm/x86.c +2 -2
index 03431b2..bfef950 100644
@@ -4222,9 +4222,9 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
 	return kvm_x86_ops->vcpu_reset(vcpu);
 }
 
-void kvm_arch_hardware_enable(void *garbage)
+int kvm_arch_hardware_enable(void *garbage)
 {
-	kvm_x86_ops->hardware_enable(garbage);
+	return kvm_x86_ops->hardware_enable(garbage);
 }
 
 void kvm_arch_hardware_disable(void *garbage)
include/linux/kvm_host.h +1 -1
index 11eb702..7678995 100644
@@ -292,7 +292,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
 
 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu);
-void kvm_arch_hardware_enable(void *garbage);
+int kvm_arch_hardware_enable(void *garbage);
 void kvm_arch_hardware_disable(void *garbage);
 int kvm_arch_hardware_setup(void);
 void kvm_arch_hardware_unsetup(void);
virt/kvm/kvm_main.c +70 -12

178 lines not on this page · raw message

Signature · 1 line
1.6.0.2
#1 lore raw
Quoted 8 lines · Alexander Graf

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

That will only become interesting if we every have such a thing in mainline. So NACK, lots of complication for no good reason.

#2 lore raw
Quoted 6 lines · Christoph Hellwig
Quoted 5 lines · Alexander Graf

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Quoted 7 lines
Quoted 3 lines

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

That will only become interesting if we every have such a thing in mainline. So NACK, lots of complication for no good reason.

I don't want to fight political battles here. Seriously - we're out of kindergarden.
There are users out there who want to have VBox/VMware and kvm installed in parallel and can't have both kernel modules loaded at the same time. We're only hurting _our_ users, not the others if we keep people from having kvm*.ko loaded.

Sigh.

Alex

#3 lore raw
Quoted 1 line · Alexander Graf

I don't want to fight political battles here.

So stop that crap.

#4 lore raw
In reply to Christoph Hellwig
Quoted 7 lines · Christoph Hellwig
Quoted 5 lines · Alexander Graf

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Quoted 8 lines
Quoted 4 lines

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

That will only become interesting if we every have such a thing in mainline. So NACK, lots of complication for no good reason.

If it were truly lots of complication, I might agree. But it isn't, and we keep getting reports from users about it.

Signature · 1 line
error compiling committee.c: too many arguments to function
#5 lore raw
In reply to Alexander Graf
Quoted 22 lines · Alexander Graf

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.

So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.

virt/kvm/kvm_main.c +4 -0
+static int hardware_enable_all(void)
+{
+	int r = 0;
+

virt/kvm/kvm_main.c static struct miscdevice kvm_dev = { · line 2379

Quoted 10 lines
virt/kvm/kvm_main.c +10 -0
+	spin_lock(&kvm_lock);
+
+	kvm_usage_count++;
+	if (kvm_usage_count == 1) {
+		atomic_set(&hardware_enable_failed, 1);
+		on_each_cpu(hardware_enable, NULL, 1);
+
+		if (!atomic_dec_and_test(&hardware_enable_failed))
+			r = -EBUSY;
+	}

That's a little obfuscated. I suggest atomic_set(..., p) and atomic_read(...).

Quoted 21 lines
virt/kvm/kvm_main.c +6 -1
+
  static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
  			   void *v)
  {
  	int cpu = (long)v;
 
+	if (!kvm_usage_count)
+		return NOTIFY_OK;
+
  	val&= ~CPU_TASKS_FROZEN;
  	switch (val) {
  	case CPU_DYING:
@@ -2513,13 +2571,15 @@ static void kvm_exit_debug(void)
 
  static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  {
-	hardware_disable(NULL);
+	if (kvm_usage_count)
+		hardware_disable(NULL);
  	return 0;

virt/kvm/kvm_main.c static void kvm_exit_debug(void) · line 2585

Quoted 10 lines
virt/kvm/kvm_main.c +2 -1
  }
 
  static int kvm_resume(struct sys_device *dev)
  {
-	hardware_enable(NULL);
+	if (kvm_usage_count)
+		hardware_enable(NULL);
  	return 0;
  }
 

Please tell me you tested suspend/resume with/without VMs and cpu hotunplug/hotplug.

Signature · 1 line
error compiling committee.c: too many arguments to function
#6 lore raw
Quoted 28 lines · Avi Kivity
Quoted 27 lines · Alexander Graf

X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).

Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.

To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.

So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.

virt/kvm/kvm_main.c +10 -0
+static int hardware_enable_all(void)
+{
+    int r = 0;
+
+    spin_lock(&kvm_lock);
+
+    kvm_usage_count++;
+    if (kvm_usage_count == 1) {
+        atomic_set(&hardware_enable_failed, 1);
+        on_each_cpu(hardware_enable, NULL, 1);
Quoted 8 lines

virt/kvm/kvm_main.c static struct miscdevice kvm_dev = { · line 2379

Quoted 5 lines
virt/kvm/kvm_main.c +4 -0
+
+        if (!atomic_dec_and_test(&hardware_enable_failed))
+            r = -EBUSY;
+    }

That's a little obfuscated. I suggest atomic_set(..., p) and atomic_read(...).

Ah, I was more searching for an atomic_test :-).

Quoted 33 lines
Quoted 21 lines

static int kvm_cpu_hotplug(struct notifier_block *notifier,
unsigned long val,
void *v)

  {
      int cpu = (long)v;
virt/kvm/kvm_main.c +5 -1
+    if (!kvm_usage_count)
+        return NOTIFY_OK;
+
      val&= ~CPU_TASKS_FROZEN;
      switch (val) {
      case CPU_DYING:
@@ -2513,13 +2571,15 @@ static void kvm_exit_debug(void)
 
  static int kvm_suspend(struct sys_device *dev, pm_message_t state)
  {
-    hardware_disable(NULL);
+    if (kvm_usage_count)
+        hardware_disable(NULL);
      return 0;

virt/kvm/kvm_main.c static void kvm_exit_debug(void) · line 2585

Quoted 10 lines
virt/kvm/kvm_main.c +2 -1
  }
 
  static int kvm_resume(struct sys_device *dev)
  {
-    hardware_enable(NULL);
+    if (kvm_usage_count)
+        hardware_enable(NULL);
      return 0;
  }
 

+

Quoted 2 lines

Please tell me you tested suspend/resume with/without VMs and cpu hotunplug/hotplug.

I tested cpu hotplugging. On the last round I tested suspend/resume, but this time I couldn't because my machine can't do suspend :-(. So I'll try hard and find a machine I can test it on for the next round.

Alex

#7 lore raw
Quoted 8 lines · Alexander Graf
Quoted 3 lines

Please tell me you tested suspend/resume with/without VMs and cpu hotunplug/hotplug.

I tested cpu hotplugging. On the last round I tested suspend/resume, but this time I couldn't because my machine can't do suspend :-(. So I'll try hard and find a machine I can test it on for the next round.

I can test suspend/resume for you if you don't have a friendly machine. I have a personal interest in keeping it working :)

Signature · 1 line
error compiling committee.c: too many arguments to function
#8 lore raw
Quoted 11 lines · Avi Kivity
Quoted 8 lines · Alexander Graf
Quoted 3 lines

Please tell me you tested suspend/resume with/without VMs and cpu hotunplug/hotplug.

I tested cpu hotplugging. On the last round I tested suspend/resume, but this time I couldn't because my machine can't do suspend :-(. So I'll try hard and find a machine I can test it on for the next round.

I can test suspend/resume for you if you don't have a friendly machine. I have a personal interest in keeping it working :)

Thinking about it again - there's only the atomic dec_and_test vs. read thing and the suspend test missing.

Is the atomic operation as is really that confusing? If not, we can keep the patch as is and you simply try s2ram on your notebook :-). I'm pretty sure it works - it used to.

Alex

#9 lore raw
Quoted 6 lines · Alexander Graf
Quoted 1 line

I can test suspend/resume for you if you don't have a friendly machine. I have a personal interest in keeping it working :)

Thinking about it again - there's only the atomic dec_and_test vs. read thing and the suspend test missing.

Is the atomic operation as is really that confusing?

Yes. It says, "something tricky is going on, see if you can find it".

Quoted 1 line

If not, we can keep the patch as is and you simply try s2ram on your notebook :-). I'm pretty sure it works - it used to.

It looks like it will work, but these things are tricky. I'll test an updated patch. Please also test reboot on Intel with a VM spinning and with no VMs loaded - Intel reboots are tricky too.

Signature · 1 line
error compiling committee.c: too many arguments to function