Activate Virtualization On Demand
PATCH 1/1 Activate Virtualization On Demand
X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).
Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.
To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.
So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.
--- arch/ia64/kvm/kvm-ia64.c | 8 +++-- arch/powerpc/kvm/powerpc.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm.c | 13 +++++-- arch/x86/kvm/vmx.c | 7 +++- arch/x86/kvm/x86.c | 4 +- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 72 +++++++++++++++++++++++++++++++++------ 9 files changed, 87 insertions(+), 25 deletions(-)
arch/ia64/kvm/kvm-ia64.c +5 -3
@@ -124,7 +124,7 @@ long ia64_pal_vp_create(u64 *vpd, u64 *host_iva, u64 *opt_handler)static DEFINE_SPINLOCK (vp_lock); -void kvm_arch_hardware_enable (void *garbage) +int kvm_arch_hardware_enable (void *garbage) {long status;long tmp_base;
@@ -137,7 +137,7 @@ void kvm_arch_hardware_enable(void *garbage) slot =ia64_itr_entry (0x3 ,KVM_VMM_BASE , pte,KVM_VMM_SHIFT );local_irq_restore (saved_psr);if (slot <0 ) -return ; +return -EINVAL ;spin_lock (&vp_lock); status = ia64_pal_vp_init_env(kvm_vsa_base ?
@@ -145,7 +145,7 @@ void kvm_arch_hardware_enable(void *garbage) __pa(kvm_vm_buffer), KVM_VM_BUFFER_BASE, &tmp_base);if (status !=0 ) {printk (KERN_WARNING "kvm: Failed to Enable VT Support!!!!\n" ); -return ; +return -EINVAL ; }if (!kvm_vsa_base) {
@@ -154,6 +154,8 @@ void kvm_arch_hardware_enable(void *garbage) }spin_unlock (&vp_lock);ia64_ptr_entry (0x3 , slot); + +return 0 ; }void kvm_arch_hardware_disable (void *garbage)
arch/powerpc/kvm/powerpc.c +1 -1
@@ -80,7 +80,7 @@ int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu)return r; } -void kvm_arch_hardware_enable (void *garbage) +int kvm_arch_hardware_enable (void *garbage) { }
arch/s390/kvm/kvm-s390.c +1 -1
@@ -70,7 +70,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {/* Section: not file related */ -void kvm_arch_hardware_enable (void *garbage) +int kvm_arch_hardware_enable (void *garbage) {/* every s390 is virtualization enabled ;-) */ }
arch/x86/include/asm/kvm_host.h +1 -1
@@ -463,7 +463,7 @@ struct descriptor_table {struct kvm_x86_ops {int (*cpu_has_kvm_support)(void );/* __init */ int (*disabled_by_bios)(void );/* __init */ -void (*hardware_enable)(void *dummy);/* __init */ +int (*hardware_enable)(void *dummy);void (*hardware_disable)(void *dummy);void (*check_processor_compatibility)(void *rtn);int (*hardware_setup)(void );/* __init */
arch/x86/kvm/svm.c +9 -4
@@ -262,7 +262,7 @@ static void svm_hardware_disable(void *garbage)cpu_svm_disable (); } -static void svm_hardware_enable (void *garbage) +static int svm_hardware_enable (void *garbage) {struct svm_cpu_data *svm_data;
@@ -271,16 +271,20 @@ static void svm_hardware_enable(void *garbage)struct desc_struct *gdt;int me =raw_smp_processor_id (); +rdmsrl (MSR_EFER , efer); +if (efer &EFER_SVME ) +return -EBUSY ; +if (!has_svm ()) {printk (KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n" , me); -return ; +return -EINVAL ; } svm_data =per_cpu (svm_data, me);if (!svm_data) {printk (KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n" , me); -return ; +return -EINVAL ; } svm_data->asid_generation =1 ;
@@ -291,11 +295,12 @@ static void svm_hardware_enable(void *garbage) gdt = (struct desc_struct *)gdt_descr.address; svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt +GDT_ENTRY_TSS ); -rdmsrl (MSR_EFER , efer);wrmsrl (MSR_EFER , efer |EFER_SVME );wrmsrl (MSR_VM_HSAVE_PA ,page_to_pfn (svm_data->save_area) <<PAGE_SHIFT ); + +return 0 ; }static void svm_cpu_uninit (int cpu)
arch/x86/kvm/vmx.c +6 -1
@@ -1068,12 +1068,15 @@ static __init int vmx_disabled_by_bios(void)/* locked but not enabled */ } -static void hardware_enable (void *garbage) +static int hardware_enable (void *garbage) {int cpu =raw_smp_processor_id ();u64 phys_addr =__pa (per_cpu (vmxarea, cpu));u64 old; +if (read_cr4 () &X86_CR4_VMXE ) +return -EBUSY ; +INIT_LIST_HEAD (&per_cpu (vcpus_on_cpu, cpu));rdmsrl (MSR_IA32_FEATURE_CONTROL , old); if ((old & (FEATURE_CONTROL_LOCKED |
@@ -1088,6 +1091,8 @@ static void hardware_enable(void *garbage) asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr) : "memory", "cc"); + +return 0 ; }static void vmclear_local_vcpus (void )
arch/x86/kvm/x86.c +2 -2
@@ -4220,9 +4220,9 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)return kvm_x86_ops->vcpu_reset (vcpu); } -void kvm_arch_hardware_enable (void *garbage) +int kvm_arch_hardware_enable (void *garbage) { - kvm_x86_ops->hardware_enable (garbage); +return kvm_x86_ops->hardware_enable (garbage); }void kvm_arch_hardware_disable (void *garbage)
include/linux/kvm_host.h +1 -1
@@ -292,7 +292,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);void kvm_arch_vcpu_destroy (struct kvm_vcpu *vcpu);int kvm_arch_vcpu_reset (struct kvm_vcpu *vcpu); -void kvm_arch_hardware_enable (void *garbage); +int kvm_arch_hardware_enable (void *garbage);void kvm_arch_hardware_disable (void *garbage);int kvm_arch_hardware_setup (void );void kvm_arch_hardware_unsetup (void );
virt/kvm/kvm_main.c +61 -11
154 lines not on this page · raw message
Signature · 1 line
1.6.0.2
Quoted 23 lines · Alexander Graf
X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).
Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.
To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.
So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.
virt/kvm/kvm_main.c +0
-0
virt/kvm/kvm_main.c DEFINE_SPINLOCK(kvm_lock); · line 70
Quoted 6 lines
virt/kvm/kvm_main.c +2
-0@@ -65,6 +65,8 @@ DEFINE_SPINLOCK(kvm_lock);LIST_HEAD (vm_list);static cpumask_var_t cpus_hardware_enabled; +static int kvm_usage_count =0 ; +static DEFINE_SPINLOCK (kvm_usage_lock);
Please use kvm_lock for this.
Quoted 12 lines
virt/kvm/kvm_main.c +4
-1@@ -2327,14 +2341,40 @@ static struct miscdevice kvm_dev = { &kvm_chardev_ops, }; -static void hardware_enable (void *junk) +static void hardware_enable (void *_r) {int cpu =raw_smp_processor_id (); +int r; + +/* If enabling a previous CPU failed already, let's not continue */
virt/kvm/kvm_main.c static struct miscdevice kvm_dev = { · line 2359
Quoted 10 lines
virt/kvm/kvm_main.c +5
-0+if (_r && *((int *)_r)) +return ;if (cpumask_test_cpu (cpu, cpus_hardware_enabled))return ; + r =kvm_arch_hardware_enable (NULL ); +if (_r) + *((int *)_r) = r;
Racy. If one cpu succeeds and another fails, the successful one could overwrite the failing one's result.
While the race will never happen (start two VMMs simultaneously) it will cause an endless stream of complaints. Let's use an atomic_t incremented on each failure.
Oh, and it can be global since we're inside a lock, so some of the changes to add a return value become unnecessary.
virt/kvm/kvm_main.c static void hardware_disable(void *junk)
Quoted 6 lines
virt/kvm/kvm_main.c +4
-0+static void hardware_disable_all (void ) +{ +if (!kvm_usage_count) +return ;
Can this happen?
virt/kvm/kvm_main.c static void hardware_disable(void *junk)
Quoted 9 lines
virt/kvm/kvm_main.c +8
-0+ +spin_lock (&kvm_usage_lock); + kvm_usage_count--; +if (!kvm_usage_count) +on_each_cpu (hardware_disable,NULL ,1 ); +spin_unlock (&kvm_usage_lock); +} +
Please make sure cpu hotplug/hotunplug (and this suspend/resume) still work.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 19 lines · Avi Kivity
Quoted 17 lines · Alexander Graf
X86 CPUs need to have some magic happening to enable the virtualization extensions on them. This magic can result in unpleasant results for users, like blocking other VMMs from working (vmx) or using invalid TLB entries (svm).
Currently KVM activates virtualization when the respective kernel module is loaded. This blocks us from autoloading KVM modules without breaking other VMMs.
To circumvent this problem at least a bit, this patch introduces on demand activation of virtualization. This means, that instead virtualization is enabled on creation of the first virtual machine and disabled on destruction of the last one.
So using this, KVM can be easily autoloaded, while keeping other hypervisors usable.
Quoted 12 lines
virt/kvm/kvm_main.c DEFINE_SPINLOCK(kvm_lock); · line 69
Quoted 10 lines
virt/kvm/kvm_main.c +2
-0@@ -65,6 +65,8 @@ DEFINE_SPINLOCK(kvm_lock);LIST_HEAD (vm_list);static cpumask_var_t cpus_hardware_enabled; +static int kvm_usage_count =0 ; +static DEFINE_SPINLOCK (kvm_usage_lock);Please use
kvm_lockfor this.
Looks good. Will do :-).
Quoted 24 lines
Quoted 14 lines
virt/kvm/kvm_main.c +4
-1@@ -2327,14 +2341,40 @@ static struct miscdevice kvm_dev = { &kvm_chardev_ops, }; -static void hardware_enable (void *junk) +static void hardware_enable (void *_r) {int cpu =raw_smp_processor_id (); +int r; + +/* If enabling a previous CPU failed already, let's not continue */
virt/kvm/kvm_main.c static struct miscdevice kvm_dev = { · line 2357
Quoted 7 lines
virt/kvm/kvm_main.c +5
-0+if (_r && *((int *)_r)) +return ;if (cpumask_test_cpu (cpu, cpus_hardware_enabled))return ; + r =kvm_arch_hardware_enable (NULL ); +if (_r) + *((int *)_r) = r;Racy. If one cpu succeeds and another fails, the successful one could overwrite the failing one's result.
Quoted 3 lines
While the race will never happen (start two VMMs simultaneously) it will cause an endless stream of complaints. Let's use an
atomic_tincremented on each failure.Oh, and it can be global since we're inside a lock, so some of the changes to add a return value become unnecessary.
Right, that probably cleans up things a bit.
Quoted 9 lines
virt/kvm/kvm_main.c static void hardware_disable(void *junk) · line 2393
Quoted 7 lines
virt/kvm/kvm_main.c +4
-0+static void hardware_disable_all (void ) +{ +if (!kvm_usage_count) +return ;Can this happen?
It should not. Better make it a BUG(...)?
Quoted 13 lines
virt/kvm/kvm_main.c static void hardware_disable(void *junk) · line 2401
Quoted 11 lines
virt/kvm/kvm_main.c +8
-0+ +spin_lock (&kvm_usage_lock); + kvm_usage_count--; +if (!kvm_usage_count) +on_each_cpu (hardware_disable,NULL ,1 ); +spin_unlock (&kvm_usage_lock); +} +Please make sure cpu hotplug/hotunplug (and this suspend/resume) still work.
Make sure as in test?
Alex
Quoted 10 lines
Signature · 8 lines
-- error compiling committee.c: too many arguments to function -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 4 lines · Alexander Graf
Quoted 2 lines
Please make sure cpu hotplug/hotunplug (and this suspend/resume) still work.
Make sure as in test?
I use this:
#!/usr/bin/python
import commands, time, random
processors = int(commands.getoutput('getconf _NPROCESSORS_CONF'))
def fname(cpu):
return '/sys/devices/system/cpu/cpu%s/online' % (cpu,)
def get_state(cpu):
return int(file(fname(cpu)).read())
def set_state(cpu, state):
file(fname(cpu), 'w').write(str(state))
try:
while True:
cpu = random.randint(1, processors - 1)
state = 1 - get_state(cpu)
set_state(cpu, state)
time.sleep(1)
except KeyboardInterrupt:
for cpu in range(1, processors):
if not get_state(cpu):
set_state(cpu, 1)
Signature · 2 lines
I have a truly marvellous patch that fixes the bug which this signature is too narrow to contain.