Skip to contentSkip to conversation

vcpu-set increases vcpus beyond the value set at domain creation

PATCH In review Ryan Grimm 2 participants 2 messages Feb 15, 2006 – Feb 16, 2006

lore
Share & tools
lore

Ryan Grimm

author
#0 lore raw

PATCH 1/1 vcpu-set increases vcpus beyond the value set at domain creation

Hi,

This patch allows a domain's vcpus to increase beyond the max (up to CONFIG_NR_CPUS) set at creation time by making 3 changes:

1. xen/common/dom0_ops.c - removes the check in DOM0_MAX_CPUS that doesnt allow vcpus to be alloc'd after the first vcpu has been initialiazed.

2. xend/XendDomainInfo.py - when vcpu-set is given a value greater than the current vcpu count, the dom0_op above is called with the new value and the store is updated.

3. drivers/xen/core/smpboot.c - the cpu_possible_map and cpu_present_map made to reflect the values outlined in include/linux/cpumask.h when CONFIG_HOTPLUG_CPU is set:

cpu_possible_map - all NR_CPUS bits set cpu_present_map - has bit 'cpu' set iff cpu is populated
cpu_online_map - has bit 'cpu' set iff cpu available to
scheduler

This has been tested on x86_32 and x86_64 with and without CONFIG_HOTPLUG_CPU.

Does anyone know what potential races are being avoided by the check in dom0_ops.c that this patch removes?

Any comments on the patch?

Thanks,
Ryan

linux-2.6-xen-sparse/drivers/xen/core/smpboot.c +27 -16
diff -r 1346a69694be -r 334b38a81c7d linux-2.6-xen-sparse/drivers/xen/core/smpboot.c
@@ -57,6 +57,7 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 DEFINE_PER_CPU(int, cpu_state) = { 0 };
+static int vcpu_allocated(unsigned int vcpu);
 #endif
 
 static DEFINE_PER_CPU(int, resched_irq);
@@ -81,15 +82,17 @@
 
 void __init prefill_possible_map(void)
 {
-	int i, rc;
+	int i;
 
 	if (!cpus_empty(cpu_possible_map))
 		return;
 
 	for (i = 0; i < NR_CPUS; i++) {
-		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
+#ifndef CONFIG_HOTPLUG_CPU
+		int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 		if (rc == -ENOENT)
 			break;
+#endif
 		cpu_set(i, cpu_possible_map);
 	}
 }
@@ -257,11 +260,15 @@
 #ifdef CONFIG_HOTPLUG_CPU
 		if (xen_start_info->flags & SIF_INITDOMAIN)
 			cpu_set(cpu, cpu_present_map);
+		if (vcpu_allocated(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 #else
 		cpu_set(cpu, cpu_present_map);
-#endif
-
 		vcpu_prepare(cpu);
+#endif
+
 	}
 
 	/* Currently, Xen gives no dynamic NUMA/HT info. */
@@ -289,17 +296,15 @@
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-/*
- * Initialize cpu_present_map late to skip SMP boot code in init/main.c.
- * But do it early enough to catch critical for_each_present_cpu() loops
- * in i386-specific code.
- */
-static int __init initialize_cpu_present_map(void)
-{
-	cpu_present_map = cpu_possible_map;
-	return 0;
-}
-core_initcall(initialize_cpu_present_map);
+static int vcpu_allocated(unsigned int vcpu)
+{
+	int rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, vcpu, NULL);
+	if (rc >= 0)
+		rc = 1;
+	else	
+		rc = 0;
+	return rc;
+}
 
 static void vcpu_hotplug(unsigned int cpu)
 {
@@ -312,11 +317,17 @@
 	sprintf(dir, "cpu/%d", cpu);
 	err = xenbus_scanf(XBT_NULL, dir, "availability", "%s", state);
 	if (err != 1) {
-		printk(KERN_ERR "XENBUS: Unable to read cpu state\n");
+		printk(KERN_ERR "XENBUS: Unable to read cpu %d state\n", cpu);
 		return;
 	}
 
 	if (strcmp(state, "online") == 0) {
+		if (!vcpu_allocated(cpu))
+			return;
+		if (!cpu_present(cpu)) {
+			vcpu_prepare(cpu);
+			cpu_set(cpu, cpu_present_map);
+		}
 		(void)cpu_up(cpu);
 	} else if (strcmp(state, "offline") == 0) {
 		(void)cpu_down(cpu);
tools/python/xen/xend/XendDomainInfo.py +3 -0
diff -r 1346a69694be -r 334b38a81c7d tools/python/xen/xend/XendDomainInfo.py
@@ -750,6 +750,9 @@
 
 
     def setVCpuCount(self, vcpus):
+        if vcpus > self.getVCpuCount():
+            xc.domain_max_vcpus(self.domid, vcpus)
+            self.info['vcpus'] = vcpus
         self.info['vcpu_avail'] = (1 << vcpus) - 1
         self.storeVm('vcpu_avail', self.info['vcpu_avail'])
         self.writeDom(self.vcpuDomDetails())
xen/common/dom0_ops.c +0 -9
diff -r 1346a69694be -r 334b38a81c7d xen/common/dom0_ops.c
@@ -236,15 +236,6 @@
         if ( (d = find_domain_by_id(op->u.max_vcpus.domain)) == NULL )
             break;
 
-        /*
-         * Can only create new VCPUs while the domain is not fully constructed
-         * (and hence not runnable). Xen needs auditing for races before
-         * removing this check.
-         */
-        ret = -EINVAL;
-        if ( test_bit(_VCPUF_initialised, &d->vcpu[0]->vcpu_flags) )
-            goto maxvcpu_out;
-
         /* We cannot reduce maximum VCPUs. */
         ret = -EINVAL;
         if ( (max != MAX_VIRT_CPUS) && (d->vcpu[max] != NULL) )
#1 lore raw
Quoted 2 lines · Ryan Grimm

This patch allows a domain's vcpus to increase beyond the max (up to CONFIG_NR_CPUS) set at creation time by making 3 changes:

I'd prefer to keep the current Xen mechanisms, but extend xend and/or config file formats so that we can distinguish max_vcpus from initial_vcpus. Currently the two values are conflated. Then you can set max_vcpus as high as you like, but xenstore will tell the guest how many CPUs to bring up during boot.

If we want a 'hard limit' check in Xen (kind of like we have a per-domain memory limit) to ensure that guests do not sneakily bring up CPUs that we didn't ask them to, then we can add that but it's an orthogonal change (i.e., different patch) to what you are trying to do here.

-- Keir