don't schedule unplugged vcpus

diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c
@@ -154,8 +154,13 @@
 				cpu_clear(cpu, cpu_idle_map);
 			rmb();
 
-			if (cpu_is_offline(cpu))
+			if (cpu_is_offline(cpu)) {
+#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU)
+            /* Tell hypervisor not to schedule dead vcpus */
+            HYPERVISOR_vcpu_down(cpu);
+#endif
 				play_dead();
+         }
 
 			irq_stat[cpu].idle_timestamp = jiffies;
 			xen_idle();

diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c c/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
@@ -1380,6 +1380,10 @@
 	}
 
 #ifdef CONFIG_HOTPLUG_CPU
+#ifdef CONFIG_XEN
+   /* Tell hypervisor to bring vcpu up */
+   HYPERVISOR_vcpu_up(cpu);
+#endif
 	/* Already up, and in cpu_quiescent now? */
 	if (cpu_isset(cpu, smp_commenced_mask)) {
 		cpu_enable(cpu);

diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h c/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h
@@ -517,4 +517,35 @@
     return ret;
 }
 
+static inline int
+HYPERVISOR_vcpu_down(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+	: "0" (__HYPERVISOR_sched_op),
+	  "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
+
+static inline int
+HYPERVISOR_vcpu_up(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+	: "0" (__HYPERVISOR_sched_op),
+	  "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
 #endif /* __HYPERCALL_H__ */

diff -urN b/tools/libxc/xc.h c/tools/libxc/xc.h
@@ -118,6 +118,7 @@
     u64           cpu_time;
     unsigned long max_memkb;
     unsigned int  vcpus;
+    u32           vcpu_online_map;
     s32           vcpu_to_cpu[MAX_VIRT_CPUS];
     cpumap_t      cpumap[MAX_VIRT_CPUS];
 } xc_dominfo_t;

diff -urN b/tools/libxc/xc_domain.c c/tools/libxc/xc_domain.c
@@ -106,6 +106,7 @@
         info->shared_info_frame = op.u.getdomaininfo.shared_info_frame;
         info->cpu_time = op.u.getdomaininfo.cpu_time;
         info->vcpus = op.u.getdomaininfo.n_vcpu;
+        info->vcpu_online_map = op.u.getdomaininfo.vcpu_online_map;
         memcpy(&info->vcpu_to_cpu, &op.u.getdomaininfo.vcpu_to_cpu, 
                sizeof(info->vcpu_to_cpu));
         memcpy(&info->cpumap, &op.u.getdomaininfo.cpumap, 

diff -urN b/tools/python/xen/lowlevel/xc/xc.c c/tools/python/xen/lowlevel/xc/xc.c
@@ -199,7 +199,7 @@
                                      PyObject *kwds)
 {
     XcObject *xc = (XcObject *)self;
-    PyObject *list, *vcpu_list, *cpumap_list, *info_dict;
+    PyObject *list, *vcpu_list, *online_list, *cpumap_list, *info_dict;
 
     u32 first_dom = 0;
     int max_doms = 1024, nr_doms, i, j;

@@ -221,11 +221,15 @@
     {
         vcpu_list = PyList_New(MAX_VIRT_CPUS);
         cpumap_list = PyList_New(MAX_VIRT_CPUS);
+        online_list = PyList_New(MAX_VIRT_CPUS);
         for ( j = 0; j < MAX_VIRT_CPUS; j++ ) {
             PyList_SetItem( vcpu_list, j, 
                             Py_BuildValue("i", info[i].vcpu_to_cpu[j]));
             PyList_SetItem( cpumap_list, j, 
                             Py_BuildValue("i", info[i].cpumap[j]));
+            PyList_SetItem( online_list, j, 
+                            Py_BuildValue("i", 
+                               ((1<<j) & info[i].vcpu_online_map)>>j));
         }
                  
         info_dict = Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i"

@@ -244,6 +248,7 @@
                                   "shutdown_reason", info[i].shutdown_reason);
         PyDict_SetItemString( info_dict, "vcpu_to_cpu", vcpu_list );
         PyDict_SetItemString( info_dict, "cpumap", cpumap_list );
+        PyDict_SetItemString( info_dict, "vcpu_online_map", online_list );
         PyList_SetItem( list, i, info_dict);
  
     }

diff -urN b/xen/common/dom0_ops.c c/xen/common/dom0_ops.c
@@ -304,6 +304,7 @@
         struct vcpu   *v;
         u64 cpu_time = 0;
         int vcpu_count = 0;
+        u32 vcpu_online_map = 0;
         int flags = DOMFLAGS_PAUSED | DOMFLAGS_BLOCKED;
 
         read_lock(&domlist_lock);

@@ -346,11 +347,15 @@
                 flags |= DOMFLAGS_RUNNING;
             if ( v->cpu_time > cpu_time )
                 cpu_time += v->cpu_time;
+            /* mark which vcpus are not down */
+            if ( !(test_bit(_VCPUF_down, &v->vcpu_flags)) )
+                vcpu_online_map |= (1<<v->vcpu_id);
             vcpu_count++;
         }
 
         op->u.getdomaininfo.cpu_time = cpu_time;
         op->u.getdomaininfo.n_vcpu = vcpu_count;
+        op->u.getdomaininfo.vcpu_online_map = vcpu_online_map;
 
         op->u.getdomaininfo.flags = flags |
             ((d->domain_flags & DOMF_dying)    ? DOMFLAGS_DYING    : 0) |

@@ -373,7 +378,8 @@
     { 
         struct vcpu_guest_context *c;
         struct domain             *d;
-        struct vcpu               *v;
+        struct vcpu               *v = NULL;
+        int i;
 
         d = find_domain_by_id(op->u.getvcpucontext.domain);
         if ( d == NULL )

@@ -388,8 +394,15 @@
             put_domain(d);
             break;
         }
+
+        /* find first valid vcpu starting from request. */
+        for ( i=op->u.getvcpucontext.vcpu; i<MAX_VIRT_CPUS; i++ ) 
+        {
+            v = d->vcpu[i];
+            if ( v != NULL && !(test_bit(_VCPUF_down, &v->vcpu_flags)) )
+                break;
+        }
         
-        v = d->vcpu[op->u.getvcpucontext.vcpu];
         if ( v == NULL )
         {
             ret = -ESRCH;

60 lines not on this page · raw message

diff -urN b/xen/include/public/dom0_ops.h c/xen/include/public/dom0_ops.h
@@ -85,7 +85,8 @@
     memory_t max_pages;
     memory_t shared_info_frame;       /* MFN of shared_info struct */
     u64      cpu_time;
-    u32      n_vcpu;
+    u32      n_vcpu;                      /* number of vcpus allocated */
+    u32      vcpu_online_map;             /* bitmap of online vcpus */
     s32      vcpu_to_cpu[MAX_VIRT_CPUS];  /* current mapping   */
     cpumap_t cpumap[MAX_VIRT_CPUS];       /* allowable mapping */
 } dom0_getdomaininfo_t;

diff -urN b/xen/include/public/xen.h c/xen/include/public/xen.h
@@ -200,8 +200,11 @@
 #define SCHEDOP_yield           0   /* Give up the CPU voluntarily.       */
 #define SCHEDOP_block           1   /* Block until an event is received.  */
 #define SCHEDOP_shutdown        2   /* Stop executing this domain.        */
+#define SCHEDOP_vcpu_down       3   /* make target VCPU not-runnable.     */
+#define SCHEDOP_vcpu_up         4   /* make target VCPU runnable.         */
 #define SCHEDOP_cmdmask       255   /* 8-bit command. */
 #define SCHEDOP_reasonshift     8   /* 8-bit reason code. (SCHEDOP_shutdown) */
+#define SCHEDOP_vcpushift       8   /* 8-bit VCPU target. (SCHEDOP_up|down) */
 
 /*
  * Reason codes for SCHEDOP_shutdown. These may be interpreted by control 

diff -urN b/xen/include/xen/sched.h c/xen/include/xen/sched.h
@@ -346,6 +346,9 @@
  /* Initialization completed. */
 #define _VCPUF_initialised     8
 #define VCPUF_initialised      (1UL<<_VCPUF_initialised)
+ /* VCPU is not-runnable */
+#define _VCPUF_down            9
+#define VCPUF_down             (1UL<<_VCPUF_down)
 
 /*
  * Per-domain flags (domain_flags).

@@ -375,7 +378,7 @@
 static inline int domain_runnable(struct vcpu *v)
 {
     return ( (atomic_read(&v->pausecnt) == 0) &&
-             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) &&
+             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) &&
              !(v->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) );
 }

diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/process.c
@@ -154,8 +154,13 @@
 				cpu_clear(cpu, cpu_idle_map);
 			rmb();
 
-			if (cpu_is_offline(cpu))
+			if (cpu_is_offline(cpu)) {
+#if defined(CONFIG_XEN) && defined(CONFIG_HOTPLUG_CPU)
+            /* Tell hypervisor not to schedule dead vcpus */
+            HYPERVISOR_vcpu_down(cpu);
+#endif
 				play_dead();
+         }
 
 			irq_stat[cpu].idle_timestamp = jiffies;
 			xen_idle();

diff -urN b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c vcpu_down/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c
@@ -1380,6 +1380,10 @@
 	}
 
 #ifdef CONFIG_HOTPLUG_CPU
+#ifdef CONFIG_XEN
+   /* Tell hypervisor to bring vcpu up */
+   HYPERVISOR_vcpu_up(cpu);
+#endif
 	/* Already up, and in cpu_quiescent now? */
 	if (cpu_isset(cpu, smp_commenced_mask)) {
 		cpu_enable(cpu);

diff -urN b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h vcpu_down/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/hypercall.h
@@ -517,4 +517,35 @@
     return ret;
 }
 
+static inline int
+HYPERVISOR_vcpu_down(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+	: "0" (__HYPERVISOR_sched_op),
+	  "1" (SCHEDOP_vcpu_down | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
+
+static inline int
+HYPERVISOR_vcpu_up(
+    int vcpu)
+{
+    int ret;
+    unsigned long ign1;
+    __asm__ __volatile__ (
+        TRAP_INSTR
+        : "=a" (ret), "=b" (ign1)
+	: "0" (__HYPERVISOR_sched_op),
+	  "1" (SCHEDOP_vcpu_up | (vcpu << SCHEDOP_vcpushift))
+        : "memory" );
+
+    return ret;
+}
 #endif /* __HYPERCALL_H__ */

diff -urN b/xen/common/dom0_ops.c vcpu_down/xen/common/dom0_ops.c
@@ -334,9 +334,14 @@
          * - domain is marked as paused or blocked only if all its vcpus 
          *   are paused or blocked 
          * - domain is marked as running if any of its vcpus is running
+         * - only map vcpus that aren't down.  Note, at some point we may
+         *   wish to demux the -1 value to indicate down vs. not-ever-booted
+         *   
          */
         for_each_vcpu ( d, v ) {
-            op->u.getdomaininfo.vcpu_to_cpu[v->vcpu_id] = v->processor;
+            /* only map vcpus that are up */
+            if ( !(test_bit(_VCPUF_down, &v->vcpu_flags)) )
+                op->u.getdomaininfo.vcpu_to_cpu[v->vcpu_id] = v->processor;
             op->u.getdomaininfo.cpumap[v->vcpu_id]      = v->cpumap;
             if ( !(v->vcpu_flags & VCPUF_ctrl_pause) )
                 flags &= ~DOMFLAGS_PAUSED;

@@ -373,7 +378,8 @@
     { 
         struct vcpu_guest_context *c;
         struct domain             *d;
-        struct vcpu               *v;
+        struct vcpu               *v=NULL;
+        int i;
 
         d = find_domain_by_id(op->u.getvcpucontext.domain);
         if ( d == NULL )

@@ -388,8 +394,15 @@
             put_domain(d);
             break;
         }
+
+        /* find first valid vcpu starting from request. */
+        for ( i=op->u.getvcpucontext.vcpu; i<MAX_VIRT_CPUS; i++ )
+        {
+            v = d->vcpu[i];
+            if ( v != NULL && !(test_bit(_VCPUF_down, &v->vcpu_flags)) )
+                break;
+        }
         
-        v = d->vcpu[op->u.getvcpucontext.vcpu];
         if ( v == NULL )
         {
             ret = -ESRCH;

60 lines not on this page · raw message

diff -urN b/xen/include/public/xen.h vcpu_down/xen/include/public/xen.h
@@ -200,8 +200,11 @@
 #define SCHEDOP_yield           0   /* Give up the CPU voluntarily.       */
 #define SCHEDOP_block           1   /* Block until an event is received.  */
 #define SCHEDOP_shutdown        2   /* Stop executing this domain.        */
+#define SCHEDOP_vcpu_down       3   /* make target VCPU not-runnable.     */
+#define SCHEDOP_vcpu_up         4   /* make target VCPU runnable.         */
 #define SCHEDOP_cmdmask       255   /* 8-bit command. */
 #define SCHEDOP_reasonshift     8   /* 8-bit reason code. (SCHEDOP_shutdown) */
+#define SCHEDOP_vcpushift       8   /* 8-bit VCPU target. (SCHEDOP_up|down) */
 
 /*
  * Reason codes for SCHEDOP_shutdown. These may be interpreted by control 

diff -urN b/xen/include/xen/sched.h vcpu_down/xen/include/xen/sched.h
@@ -346,6 +346,9 @@
  /* Initialization completed. */
 #define _VCPUF_initialised     8
 #define VCPUF_initialised      (1UL<<_VCPUF_initialised)
+ /* VCPU is not-runnable */
+#define _VCPUF_down            9
+#define VCPUF_down             (1UL<<_VCPUF_down)
 
 /*
  * Per-domain flags (domain_flags).

@@ -375,7 +378,7 @@
 static inline int domain_runnable(struct vcpu *v)
 {
     return ( (atomic_read(&v->pausecnt) == 0) &&
-             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) &&
+             !(v->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause|VCPUF_down)) &&
              !(v->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) );
 }