Skip to contentSkip to conversation

x86/S3: Restore broken vcpu affinity on resume

PATCH In review Ben Guthro 2 participants 4 messages Mar 26, 2013

lore
Share & tools
lore

Ben Guthro

author
#0 lore raw

PATCH 1/1 x86/S3: Restore broken vcpu affinity on resume

When in SYS_STATE_suspend, and going through the cpu_disable_scheduler path, save a copy of the current cpu affinity, and mark a flag to restore it later.

Later, in the resume process, when enabling nonboot cpus restore these affinities.

---
 xen/common/cpu.c           |    3 +++
 xen/common/cpupool.c       |    5 +----
 xen/common/domain.c        |    2 ++
 xen/common/schedule.c      |   46 ++++++++++++++++++++++++++++++++++++++++++++
 xen/include/xen/sched-if.h |    5 +++++
 xen/include/xen/sched.h    |    6 ++++++
 6 files changed, 63 insertions(+), 4 deletions(-)
xen/common/cpu.c +3 -0
index 630881e..786966a 100644
@@ -215,4 +215,7 @@ void enable_nonboot_cpus(void)
     }
 
     cpumask_clear(&frozen_cpus);
+
+    if (system_state == SYS_STATE_resume)
+	restore_vcpu_affinity();
 }
xen/common/cpupool.c +1 -4
index 10b10f8..7a04f5e 100644
@@ -19,13 +19,10 @@
 #include <xen/sched-if.h>
 #include <xen/cpu.h>
 
-#define for_each_cpupool(ptr)    \
-    for ((ptr) = &cpupool_list; *(ptr) != NULL; (ptr) = &((*(ptr))->next))
-
 struct cpupool *cpupool0;                /* Initial cpupool with Dom0 */
 cpumask_t cpupool_free_cpus;             /* cpus not in any cpupool */
 
-static struct cpupool *cpupool_list;     /* linked list, sorted by poolid */
+struct cpupool *cpupool_list;     /* linked list, sorted by poolid */
 
 static int cpupool_moving_cpu = -1;
 static struct cpupool *cpupool_cpu_moving = NULL;
xen/common/domain.c +2 -0
index 64ee29d..590548e 100644
@@ -126,6 +126,7 @@ struct vcpu *alloc_vcpu(
 
     if ( !zalloc_cpumask_var(&v->cpu_affinity) ||
          !zalloc_cpumask_var(&v->cpu_affinity_tmp) ||
+         !zalloc_cpumask_var(&v->cpu_affinity_saved) ||
          !zalloc_cpumask_var(&v->vcpu_dirty_cpumask) )
         goto fail_free;
 
@@ -155,6 +156,7 @@ struct vcpu *alloc_vcpu(
  fail_free:
         free_cpumask_var(v->cpu_affinity);
         free_cpumask_var(v->cpu_affinity_tmp);
+        free_cpumask_var(v->cpu_affinity_saved);
         free_cpumask_var(v->vcpu_dirty_cpumask);
         free_vcpu_struct(v);
         return NULL;
xen/common/schedule.c +46 -0

58 lines not on this page · raw message

xen/include/xen/sched-if.h +5 -0
index 9ace22c..547e71e 100644
@@ -13,6 +13,9 @@
 /* A global pointer to the initial cpupool (POOL0). */
 extern struct cpupool *cpupool0;
 
+/* linked list of cpu pools */
+extern struct cpupool *cpupool_list;
+
 /* cpus currently in no cpupool */
 extern cpumask_t cpupool_free_cpus;
 
@@ -211,5 +214,7 @@ struct cpupool
     (((_pool) == NULL) ? &cpupool_free_cpus : (_pool)->cpu_valid)
 #define cpupool_online_cpumask(_pool) \
     (((_pool) == NULL) ? &cpu_online_map : (_pool)->cpu_valid)
+#define for_each_cpupool(ptr)    \
+    for ((ptr) = &cpupool_list; *(ptr) != NULL; (ptr) = &((*(ptr))->next))
 
 #endif /* __XEN_SCHED_IF_H__ */
xen/include/xen/sched.h +6 -0
index cabaf27..d24fc6b 100644
@@ -153,6 +153,9 @@ struct vcpu
     bool_t           defer_shutdown;
     /* VCPU is paused following shutdown request (d->is_shutting_down)? */
     bool_t           paused_for_shutdown;
+    /* VCPU need affinity restored */
+    bool_t           affinity_broken;
+
 
     /*
      * > 0: a single port is being polled;
@@ -175,6 +178,8 @@ struct vcpu
     cpumask_var_t    cpu_affinity;
     /* Used to change affinity temporarily. */
     cpumask_var_t    cpu_affinity_tmp;
+    /* Used to restore affinity across S3. */
+    cpumask_var_t    cpu_affinity_saved;
 
     /* Bitmask of CPUs which are holding onto this VCPU's state. */
     cpumask_var_t    vcpu_dirty_cpumask;
@@ -697,6 +702,7 @@ int schedule_cpu_switch(unsigned int cpu, struct cpupool *c);
 void vcpu_force_reschedule(struct vcpu *v);
 int cpu_disable_scheduler(unsigned int cpu);
 int vcpu_set_affinity(struct vcpu *v, const cpumask_t *affinity);
+void restore_vcpu_affinity(void);
 
 void vcpu_runstate_get(struct vcpu *v, struct vcpu_runstate_info *runstate);
 uint64_t get_cpu_idle_time(unsigned int cpu);
Signature · 1 line
1.7.9.5
#1 lore raw
Quoted 9 lines
Quoted 1 line
Quoted 1 line · Ben Guthro

When in SYS_STATE_suspend, and going through the cpu_disable_scheduler path, save a copy of the current cpu affinity, and mark a flag to restore it later.

Later, in the resume process, when enabling nonboot cpus restore these affinities.

Looks reasonable to me, minus severe formatting issues (hard tabs being the most obvious, but not the only problem).

Quoted 13 lines
xen/include/xen/sched-if.h +3 -0
@@ -13,6 +13,9 @@
 /* A global pointer to the initial cpupool (POOL0). */
 extern struct cpupool *cpupool0;
 
+/* linked list of cpu pools */
+extern struct cpupool *cpupool_list;
+
 /* cpus currently in no cpupool */
 extern cpumask_t cpupool_free_cpus;
 

xen/include/xen/sched-if.h struct cpupool · line 220

Quoted 7 lines
xen/include/xen/sched-if.h +2 -0
@@ -211,5 +214,7 @@ struct cpupool
     (((_pool) == NULL) ? &cpupool_free_cpus : (_pool)->cpu_valid)
 #define cpupool_online_cpumask(_pool) \
     (((_pool) == NULL) ? &cpu_online_map : (_pool)->cpu_valid)
+#define for_each_cpupool(ptr)    \
+    for ((ptr) = &cpupool_list; *(ptr) != NULL; (ptr) = &((*(ptr))->next))
 
 #endif /* __XEN_SCHED_IF_H__ */

I don't particularly like the increase of scope for these two, but I guess there aren't many alternatives.

Jan

author
#2 lore raw
Quoted 3 lines · Jan Beulich

Looks reasonable to me, minus severe formatting issues (hard tabs being the most obvious, but not the only problem).

Thanks for the review - Apologies on the formatting.
Would you prefer I clean up the formatting and re-submit, or would you prefer to take care of it prior to commit

Quoted 3 lines

I don't particularly like the increase of scope for these two, but I guess there aren't many alternatives.

Indeed.

My first pass at this only operated on cpupool0, which did not require this scope increase...but it was pointed out that was probably not sufficient to be acceptable in a more general solution appropriate to upstream.

Thanks

Ben

#3 lore raw
Quoted 10 lines
Quoted 1 line
Quoted 1 line · Ben Guthro
Quoted 3 lines · Jan Beulich

Looks reasonable to me, minus severe formatting issues (hard tabs being the most obvious, but not the only problem).

Thanks for the review - Apologies on the formatting.
Would you prefer I clean up the formatting and re-submit, or would you prefer to take care of it prior to commit

Considering that this isn't just one or two places, I'd prefer you to re-submit.

Jan