Skip to contentSkip to conversation

x86/xen: Fix lazy mmu handling across context switch

PATCH In review Juergen Gross 2 participants 4 messages May 8

Subsystems (4)
lore
Share & tools
lore

Juergen Gross

author maintainer
#0 lore raw

PATCH 1/1 x86/xen: Fix lazy mmu handling across context switch

The recent rework of mmu lazy mode has resulted in problems when running as a Xen PV guest. Enabling lazy mmu mode for the new context during context switch is done from the arch_end_context_switch() hook, but when calling this hook current hasn't been changed yet, so the lazy mmu mode state of the wrong task is modified.

Additionally it is much cleaner to use lazy_mmu_mode_pause() and lazy_mmu_mode_resume() in the Xen context switch hooks, as it avoids conditionals in those hooks.

In order not having to add another hook to be called after switching current, modify lazy_mmu_mode_resume() to use a new sub-function which takes a task pointer as parameter. This new sub-function can then be used in the xen_end_context_switch() hook.

Fixes 291b3abed657 ("x86/xen: use lazy_mmu_state when context-switching")
---
 arch/x86/xen/enlighten_pv.c |  7 ++-----
 include/linux/pgtable.h     | 33 ++++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 14 deletions(-)
arch/x86/xen/enlighten_pv.c +2 -5
index ed2d7a3756ce..67bb6bf6d240 100644
@@ -424,9 +424,7 @@ static void xen_start_context_switch(struct task_struct *prev)
 {
 	BUG_ON(preemptible());
 
-	if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) {
-		arch_leave_lazy_mmu_mode();
-	}
+	lazy_mmu_mode_pause();
 	enter_lazy(XEN_LAZY_CPU);
 }
 
@@ -436,8 +434,7 @@ static void xen_end_context_switch(struct task_struct *next)
 
 	xen_mc_flush();
 	leave_lazy(XEN_LAZY_CPU);
-	if (__task_lazy_mmu_mode_active(next))
-		arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_resume_task(next);
 }
 
 static unsigned long xen_store_tr(void)
include/linux/pgtable.h +24 -9
index cdd68ed3ae1a..83a099bf2038 100644
@@ -326,6 +326,28 @@ static inline void lazy_mmu_mode_pause(void)
 		arch_leave_lazy_mmu_mode();
 }
 
+/**
+ * lazy_mmu_mode_resume_task() - Resume the lazy MMU mode for a specific task.
+ *
+ * Like lazy_mmu_mode_resume() below, but with a task specified.
+ * Must be called only by lazy_mmu_mode_resume() or during context switch.
+ * Must never be called in interrupt context.
+ *
+ * Must match a call to lazy_mmu_mode_pause().
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ */
+static inline void lazy_mmu_mode_resume_task(struct task_struct *task)
+{
+	struct lazy_mmu_state *state = &task->lazy_mmu_state;
+
+	VM_WARN_ON_ONCE(state->pause_count == 0);
+
+	if (--state->pause_count == 0 && state->enable_count > 0)
+		arch_enter_lazy_mmu_mode();
+}
+
 /**
  * lazy_mmu_mode_resume() - Resume the lazy MMU mode.
  *
@@ -341,15 +363,8 @@ static inline void lazy_mmu_mode_pause(void)
  */
 static inline void lazy_mmu_mode_resume(void)
 {
-	struct lazy_mmu_state *state = &current->lazy_mmu_state;
-
-	if (in_interrupt())
-		return;
-
-	VM_WARN_ON_ONCE(state->pause_count == 0);
-
-	if (--state->pause_count == 0 && state->enable_count > 0)
-		arch_enter_lazy_mmu_mode();
+	if (!in_interrupt())
+		lazy_mmu_mode_resume_task(current);
 }
 #else
 static inline void lazy_mmu_mode_enable(void) {}
Signature · 1 line
2.54.0
author maintainer
#1 lore raw

Please disregard this patch. It isn't fixing the real problem.

Quoted 98 lines · Juergen Gross

The recent rework of mmu lazy mode has resulted in problems when running as a Xen PV guest. Enabling lazy mmu mode for the new context during context switch is done from the arch_end_context_switch() hook, but when calling this hook current hasn't been changed yet, so the lazy mmu mode state of the wrong task is modified.

Additionally it is much cleaner to use lazy_mmu_mode_pause() and lazy_mmu_mode_resume() in the Xen context switch hooks, as it avoids conditionals in those hooks.

In order not having to add another hook to be called after switching current, modify lazy_mmu_mode_resume() to use a new sub-function which takes a task pointer as parameter. This new sub-function can then be used in the xen_end_context_switch() hook.

Fixes 291b3abed657 ("x86/xen: use lazy_mmu_state when context-switching")
---
  arch/x86/xen/enlighten_pv.c |  7 ++-----
  include/linux/pgtable.h     | 33 ++++++++++++++++++++++++---------
  2 files changed, 26 insertions(+), 14 deletions(-)
arch/x86/xen/enlighten_pv.c +2 -5
index ed2d7a3756ce..67bb6bf6d240 100644
@@ -424,9 +424,7 @@ static void xen_start_context_switch(struct task_struct *prev)
  {
  	BUG_ON(preemptible());
 
-	if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) {
-		arch_leave_lazy_mmu_mode();
-	}
+	lazy_mmu_mode_pause();
  	enter_lazy(XEN_LAZY_CPU);
  }
 
@@ -436,8 +434,7 @@ static void xen_end_context_switch(struct task_struct *next)
 
  	xen_mc_flush();
  	leave_lazy(XEN_LAZY_CPU);
-	if (__task_lazy_mmu_mode_active(next))
-		arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_resume_task(next);
  }
 
  static unsigned long xen_store_tr(void)
include/linux/pgtable.h +24 -9
index cdd68ed3ae1a..83a099bf2038 100644
@@ -326,6 +326,28 @@ static inline void lazy_mmu_mode_pause(void)
  		arch_leave_lazy_mmu_mode();
  }
 
+/**
+ * lazy_mmu_mode_resume_task() - Resume the lazy MMU mode for a specific task.
+ *
+ * Like lazy_mmu_mode_resume() below, but with a task specified.
+ * Must be called only by lazy_mmu_mode_resume() or during context switch.
+ * Must never be called in interrupt context.
+ *
+ * Must match a call to lazy_mmu_mode_pause().
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ */
+static inline void lazy_mmu_mode_resume_task(struct task_struct *task)
+{
+	struct lazy_mmu_state *state = &task->lazy_mmu_state;
+
+	VM_WARN_ON_ONCE(state->pause_count == 0);
+
+	if (--state->pause_count == 0 && state->enable_count > 0)
+		arch_enter_lazy_mmu_mode();
+}
+
  /**
   * lazy_mmu_mode_resume() - Resume the lazy MMU mode.
   *
@@ -341,15 +363,8 @@ static inline void lazy_mmu_mode_pause(void)
   */
  static inline void lazy_mmu_mode_resume(void)
  {
-	struct lazy_mmu_state *state = &current->lazy_mmu_state;
-
-	if (in_interrupt())
-		return;
-
-	VM_WARN_ON_ONCE(state->pause_count == 0);
-
-	if (--state->pause_count == 0 && state->enable_count > 0)
-		arch_enter_lazy_mmu_mode();
+	if (!in_interrupt())
+		lazy_mmu_mode_resume_task(current);
  }
  #else
  static inline void lazy_mmu_mode_enable(void) {}
#2 lore raw
Quoted 1 line · Jürgen Groß

Please disregard this patch. It isn't fixing the real problem.

That's what I would expect, see below.

Quoted 7 lines · Juergen Gross
Quoted 5 lines

The recent rework of mmu lazy mode has resulted in problems when running as a Xen PV guest. Enabling lazy mmu mode for the new context during context switch is done from the arch_end_context_switch() hook, but when calling this hook current hasn't been changed yet, so the lazy mmu mode state of the wrong task is modified.

Currently xen_end_context_switch() checks if next has lazy MMU mode enabled and if so calls arch_enter_lazy_mmu_mode(), i.e. enter_lazy(XEN_LAZY_MMU). This does *not* modify any task state, rather it writes to the xen_lazy_mode percpu variable.

I've thought about this from various angles when reworking lazy MMU, and the conclusion I made is that arch_{start,end}_context_switch() have no reason to change any task state. On arm64, for instance, we do nothing at all on context switching, since everything lazy MMU-related is tracked in task_struct and therefore already switched.

Xen is trickier because it tracks lazy MMU/CPU state in a percpu variable, so these hooks do need to do something about it. This is entirely Xen-internal though, and there's no reason to be calling generic functions like lazy_mmu_mode_pause() that modify task state.

The idea behind commit 291b3abed657 ("x86/xen: use lazy_mmu_state when context-switching") is that TIF_LAZY_MMU_UPDATES now duplicates lazy_mmu_state in task_struct and we can therefore replace the former with the latter. More specifically, the assumption is that TIF_LAZY_MMU_UPDATES is set if and only if the task has been scheduled out and __task_lazy_mmu_mode_active(task) is true.

Clearly there is something wrong with this assumption, but I still can't put my finger on it. For now I would suggest reverting this commit if that solves the issue Marek reported; the intention was not to introduce any functional change, but only a (minor) optimisation.

- Kevin

Quoted 94 lines
Quoted 93 lines

Additionally it is much cleaner to use lazy_mmu_mode_pause() and lazy_mmu_mode_resume() in the Xen context switch hooks, as it avoids conditionals in those hooks.

In order not having to add another hook to be called after switching current, modify lazy_mmu_mode_resume() to use a new sub-function which takes a task pointer as parameter. This new sub-function can then be used in the xen_end_context_switch() hook.

Fixes 291b3abed657 ("x86/xen: use lazy_mmu_state when

context-switching")

---
  arch/x86/xen/enlighten_pv.c |  7 ++----- include/linux/pgtable.h     | 33 ++++++++++++++++++++++++---------

  2 files changed, 26 insertions(+), 14 deletions(-)
arch/x86/xen/enlighten_pv.c +1 -2
index ed2d7a3756ce..67bb6bf6d240 100644
@@ -424,9 +424,7 @@ static void xen_start_context_switch(struct
 task_struct *prev)
   {
       BUG_ON(preemptible());
   -    if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) {
-        arch_leave_lazy_mmu_mode();
-    }
+    lazy_mmu_mode_pause();
       enter_lazy(XEN_LAZY_CPU);
   }

  @@ -436,8 +434,7 @@ static void xen_end_context_switch(struct task_struct *next)
        xen_mc_flush();
      leave_lazy(XEN_LAZY_CPU);
-    if (__task_lazy_mmu_mode_active(next))
-        arch_enter_lazy_mmu_mode();
+    lazy_mmu_mode_resume_task(next);
  }
    static unsigned long xen_store_tr(void)

include/linux/pgtable.h +0 -0
index cdd68ed3ae1a..83a099bf2038 100644
@@ -326,6 +326,28 @@ static inline void lazy_mmu_mode_pause(void)
           arch_leave_lazy_mmu_mode();
   }

  +/**
+ * lazy_mmu_mode_resume_task() - Resume the lazy MMU mode for a specific task.
+ *
+ * Like lazy_mmu_mode_resume() below, but with a task specified.
+ * Must be called only by lazy_mmu_mode_resume() or during context switch.
+ * Must never be called in interrupt context.
+ *
+ * Must match a call to lazy_mmu_mode_pause().
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ */

include/linux/pgtable.h +10 -0
+static inline void lazy_mmu_mode_resume_task(struct task_struct *task)
+{
+    struct lazy_mmu_state *state = &task->lazy_mmu_state;
+
+    VM_WARN_ON_ONCE(state->pause_count == 0);
+
+    if (--state->pause_count == 0 && state->enable_count > 0)
+        arch_enter_lazy_mmu_mode();
+}
+

  /**
   * lazy_mmu_mode_resume() - Resume the lazy MMU mode. *

include/linux/pgtable.h +2 -9
@@ -341,15 +363,8 @@ static inline void lazy_mmu_mode_pause(void)
    */
   static inline void lazy_mmu_mode_resume(void)
   {
-    struct lazy_mmu_state *state = &current->lazy_mmu_state;
-
-    if (in_interrupt())
-        return;
-
-    VM_WARN_ON_ONCE(state->pause_count == 0);
-
-    if (--state->pause_count == 0 && state->enable_count > 0)
-        arch_enter_lazy_mmu_mode();
+    if (!in_interrupt())
+        lazy_mmu_mode_resume_task(current);
   }
   #else
   static inline void lazy_mmu_mode_enable(void) {}
#3 lore raw

+ Marek (the wrong address was in Cc)

Quoted 132 lines · Kevin Brodsky
Quoted 1 line · Jürgen Groß

Please disregard this patch. It isn't fixing the real problem.

That's what I would expect, see below.

Quoted 6 lines · Juergen Gross
Quoted 5 lines

The recent rework of mmu lazy mode has resulted in problems when running as a Xen PV guest. Enabling lazy mmu mode for the new context during context switch is done from the arch_end_context_switch() hook, but when calling this hook current hasn't been changed yet, so the lazy mmu mode state of the wrong task is modified.

Currently xen_end_context_switch() checks if next has lazy MMU mode enabled and if so calls arch_enter_lazy_mmu_mode(), i.e. enter_lazy(XEN_LAZY_MMU). This does *not* modify any task state, rather it writes to the xen_lazy_mode percpu variable.

I've thought about this from various angles when reworking lazy MMU, and the conclusion I made is that arch_{start,end}_context_switch() have no reason to change any task state. On arm64, for instance, we do nothing at all on context switching, since everything lazy MMU-related is tracked in task_struct and therefore already switched.

Xen is trickier because it tracks lazy MMU/CPU state in a percpu variable, so these hooks do need to do something about it. This is entirely Xen-internal though, and there's no reason to be calling generic functions like lazy_mmu_mode_pause() that modify task state.

The idea behind commit 291b3abed657 ("x86/xen: use lazy_mmu_state when context-switching") is that TIF_LAZY_MMU_UPDATES now duplicates lazy_mmu_state in task_struct and we can therefore replace the former with the latter. More specifically, the assumption is that TIF_LAZY_MMU_UPDATES is set if and only if the task has been scheduled out and __task_lazy_mmu_mode_active(task) is true.

Clearly there is something wrong with this assumption, but I still can't put my finger on it. For now I would suggest reverting this commit if that solves the issue Marek reported; the intention was not to introduce any functional change, but only a (minor) optimisation.

- Kevin

Quoted 92 lines
Quoted 92 lines

Additionally it is much cleaner to use lazy_mmu_mode_pause() and lazy_mmu_mode_resume() in the Xen context switch hooks, as it avoids conditionals in those hooks.

In order not having to add another hook to be called after switching current, modify lazy_mmu_mode_resume() to use a new sub-function which takes a task pointer as parameter. This new sub-function can then be used in the xen_end_context_switch() hook.

Fixes 291b3abed657 ("x86/xen: use lazy_mmu_state when

context-switching")

---
  arch/x86/xen/enlighten_pv.c |  7 ++----- include/linux/pgtable.h     | 33 ++++++++++++++++++++++++---------

  2 files changed, 26 insertions(+), 14 deletions(-)
arch/x86/xen/enlighten_pv.c +1 -2
index ed2d7a3756ce..67bb6bf6d240 100644
@@ -424,9 +424,7 @@ static void xen_start_context_switch(struct
 task_struct *prev)
   {
       BUG_ON(preemptible());
   -    if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) {
-        arch_leave_lazy_mmu_mode();
-    }
+    lazy_mmu_mode_pause();
       enter_lazy(XEN_LAZY_CPU);
   }

  @@ -436,8 +434,7 @@ static void xen_end_context_switch(struct task_struct *next)
        xen_mc_flush();
      leave_lazy(XEN_LAZY_CPU);
-    if (__task_lazy_mmu_mode_active(next))
-        arch_enter_lazy_mmu_mode();
+    lazy_mmu_mode_resume_task(next);
  }
    static unsigned long xen_store_tr(void)

include/linux/pgtable.h +0 -0
index cdd68ed3ae1a..83a099bf2038 100644
@@ -326,6 +326,28 @@ static inline void lazy_mmu_mode_pause(void)
           arch_leave_lazy_mmu_mode();
   }

  +/**
+ * lazy_mmu_mode_resume_task() - Resume the lazy MMU mode for a specific task.
+ *
+ * Like lazy_mmu_mode_resume() below, but with a task specified.
+ * Must be called only by lazy_mmu_mode_resume() or during context switch.
+ * Must never be called in interrupt context.
+ *
+ * Must match a call to lazy_mmu_mode_pause().
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ */

include/linux/pgtable.h +10 -0
+static inline void lazy_mmu_mode_resume_task(struct task_struct *task)
+{
+    struct lazy_mmu_state *state = &task->lazy_mmu_state;
+
+    VM_WARN_ON_ONCE(state->pause_count == 0);
+
+    if (--state->pause_count == 0 && state->enable_count > 0)
+        arch_enter_lazy_mmu_mode();
+}
+

  /**
   * lazy_mmu_mode_resume() - Resume the lazy MMU mode. *

include/linux/pgtable.h +2 -9
@@ -341,15 +363,8 @@ static inline void lazy_mmu_mode_pause(void)
    */
   static inline void lazy_mmu_mode_resume(void)
   {
-    struct lazy_mmu_state *state = &current->lazy_mmu_state;
-
-    if (in_interrupt())
-        return;
-
-    VM_WARN_ON_ONCE(state->pause_count == 0);
-
-    if (--state->pause_count == 0 && state->enable_count > 0)
-        arch_enter_lazy_mmu_mode();
+    if (!in_interrupt())
+        lazy_mmu_mode_resume_task(current);
   }
   #else
   static inline void lazy_mmu_mode_enable(void) {}