Skip to contentSkip to conversation

x86/fpu: two more FPU cleanups

PATCH In review [email protected] 4 participants 9 messages Oct 14, 2016 – Feb 10, 2017 x86

lore
Share & tools
lore
#0 lore raw

cover x86/fpu: two more FPU cleanups

Two more small cleanups to the x86 FPU code, before I re-do the functional FPU context switching changes I have been working on.

Neither of these should result in any noticeable differences, besides for slightly fewer lines of code and updated comments :)

#1 lore raw

PATCH 1/2 x86/fpu: remove cpu argument to __cpu_invalidate_fpregs_state

From: Rik van Riel <[email protected]>

The __{fpu,cpu}_invalidate_fpregs_state functions can only be used to invalidate a resource they control. Document that, and change the API a little bit to reflect that.

Go back to open coding the fpu_fpregs_owner_ctx write in the CPU hotplug code, which should be the exception, and move __kernel_fpu_begin to this API.

This patch has no functional changes to the current code.

---
 arch/x86/include/asm/fpu/internal.h | 13 +++++++------
 arch/x86/kernel/fpu/core.c          |  2 +-
 arch/x86/kernel/smpboot.c           |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)
arch/x86/include/asm/fpu/internal.h +7 -6
index ef52935f8a17..a75324675311 100644
@@ -488,15 +488,16 @@ DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
  *
  * Any code that clobbers the FPU registers or updates the in-memory
  * FPU state for a task MUST let the rest of the kernel know that the
- * FPU registers are no longer valid for this task. Calling either of
- * these two invalidate functions is enough, use whichever is convenient.
+ * FPU registers are no longer valid for this task.
  *
- * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
- * on this CPU.
+ * Either one of these invalidation functions is enough. Invalidate
+ * a resource you control: CPU if using the CPU for something else
+ * (with preemption disabled), FPU for the current task, or a task that
+ * is prevented from running by the current task.
  */
-static inline void __cpu_invalidate_fpregs_state(unsigned int cpu)
+static inline void __cpu_invalidate_fpregs_state(void)
 {
-	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
+	__this_cpu_write(fpu_fpregs_owner_ctx, NULL);
 }
 
 static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
arch/x86/kernel/fpu/core.c +1 -1
index 25a45ddfdbcf..30f11ab6c07e 100644
@@ -106,7 +106,7 @@ void __kernel_fpu_begin(void)
 		 */
 		copy_fpregs_to_fpstate(fpu);
 	} else {
-		this_cpu_write(fpu_fpregs_owner_ctx, NULL);
+		__cpu_invalidate_fpregs_state();
 	}
 }
 EXPORT_SYMBOL(__kernel_fpu_begin);
arch/x86/kernel/smpboot.c +1 -1
index ca4c4ca2f6af..5cb801acc2e5 100644
@@ -1111,7 +1111,7 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 		return err;
 
 	/* the FPU context is blank, nobody can own it */
-	__cpu_invalidate_fpregs_state(cpu);
+	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
 
 	common_cpu_up(cpu, tidle);
 
Signature · 1 line
2.7.4
#4 lore raw

PATCH 2/2 x86/fpu: split old & new fpu handling into separate functions

In reply to [email protected]

From: Rik van Riel <[email protected]>

By moving all of the new fpu state handling into switch_fpu_finish, the code can be simplified some more. This does get rid of the prefetch, but given the size of the fpu register state on modern CPUs, and the amount of work done by __switch_to in-between both functions, the value of a single cache line prefetch seems somewhat dubious anyway.

---
 arch/x86/include/asm/fpu/internal.h | 48 ++++++++++++-------------------------
 arch/x86/kernel/process_32.c        |  5 ++--
 arch/x86/kernel/process_64.c        |  5 ++--
 3 files changed, 19 insertions(+), 39 deletions(-)
arch/x86/include/asm/fpu/internal.h +15 -33
index a75324675311..621ba3bfa2a7 100644
@@ -552,27 +552,15 @@ static inline int fpregs_active(void)
  *
  * This is a two-stage process:
  *
- *  - switch_fpu_prepare() saves the old state and
- *    sets the new state of the CR0.TS bit. This is
- *    done within the context of the old process.
+ *  - switch_fpu_prepare() saves the old state.
+ *    This is done within the context of the old process.
  *
  *  - switch_fpu_finish() restores the new state as
  *    necessary.
  */
-typedef struct { int preload; } fpu_switch_t;
-
-static inline fpu_switch_t
-switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
+static inline void
+switch_fpu_prepare(struct fpu *old_fpu, int cpu)
 {
-	fpu_switch_t fpu;
-
-	/*
-	 * If the task has used the math, pre-load the FPU on xsave processors
-	 * or if the past 5 consecutive context-switches used math.
-	 */
-	fpu.preload = static_cpu_has(X86_FEATURE_FPU) &&
-		      new_fpu->fpstate_active;
-
 	if (old_fpu->fpregs_active) {
 		if (!copy_fpregs_to_fpstate(old_fpu))
 			old_fpu->last_cpu = -1;
@@ -584,16 +572,6 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
 		trace_x86_fpu_regs_deactivated(old_fpu);
 	} else
 		old_fpu->last_cpu = -1;
-
-	if (fpu.preload) {
-		if (fpregs_state_valid(new_fpu, cpu))
-			fpu.preload = 0;
-		else
-			prefetch(&new_fpu->state);
-		fpregs_activate(new_fpu);
-	}
-
-	return fpu;
 }
 
 /*
@@ -601,15 +579,19 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
  */
 
 /*
- * By the time this gets called, we've already cleared CR0.TS and
- * given the process the FPU if we are going to preload the FPU
- * state - all we need to do is to conditionally restore the register
- * state itself.
+ * Set up the userspace FPU context for the new task, if the task
+ * has used the FPU.
  */
-static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch)
+static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu)
 {
-	if (fpu_switch.preload)
-		copy_kernel_to_fpregs(&new_fpu->state);
+	bool preload = static_cpu_has(X86_FEATURE_FPU) &&
+		       new_fpu->fpstate_active;
+
+	if (preload) {
+		if (!fpregs_state_valid(new_fpu, cpu))
+			copy_kernel_to_fpregs(&new_fpu->state);
+		fpregs_activate(new_fpu);
+	}
 }
 
 /*
arch/x86/kernel/process_32.c +2 -3
index bd7be8efdc4c..7dc8c9c3d801 100644
@@ -232,11 +232,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	struct fpu *next_fpu = &next->fpu;
 	int cpu = smp_processor_id();
 	struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
-	fpu_switch_t fpu_switch;
 
 	/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-	fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
+	switch_fpu_prepare(prev_fpu, cpu);
 
 	/*
 	 * Save away %gs. No need to save %fs, as it was saved on the
@@ -295,7 +294,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	if (prev->gs | next->gs)
 		lazy_load_gs(next->gs);
 
-	switch_fpu_finish(next_fpu, fpu_switch);
+	switch_fpu_finish(next_fpu, cpu);
 
 	this_cpu_write(current_task, next_p);
 
arch/x86/kernel/process_64.c +2 -3
index ee944bd2310d..705669efb762 100644
@@ -264,9 +264,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	int cpu = smp_processor_id();
 	struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
 	unsigned prev_fsindex, prev_gsindex;
-	fpu_switch_t fpu_switch;
 
-	fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
+	switch_fpu_prepare(prev_fpu, cpu);
 
 	/* We must save %fs and %gs before load_TLS() because
 	 * %fs and %gs may be cleared by load_TLS().
@@ -416,7 +415,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 		prev->gsbase = 0;
 	prev->gsindex = prev_gsindex;
 
-	switch_fpu_finish(next_fpu, fpu_switch);
+	switch_fpu_finish(next_fpu, cpu);
 
 	/*
 	 * Switch the PDA and FPU contexts.
Signature · 1 line
2.7.4
maintainer Acked-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] x86/fpu: remove cpu argument to __cpu_invalidate_fpregs_state

In reply to [email protected]
Quoted 6 lines · On 10/14/2016 05:15 AM, [email protected] wrote:

From: Rik van Riel <[email protected]>

The __{fpu,cpu}_invalidate_fpregs_state functions can only be used to invalidate a resource they control. Document that, and change the API a little bit to reflect that.

Quoted 7 lines

Go back to open coding the fpu_fpregs_owner_ctx write in the CPU hotplug code, which should be the exception, and move __kernel_fpu_begin to this API.

This patch has no functional changes to the current code.

Looks totally sane to me.

maintainer Acked-by on v1 2/2
#5 lore raw

Re: [PATCH 2/2] x86/fpu: split old & new fpu handling into separate functions

In reply to [email protected]
Quoted 8 lines · On 10/14/2016 05:15 AM, [email protected] wrote:

From: Rik van Riel <[email protected]>

By moving all of the new fpu state handling into switch_fpu_finish, the code can be simplified some more. This does get rid of the prefetch, but given the size of the fpu register state on modern CPUs, and the amount of work done by __switch_to in-between both functions, the value of a single cache line prefetch seems somewhat dubious anyway.

...

arch/x86/include/asm/fpu/internal.h switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) · line 575

Quoted 11 lines
arch/x86/include/asm/fpu/internal.h +0 -10
-
-	if (fpu.preload) {
-		if (fpregs_state_valid(new_fpu, cpu))
-			fpu.preload = 0;
-		else
-			prefetch(&new_fpu->state);
-		fpregs_activate(new_fpu);
-	}
-
-	return fpu;
 }

Yeah, that prefetch is highly dubious. XRSTOR might not even be _reading_ that cacheline if the state isn't present (xstate->xfeatures bit is 0). If we had to pick *a* cacheline to prefetch for XRSTOR, it would be the XSAVE header, *not* the FPU state.

I actually did some attempts to optimize the PKRU handling by touching and prefetching the state before calling XRSTOR. It actually made things overall _worse_ when I touched it before the XRSTOR.

It would be ideal to have some data on whether this actually _does_ anything, but I can't imagine it being a real delta in either direction.

#3 lore raw

[tip:x86/fpu] x86/fpu: Remove 'cpu' argument from __cpu_invalidate_fpregs_state()

In reply to [email protected]
x86/fpu: Remove 'cpu' argument from __cpu_invalidate_fpregs_state()
Commit-ID:  317b622cb2fda1812d8646e211cdb23dce2564d0
Gitweb:     http://git.kernel.org/tip/317b622cb2fda1812d8646e211cdb23dce2564d0
Author:     Rik van Riel <[email protected]>
AuthorDate: Fri, 14 Oct 2016 08:15:30 -0400
Committer:  Ingo Molnar <[email protected]>
CommitDate: Sun, 16 Oct 2016 11:38:31 +0200

x86/fpu: Remove 'cpu' argument from __cpu_invalidate_fpregs_state()

The __{fpu,cpu}_invalidate_fpregs_state() functions can only be used to invalidate a resource they control. Document that, and change the API a little bit to reflect that.

Go back to open coding the fpu_fpregs_owner_ctx write in the CPU hotplug code, which should be the exception, and move __kernel_fpu_begin() to this API.

This patch has no functional changes to the current code.

Cc Andy Lutomirski <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Dave Hansen <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc Fenghua Yu <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Josh Poimboeuf <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Oleg Nesterov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Quentin Casasnovas <[email protected]>
Cc Thomas Gleixner <[email protected]>
---
 arch/x86/include/asm/fpu/internal.h | 13 +++++++------
 arch/x86/kernel/fpu/core.c          |  2 +-
 arch/x86/kernel/smpboot.c           |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)
arch/x86/include/asm/fpu/internal.h +7 -6
index 1dcb29e..590f274 100644
@@ -488,15 +488,16 @@ DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
  *
  * Any code that clobbers the FPU registers or updates the in-memory
  * FPU state for a task MUST let the rest of the kernel know that the
- * FPU registers are no longer valid for this task. Calling either of
- * these two invalidate functions is enough, use whichever is convenient.
+ * FPU registers are no longer valid for this task.
  *
- * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
- * on this CPU.
+ * Either one of these invalidation functions is enough. Invalidate
+ * a resource you control: CPU if using the CPU for something else
+ * (with preemption disabled), FPU for the current task, or a task that
+ * is prevented from running by the current task.
  */
-static inline void __cpu_invalidate_fpregs_state(unsigned int cpu)
+static inline void __cpu_invalidate_fpregs_state(void)
 {
-	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
+	__this_cpu_write(fpu_fpregs_owner_ctx, NULL);
 }
 
 static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
arch/x86/kernel/fpu/core.c +1 -1
index 25a45dd..30f11ab 100644
@@ -106,7 +106,7 @@ void __kernel_fpu_begin(void)
 		 */
 		copy_fpregs_to_fpstate(fpu);
 	} else {
-		this_cpu_write(fpu_fpregs_owner_ctx, NULL);
+		__cpu_invalidate_fpregs_state();
 	}
 }
 EXPORT_SYMBOL(__kernel_fpu_begin);
arch/x86/kernel/smpboot.c +1 -1
index ca4c4ca..5cb801a 100644
@@ -1111,7 +1111,7 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 		return err;
 
 	/* the FPU context is blank, nobody can own it */
-	__cpu_invalidate_fpregs_state(cpu);
+	per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
 
 	common_cpu_up(cpu, tidle);
 
#6 lore raw

[tip:x86/fpu] x86/fpu: Split old_fpu & new_fpu handling into separate functions

In reply to [email protected]
x86/fpu: Split old_fpu & new_fpu handling into separate functions
Commit-ID:  c474e50711aa79b7bd0ea30b44744baca5650375
Gitweb:     http://git.kernel.org/tip/c474e50711aa79b7bd0ea30b44744baca5650375
Author:     Rik van Riel <[email protected]>
AuthorDate: Fri, 14 Oct 2016 08:15:31 -0400
Committer:  Ingo Molnar <[email protected]>
CommitDate: Sun, 16 Oct 2016 11:38:41 +0200

x86/fpu: Split old_fpu & new_fpu handling into separate functions

By moving all of the new_fpu state handling into switch_fpu_finish(), the code can be simplified some more.

This gets rid of the prefetch, but given the size of the FPU register state on modern CPUs, and the amount of work done by __switch_to() inbetween both functions, the value of a single cache line prefetch seems somewhat dubious anyway.

Cc Andy Lutomirski <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Dave Hansen <[email protected]>
Cc Denys Vlasenko <[email protected]>
Cc Fenghua Yu <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Josh Poimboeuf <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Oleg Nesterov <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Quentin Casasnovas <[email protected]>
Cc Thomas Gleixner <[email protected]>
---
 arch/x86/include/asm/fpu/internal.h | 48 ++++++++++++-------------------------
 arch/x86/kernel/process_32.c        |  5 ++--
 arch/x86/kernel/process_64.c        |  5 ++--
 3 files changed, 19 insertions(+), 39 deletions(-)
arch/x86/include/asm/fpu/internal.h +15 -33
index 590f274..d4a6849 100644
@@ -552,27 +552,15 @@ static inline int fpregs_active(void)
  *
  * This is a two-stage process:
  *
- *  - switch_fpu_prepare() saves the old state and
- *    sets the new state of the CR0.TS bit. This is
- *    done within the context of the old process.
+ *  - switch_fpu_prepare() saves the old state.
+ *    This is done within the context of the old process.
  *
  *  - switch_fpu_finish() restores the new state as
  *    necessary.
  */
-typedef struct { int preload; } fpu_switch_t;
-
-static inline fpu_switch_t
-switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
+static inline void
+switch_fpu_prepare(struct fpu *old_fpu, int cpu)
 {
-	fpu_switch_t fpu;
-
-	/*
-	 * If the task has used the math, pre-load the FPU on xsave processors
-	 * or if the past 5 consecutive context-switches used math.
-	 */
-	fpu.preload = static_cpu_has(X86_FEATURE_FPU) &&
-		      new_fpu->fpstate_active;
-
 	if (old_fpu->fpregs_active) {
 		if (!copy_fpregs_to_fpstate(old_fpu))
 			old_fpu->last_cpu = -1;
@@ -584,16 +572,6 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
 		trace_x86_fpu_regs_deactivated(old_fpu);
 	} else
 		old_fpu->last_cpu = -1;
-
-	if (fpu.preload) {
-		if (fpregs_state_valid(new_fpu, cpu))
-			fpu.preload = 0;
-		else
-			prefetch(&new_fpu->state);
-		fpregs_activate(new_fpu);
-	}
-
-	return fpu;
 }
 
 /*
@@ -601,15 +579,19 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
  */
 
 /*
- * By the time this gets called, we've already cleared CR0.TS and
- * given the process the FPU if we are going to preload the FPU
- * state - all we need to do is to conditionally restore the register
- * state itself.
+ * Set up the userspace FPU context for the new task, if the task
+ * has used the FPU.
  */
-static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch)
+static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu)
 {
-	if (fpu_switch.preload)
-		copy_kernel_to_fpregs(&new_fpu->state);
+	bool preload = static_cpu_has(X86_FEATURE_FPU) &&
+		       new_fpu->fpstate_active;
+
+	if (preload) {
+		if (!fpregs_state_valid(new_fpu, cpu))
+			copy_kernel_to_fpregs(&new_fpu->state);
+		fpregs_activate(new_fpu);
+	}
 }
 
 /*
arch/x86/kernel/process_32.c +2 -3
index bd7be8e..7dc8c9c 100644
@@ -232,11 +232,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	struct fpu *next_fpu = &next->fpu;
 	int cpu = smp_processor_id();
 	struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
-	fpu_switch_t fpu_switch;
 
 	/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-	fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
+	switch_fpu_prepare(prev_fpu, cpu);
 
 	/*
 	 * Save away %gs. No need to save %fs, as it was saved on the
@@ -295,7 +294,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	if (prev->gs | next->gs)
 		lazy_load_gs(next->gs);
 
-	switch_fpu_finish(next_fpu, fpu_switch);
+	switch_fpu_finish(next_fpu, cpu);
 
 	this_cpu_write(current_task, next_p);
 
arch/x86/kernel/process_64.c +2 -3
index ee944bd..705669e 100644
@@ -264,9 +264,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	int cpu = smp_processor_id();
 	struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
 	unsigned prev_fsindex, prev_gsindex;
-	fpu_switch_t fpu_switch;
 
-	fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
+	switch_fpu_prepare(prev_fpu, cpu);
 
 	/* We must save %fs and %gs before load_TLS() because
 	 * %fs and %gs may be cleared by load_TLS().
@@ -416,7 +415,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 		prev->gsbase = 0;
 	prev->gsindex = prev_gsindex;
 
-	switch_fpu_finish(next_fpu, fpu_switch);
+	switch_fpu_finish(next_fpu, cpu);
 
 	/*
 	 * Switch the PDA and FPU contexts.
#7 lore raw

Re: you have a new message from... c92Y

bless may i know u

Quoted 14 lines · tip-bot for Rik van Riel

You can do whatever you want to me izundunaoyi

Tell me where you want my cum

and don't let it be a disappointing location

[image]
[image]

To OPT_OUT,Please Click Here....

#8 lore raw

Re: you have a new message from... c92Y

sori am ntt around since wia re u nw

Quoted 18 lines · Izuchukwu Francis

bless may i know u

Quoted 14 lines · tip-bot for Rik van Riel

You can do whatever you want to me izundunaoyi

Tell me where you want my cum

and don't let it be a disappointing location

[image]
[image]

To OPT_OUT,Please Click Here....