Fast path context switch - microoptimize FPU reload

PATCH New Andi Kleen 1 participant 1 message Mar 9, 2003

lore

Share & tools

lore

Andi Kleen

author

2003-03-09 16:29 UTC #0 lore raw

PATCH 1/1 Fast path context switch - microoptimize FPU reload

Following some changes on x86-64.

When cpu_has_fxsr is defined to 1 like in many kernels unlazy_fpu can collapse to three instructions. For that inlining is a very good idea. Otherwise it's 10 instructions or so, which can be still inlined.

We don't need the lock prefix to test our local thread flags state. Unfortunately test_thread_flag currently always uses test_bit which has a LOCK on SMP, but that's unnecessary. LOCK is costly on P4, so it's a good idea to avoid it.

Work around this for now by testing directly. Better would be probably to define __set_bit for all architectures to not guarantee atomicity and then always use that for local thread_info accesses in linux/thread_info.h

-Andi

arch/i386/kernel/i387.c renamed from arch/i386/kernel/i387.c-FPU +0 ~~-18~~

diff -u linux-2.5.63-work/arch/i386/kernel/i387.c-FPU linux-2.5.63-work/arch/i386/kernel/i387.c
@@ -52,24 +52,6 @@
  * FPU lazy state save handling.
  */
 
-static inline void __save_init_fpu( struct task_struct *tsk )
-{
-	if ( cpu_has_fxsr ) {
-		asm volatile( "fxsave %0 ; fnclex"
-			      : "=m" (tsk->thread.i387.fxsave) );
-	} else {
-		asm volatile( "fnsave %0 ; fwait"
-			      : "=m" (tsk->thread.i387.fsave) );
-	}
-	clear_tsk_thread_flag(tsk, TIF_USEDFPU);
-}
-
-void save_init_fpu( struct task_struct *tsk )
-{
-	__save_init_fpu(tsk);
-	stts();
-}
-
 void kernel_fpu_begin(void)
 {
 	preempt_disable();

include/asm-i386/i387.h renamed from include/asm-i386/i387.h-FPU +22 -4

diff -u linux-2.5.63-work/include/asm-i386/i387.h-FPU linux-2.5.63-work/include/asm-i386/i387.h
@@ -21,23 +21,41 @@
 /*
  * FPU lazy state save handling...
  */
-extern void save_init_fpu( struct task_struct *tsk );
 extern void restore_fpu( struct task_struct *tsk );
 
 extern void kernel_fpu_begin(void);
 #define kernel_fpu_end() do { stts(); preempt_enable(); } while(0)
 
 
+static inline void __save_init_fpu( struct task_struct *tsk )
+{
+	if ( cpu_has_fxsr ) {
+		asm volatile( "fxsave %0 ; fnclex"
+			      : "=m" (tsk->thread.i387.fxsave) );
+	} else {
+		asm volatile( "fnsave %0 ; fwait"
+			      : "=m" (tsk->thread.i387.fsave) );
+	}
+	tsk->thread_info->flags &= ~TIF_USEDFPU;
+}
+
+static inline void save_init_fpu( struct task_struct *tsk )
+{
+	__save_init_fpu(tsk);
+	stts();
+}
+
+
 #define unlazy_fpu( tsk ) do { \
-	if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) \
+	if ((tsk)->thread_info->flags & _TIF_USEDFPU) \
 		save_init_fpu( tsk ); \
 } while (0)
 
 #define clear_fpu( tsk )					\
 do {								\
-	if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) {		\
+	if ((tsk)->thread_info->flags & _TIF_USEDFPU) {		\
 		asm volatile("fwait");				\
-		clear_tsk_thread_flag(tsk, TIF_USEDFPU);	\
+		(tsk)->thread_info->flags &= ~_TIF_USEDFPU;	\
 		stts();						\
 	}							\
 } while (0)