Skip to contentSkip to conversation

x86_32: use 8 IPI vectors for tlb flush (as x86_64)

PATCH In review Frederik Deweerdt 2 participants 2 messages Feb 2, 2009 – Feb 3, 2009 x86

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86_32: use 8 IPI vectors for tlb flush (as x86_64)

Hi,

The following patch allows using 8 different IPI vectors on ia32 for tlb flushes, this reduces the contention on the tlbstate_lock. It also makes the 32 bits version closer to the 64 bits version.
Credits go to Andy Kleen who pointed that the 64 bits version could be ported in a comment below.

Regards,
Frederik

arch/x86/include/asm/hw_irq.h +0 -4
index 8de644b..491c034 100644
@@ -102,11 +102,7 @@ extern void smp_error_interrupt(struct pt_regs *);
 extern void smp_reschedule_interrupt(struct pt_regs *);
 extern void smp_call_function_interrupt(struct pt_regs *);
 extern void smp_call_function_single_interrupt(struct pt_regs *);
-#ifdef CONFIG_X86_32
 extern void smp_invalidate_interrupt(struct pt_regs *);
-#else
-extern asmlinkage void smp_invalidate_interrupt(struct pt_regs *);
-#endif
 #endif
 
 extern void (*__initconst interrupt[NR_VECTORS-FIRST_EXTERNAL_VECTOR])(void);
arch/x86/include/asm/irq_vectors.h +11 -9
index f7ff650..42c7ba1 100644
@@ -49,18 +49,20 @@
  *  some of the following vectors are 'rare', they are merged
  *  into a single vector (CALL_FUNCTION_VECTOR) to save vector space.
  *  TLB, reschedule and local APIC vectors are performance-critical.
- *
- *  Vectors 0xf0-0xfa are free (reserved for future Linux use).
  */
 #ifdef CONFIG_X86_32
 
-# define SPURIOUS_APIC_VECTOR		0xff
-# define ERROR_APIC_VECTOR		0xfe
-# define INVALIDATE_TLB_VECTOR		0xfd
-# define RESCHEDULE_VECTOR		0xfc
-# define CALL_FUNCTION_VECTOR		0xfb
-# define CALL_FUNCTION_SINGLE_VECTOR	0xfa
-# define THERMAL_APIC_VECTOR		0xf0
+ /*  Vectors 0xf8-0xf9 are free (reserved for future Linux use). */
+#define SPURIOUS_APIC_VECTOR		0xff
+#define ERROR_APIC_VECTOR		0xfe
+#define RESCHEDULE_VECTOR		0xfd
+#define CALL_FUNCTION_VECTOR		0xfc
+#define CALL_FUNCTION_SINGLE_VECTOR	0xfb
+#define THERMAL_APIC_VECTOR		0xfa
+#define INVALIDATE_TLB_VECTOR_END	0xf7
+#define INVALIDATE_TLB_VECTOR_START	0xf0	/* f0-f7 used for TLB flush */
+
+#define NUM_INVALIDATE_TLB_VECTORS	8
 
 #else
 
arch/x86/include/asm/mach-default/entry_arch.h +24 -1
index 6b1add8..bb76518 100644
@@ -11,10 +11,33 @@
  */
 #ifdef CONFIG_X86_SMP
 BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR)
-BUILD_INTERRUPT(invalidate_interrupt,INVALIDATE_TLB_VECTOR)
 BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
 BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
 BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR)
+BUILD_APIC_INTERRUPT(invalidate_interrupt0,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+0)
+BUILD_APIC_INTERRUPT(invalidate_interrupt1,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+1)
+BUILD_APIC_INTERRUPT(invalidate_interrupt2,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+2)
+BUILD_APIC_INTERRUPT(invalidate_interrupt3,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+3)
+BUILD_APIC_INTERRUPT(invalidate_interrupt4,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+4)
+BUILD_APIC_INTERRUPT(invalidate_interrupt5,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+5)
+BUILD_APIC_INTERRUPT(invalidate_interrupt6,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+6)
+BUILD_APIC_INTERRUPT(invalidate_interrupt7,
+		     smp_invalidate_interrupt,
+		     INVALIDATE_TLB_VECTOR_START+7)
 #endif
 
 /*
arch/x86/kernel/entry_32.S +13 -0
index 4646902..c045afa 100644
@@ -672,6 +672,19 @@ common_interrupt:
 ENDPROC(common_interrupt)
 	CFI_ENDPROC
 
+#define BUILD_APIC_INTERRUPT(name, fname, nr)	\
+ENTRY(name)				\
+	RING0_INT_FRAME;		\
+	pushl $~(nr);			\
+	CFI_ADJUST_CFA_OFFSET 4;	\
+	SAVE_ALL;			\
+	TRACE_IRQS_OFF			\
+	movl %esp,%eax;			\
+	call fname;			\
+	jmp ret_from_intr;		\
+	CFI_ENDPROC;			\
+ENDPROC(name)
+
 #define BUILD_INTERRUPT(name, nr)	\
 ENTRY(name)				\
 	RING0_INT_FRAME;		\
arch/x86/kernel/irqinit_32.c +8 -1
index 1507ad4..209edd6 100644
@@ -150,7 +150,14 @@ void __init native_init_IRQ(void)
 	alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
 
 	/* IPI for invalidation */
-	alloc_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
+	alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
 
 	/* IPI for generic function call */
 	alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
arch/x86/kernel/tlb_32.c +49 -23

137 lines not on this page · raw message

#1 lore raw
Quoted 5 lines · Frederik Deweerdt

Hi,

The following patch allows using 8 different IPI vectors on ia32 for tlb flushes, this reduces the contention on the tlbstate_lock. It also makes the 32 bits version closer to the 64 bits version.

This was done recently :-(

Are you aware of the x86 development tree? It's included in the -tip tree
and can be found at:

http://people.redhat.com/mingo/tip.git/README

Credit goes to Tejun Heo who unified the 32-bit and 64-bit TLB code (which also means 32-bit uses the 8 IPI vectors) - the code is now all in arch/x86/mm/tlb.c.

Ingo