x86_32: use 8 IPI vectors for tlb flush (as x86_64)
PATCH 1/1 x86_32: use 8 IPI vectors for tlb flush (as x86_64)
Hi,
The following patch allows using 8 different IPI vectors on ia32 for tlb flushes, this reduces the contention on the tlbstate_lock. It also makes the 32 bits version closer to the 64 bits version.
Credits go to Andy Kleen who pointed that the 64 bits version could be ported in a comment below.
Regards,
Frederik
arch/x86/include/asm/hw_irq.h +0 -4
@@ -102,11 +102,7 @@ extern void smp_error_interrupt(struct pt_regs *);extern void smp_reschedule_interrupt (struct pt_regs *);extern void smp_call_function_interrupt (struct pt_regs *);extern void smp_call_function_single_interrupt (struct pt_regs *); -#ifdef CONFIG_X86_32 extern void smp_invalidate_interrupt (struct pt_regs *); -#else -extern asmlinkage void smp_invalidate_interrupt(struct pt_regs *); -#endif #endif extern void (*__initconst interrupt[NR_VECTORS -FIRST_EXTERNAL_VECTOR ])(void );
arch/x86/include/asm/irq_vectors.h +11 -9
@@ -49,18 +49,20 @@* some of the following vectors are 'rare', they are merged * into a single vector (CALL_FUNCTION_VECTOR) to save vector space. * TLB, reschedule and local APIC vectors are performance-critical. -* -* Vectors 0xf0-0xfa are free (reserved for future Linux use). */ #ifdef CONFIG_X86_32 -# define SPURIOUS_APIC_VECTOR 0xff -# define ERROR_APIC_VECTOR 0xfe -# define INVALIDATE_TLB_VECTOR 0xfd -# define RESCHEDULE_VECTOR 0xfc -# define CALL_FUNCTION_VECTOR 0xfb -# define CALL_FUNCTION_SINGLE_VECTOR 0xfa -# define THERMAL_APIC_VECTOR 0xf0 +/* Vectors 0xf8-0xf9 are free (reserved for future Linux use). */ +#define SPURIOUS_APIC_VECTOR 0xff +#define ERROR_APIC_VECTOR 0xfe +#define RESCHEDULE_VECTOR 0xfd +#define CALL_FUNCTION_VECTOR 0xfc +#define CALL_FUNCTION_SINGLE_VECTOR 0xfb +#define THERMAL_APIC_VECTOR 0xfa +#define INVALIDATE_TLB_VECTOR_END 0xf7 +#define INVALIDATE_TLB_VECTOR_START 0xf0/* f0-f7 used for TLB flush */ + +#define NUM_INVALIDATE_TLB_VECTORS 8#else
arch/x86/include/asm/mach-default/entry_arch.h +24 -1
@@ -11,10 +11,33 @@*/ #ifdef CONFIG_X86_SMP BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR) -BUILD_INTERRUPT(invalidate_interrupt,INVALIDATE_TLB_VECTOR) BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) +BUILD_APIC_INTERRUPT (invalidate_interrupt0, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+0) +BUILD_APIC_INTERRUPT (invalidate_interrupt1, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+1) +BUILD_APIC_INTERRUPT (invalidate_interrupt2, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+2) +BUILD_APIC_INTERRUPT (invalidate_interrupt3, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+3) +BUILD_APIC_INTERRUPT (invalidate_interrupt4, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+4) +BUILD_APIC_INTERRUPT (invalidate_interrupt5, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+5) +BUILD_APIC_INTERRUPT (invalidate_interrupt6, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+6) +BUILD_APIC_INTERRUPT (invalidate_interrupt7, + smp_invalidate_interrupt, + INVALIDATE_TLB_VECTOR_START+7)#endif /*
arch/x86/kernel/entry_32.S +13 -0
@@ -672,6 +672,19 @@ common_interrupt: ENDPROC(common_interrupt) CFI_ENDPROC +#define BUILD_APIC_INTERRUPT(name, fname, nr) \ +ENTRY(name) \ + RING0_INT_FRAME; \ + pushl $~(nr); \ + CFI_ADJUST_CFA_OFFSET 4; \ + SAVE_ALL; \ + TRACE_IRQS_OFF \ + movl %esp,%eax; \ + call fname; \ + jmp ret_from_intr; \ + CFI_ENDPROC; \ +ENDPROC(name) + #define BUILD_INTERRUPT(name, nr) \ ENTRY(name) \ RING0_INT_FRAME; \
arch/x86/kernel/irqinit_32.c +8 -1
@@ -150,7 +150,14 @@ void __init native_init_IRQ(void)alloc_intr_gate (RESCHEDULE_VECTOR , reschedule_interrupt);/* IPI for invalidation */ -alloc_intr_gate (INVALIDATE_TLB_VECTOR , invalidate_interrupt); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +0 , invalidate_interrupt0); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +1 , invalidate_interrupt1); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +2 , invalidate_interrupt2); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +3 , invalidate_interrupt3); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +4 , invalidate_interrupt4); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +5 , invalidate_interrupt5); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +6 , invalidate_interrupt6); +alloc_intr_gate (INVALIDATE_TLB_VECTOR_START +7 , invalidate_interrupt7);/* IPI for generic function call */ alloc_intr_gate (CALL_FUNCTION_VECTOR , call_function_interrupt);
arch/x86/kernel/tlb_32.c +49 -23
137 lines not on this page · raw message
Quoted 5 lines · Frederik Deweerdt
Hi,
The following patch allows using 8 different IPI vectors on ia32 for tlb flushes, this reduces the contention on the
tlbstate_lock. It also makes the 32 bits version closer to the 64 bits version.
This was done recently :-(
Are you aware of the x86 development tree? It's included in the -tip tree
and can be found at:
http://people.redhat.com/mingo/tip.git/README
Credit goes to Tejun Heo who unified the 32-bit and 64-bit TLB code (which also means 32-bit uses the 8 IPI vectors) - the code is now all in arch/x86/mm/tlb.c.
Ingo