improve alternative instruction code and optimize get_cycles_sync
cover improve alternative instruction code and optimize get_cycles_sync
This series of patches extend the alternative instructions framework on i386 and x86_64 architectures to support two alternative instruction replacements. This code is used together with the introduction of the X86_FEATURE_SYNC_RDTSC flag on i386 to simplify and optimize the get_cycles_sync() function. The optimization changes this function to use RDTSCP instead of CPUID;RDTSC if this instruction is available. Don't use CPUID there is really important if the kernel runs as a KVM guest, because this instruction is intercepted and causes an expensive VMEXIT.
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG
PATCH 1/4 i386: extend alternative instructions framework
From: Joerg Roedel <[email protected]>
This patch extends the alternative instructions framework to support 2 alternative instructions.
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG
arch/i386/kernel/alternative.c +15 -6
@@ -153,14 +153,23 @@ extern u8 __smp_alt_begin[], __smp_alt_end[];void apply_alternatives (struct alt_instr *start,struct alt_instr *end) {struct alt_instr *a; -u8 *instr; +u8 *instr, *replacement; +u8 replacementlen;int diff;DPRINTK ("%s: alt table %p -> %p\n" , __FUNCTION__, start, end);for (a = start; a < end; a++) { -BUG_ON (a->replacementlen > a->instrlen); -if (!boot_cpu_has (a->cpuid)) +if (boot_cpu_has (a->cpuid)) { + replacement = a->replacement; + replacementlen = a->replacementlen; + }else if ((a->replacementlen2 >0 ) && + (boot_cpu_has (a->cpuid2))) { + replacement = a->replacement2; + replacementlen = a->replacementlen2; + }else continue ; + +BUG_ON (replacementlen > a->instrlen); instr = a->instr;#ifdef CONFIG_X86_64 /* vsyscall code is not mapped yet. resolve it manually. */
@@ -170,9 +179,9 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end) __FUNCTION__, a->instr, instr); }#endif -memcpy (instr, a->replacement, a->replacementlen); - diff = a->instrlen - a->replacementlen; -nop_out (instr + a->replacementlen, diff); +memcpy (instr, replacement, replacementlen); + diff = a->instrlen - replacementlen; +nop_out (instr + replacementlen, diff); } }
include/asm-i386/alternative.h +38 -2
@@ -10,11 +10,14 @@struct alt_instr {u8 *instr;/* original instruction */ u8 *replacement; +u8 *replacement2;u8 cpuid;/* cpuid bit set for replacement */ +u8 cpuid2;/* cpuid bit set for replacement2 */ u8 instrlen;/* length of original instruction */ u8 replacementlen;/* length of new instruction, <= instrlen */ -u8 pad; -}; +u8 replacementlen2; +u8 pad[3 ]; +} __attribute__ ((packed));extern void apply_alternatives (struct alt_instr *start,struct alt_instr *end);
@@ -36,6 +39,12 @@ static inline void alternatives_smp_switch(int smp) {}#endif /* +* use this macro(s) if you need more than one output parameter +* in alternative_io_* +*/ +#define ASM_OUTPUT2 (a, b) a, b + +/* * Alternative instructions for different CPU types or capabilities. * * This allows to use optimized instructions even on generic binary
@@ -53,9 +62,12 @@ static inline void alternatives_smp_switch(int smp) {}" .align 4\n" \" .long 661b\n" /* label */ \" .long 663f\n" /* new instruction */ \ +" .long 0x00\n" \" .byte %c0\n" /* feature bit */ \ +" .byte 0x00\n" \" .byte 662b-661b\n" /* sourcelen */ \" .byte 664f-663f\n" /* replacementlen */ \ +" .byte 0x00\n" \".previous\n" \".section .altinstr_replacement,\"ax\"\n" \"663:\n\t" newinstr"\n664:\n" /* replacement */ \
@@ -77,14 +89,38 @@ static inline void alternatives_smp_switch(int smp) {}" .align 4\n" \" .long 661b\n" /* label */ \" .long 663f\n" /* new instruction */ \ +" .long 0x00\n" \" .byte %c0\n" /* feature bit */ \ +" .byte 0x00\n" \" .byte 662b-661b\n" /* sourcelen */ \" .byte 664f-663f\n" /* replacementlen */ \ +" .byte 0x00\n" \".previous\n" \".section .altinstr_replacement,\"ax\"\n" \"663:\n\t" newinstr"\n664:\n" /* replacement */ \ ".previous" :: "i" (feature), ##input) +/* Like alternative_io, but supports 2 possible alternatives */ +#define alternative_io_two(oldinstr, newinstr, feat, newinstr2, feat2,\ + output, input...) \ + asmvolatile ("661:\n\t" oldinstr"\n662:\n" \ +".section .altinstructions,\"a\"\n" \ +" .align 4\n" \ +" .long 661b\n" /* label */ \ +" .long 663f\n" /* new instruction */ \ +" .long 665f\n" /* new instruction 2 */ \ +" .byte %c[f]\n" /* feature bit */ \ +" .byte %c[f2]\n" /* feature bit 2*/ \ +" .byte 662b-661b\n" /* sourcelen */ \ +" .byte 664f-663f\n" /* replacementlen */ \ +" .byte 666f-665f\n" /* replacementlen 2 */ \ +".previous\n" \ +".section .altinstr_replacement,\"ax\"\n" \ +"663:\n\t" newinstr"\n664:\n" /* replacement */ \ +"665:\n\t" newinstr2"\n666:\n" /* replacement2 */ \ + ".previous" : output : [f] "i" (feat), \ + [f2] "i" (feat2),##input) +/* * Alternative inline assembly for SMP. *
PATCH 2/4 x86_64: changes to x86_64 architecture for alternative instruction improvements
In reply to Joerg RoedelFrom: Joerg Roedel <[email protected]>
In this patch updates the x86_64 architecture to work with the changes to alternative instructions in i386
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG
arch/x86_64/lib/clear_page.S +3 -0
@@ -53,7 +53,10 @@ ENDPROC(clear_page) .align 8 .quad clear_page .quad 1b + .quad 0 .byte X86_FEATURE_REP_GOOD + .byte 0 .byte .Lclear_page_end - clear_page .byte 2b - 1b + .byte 0 .previous
arch/x86_64/lib/copy_page.S +3 -0
@@ -113,7 +113,10 @@ ENDPROC(copy_page) .align 8 .quad copy_page .quad 1b + .quad 0 .byte X86_FEATURE_REP_GOOD + .byte 0 .byte .Lcopy_page_end - copy_page .byte 2b - 1b + .byte 0 .previous
arch/x86_64/lib/copy_user.S +3 -0
@@ -27,9 +27,12 @@ .align 8 .quad 0b .quad 2b + .quad 0 .byte \feature /* when feature is set */ + .byte 0 .byte 5 .byte 5 + .byte 0 .previous .endm
arch/x86_64/lib/memcpy.S +3 -0
@@ -123,7 +123,10 @@ ENDPROC(__memcpy) .align 8 .quad memcpy .quad 1b + .quad 0 .byte X86_FEATURE_REP_GOOD + .byte 0 .byte .Lfinal - memcpy .byte 2b - 1b + .byte 0 .previous
arch/x86_64/lib/memset.S +3 -0
@@ -127,7 +127,10 @@ ENDPROC(__memset) .align 8 .quad memset .quad 1b + .quad 0 .byte X86_FEATURE_REP_GOOD + .byte 0 .byte .Lfinal - memset .byte 2b - 1b + .byte 0 .previous
include/asm-x86_64/alternative.h +40 -2
89 lines not on this page · raw message
PATCH 3/4 i386: add the X86_FEATURE_SYNC_RDTSC flag
In reply to Joerg RoedelFrom: Joerg Roedel <[email protected]>
This patch adds the X86_FEATURE_SYNC_RDTSC to the i386 architecture. This is very helpfull to simplify the get_cycles_sync() function and remove the #ifdefs from it.
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG
arch/i386/kernel/cpu/amd.c +2 -0
@@ -241,6 +241,8 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)if (cpuid_eax (0x80000000 ) >=0x80000006 ) num_cache_leaves =3 ; + +clear_bit (X86_FEATURE_SYNC_RDTSC , c->x86_capability); }static unsigned int __cpuinit amd_size_cache (struct cpuinfo_x86 * c,unsigned int size)
arch/i386/kernel/cpu/intel.c +4 -1
@@ -188,8 +188,11 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) }#endif -if (c->x86 ==15 ) +if (c->x86 ==15 ) {set_bit (X86_FEATURE_P4 , c->x86_capability); +set_bit (X86_FEATURE_SYNC_RDTSC , c->x86_capability); + }else +clear_bit (X86_FEATURE_SYNC_RDTSC , c->x86_capability);if (c->x86 ==6 )set_bit (X86_FEATURE_P3 , c->x86_capability); if ((c->x86 == 0xf && c->x86_model >= 0x03) ||
include/asm-i386/cpufeature.h +1 -0
@@ -75,6 +76,7 @@#define X86_FEATURE_ARCH_PERFMON (3*32+11)/* Intel Architectural PerfMon */ #define X86_FEATURE_PEBS (3*32+12)/* Precise-Event Based Sampling */ #define X86_FEATURE_BTS (3*32+13)/* Branch Trace Store */ +#define X86_FEATURE_SYNC_RDTSC (3*32+14)/* RDTSC is serializing */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0)/* Streaming SIMD Extensions-3 */
PATCH 4/4 optimize and simplify get_cycles_sync()
In reply to Joerg RoedelFrom: Joerg Roedel <[email protected]>
This patch simplifies the get_cycles_sync() function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG
include/asm-i386/cpufeature.h +1 -0
@@ -49,6 +49,7 @@#define X86_FEATURE_MP (1*32+19)/* MP Capable. */ #define X86_FEATURE_NX (1*32+20)/* Execute Disable */ #define X86_FEATURE_MMXEXT (1*32+22)/* AMD MMX extensions */ +#define X86_FEATURE_RDTSCP (1*32+27)/* RDTSCP */ #define X86_FEATURE_LM (1*32+29)/* Long Mode (x86-64) */ #define X86_FEATURE_3DNOWEXT (1*32+30)/* AMD 3DNow! extensions */ #define X86_FEATURE_3DNOW (1*32+31)/* 3DNow! */
include/asm-x86_64/tsc.h +7 -14
@@ -34,22 +34,15 @@ static inline cycles_t get_cycles(void)/* Like get_cycles, but make sure the CPU is synchronized. */ static __always_inline cycles_t get_cycles_sync(void) { -unsigned long long ret; -#ifdef X86_FEATURE_SYNC_RDTSC -unsigned eax; +unsigned int a, d; -/* -* Don't do an additional sync on CPUs where we know -* RDTSC is already synchronous: -*/ -alternative_io ("cpuid" ,ASM_NOP2 ,X86_FEATURE_SYNC_RDTSC , - "=a" (eax), "0" (1) : "ebx","ecx","edx","memory"); -#else -sync_core (); -#endif -rdtscll (ret); +alternative_io_two ("cpuid\nrdtsc" , +"rdtsc" ,X86_FEATURE_SYNC_RDTSC , +"rdtscp" ,X86_FEATURE_RDTSCP , +ASM_OUTPUT2 ("=a" (a),"=d" (d)), + "0" (1) : "ecx", "memory"); -return ret; +return ((unsigned long long )a) | (((unsigned long long )d)<<32 ); }extern void tsc_init (void );
Re: [PATCH 4/4] optimize and simplify get_cycles_sync()
Quoted 7 lines · On Wed, 28 Feb 2007 15:25:54 +0100 "Joerg Roedel" <[email protected]> wrote:
From: Joerg Roedel <[email protected]>
This patch simplifies the
get_cycles_sync()function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.
Problem:
http://test.kernel.org/functional/index.html
lots of builds fell over because their binutils versions don't understand rdtscp.
I don't know how recently rdtscp support was added to binutils, but this is likely to be a problem. Perhaps we can change this patch to embed the hex code for that opcode instead?
Re: [PATCH 4/4] optimize and simplify get_cycles_sync()
Quoted 13 lines · Andrew Morton
Quoted 7 lines · On Wed, 28 Feb 2007 15:25:54 +0100 "Joerg Roedel" <[email protected]> wrote:
From: Joerg Roedel <[email protected]>
This patch simplifies the
get_cycles_sync()function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.Problem:
Quoted 8 lines
http://test.kernel.org/functional/index.html
lots of builds fell over because their binutils versions don't understand rdtscp.
I don't know how recently rdtscp support was added to binutils, but this is likely to be a problem. Perhaps we can change this patch to embed the hex code for that opcode instead?
Ok, I will submit an updated version soon. Thanks for pointing this out.
Joerg
Signature · 3 lines
Joerg Roedel Operating System Research Center AMD Saxony LLC & Co. KG