Skip to contentSkip to conversation

improve alternative instruction code and optimize get_cycles_sync

PATCH In review Joerg Roedel 2 participants 7 messages Feb 28, 2007 – Mar 9, 2007

lore
Share & tools
lore
#0 lore raw

cover improve alternative instruction code and optimize get_cycles_sync

This series of patches extend the alternative instructions framework on i386 and x86_64 architectures to support two alternative instruction replacements. This code is used together with the introduction of the X86_FEATURE_SYNC_RDTSC flag on i386 to simplify and optimize the get_cycles_sync() function. The optimization changes this function to use RDTSCP instead of CPUID;RDTSC if this instruction is available. Don't use CPUID there is really important if the kernel runs as a KVM guest, because this instruction is intercepted and causes an expensive VMEXIT.

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG
#1 lore raw

PATCH 1/4 i386: extend alternative instructions framework

From: Joerg Roedel <[email protected]>

This patch extends the alternative instructions framework to support 2 alternative instructions.

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG
arch/i386/kernel/alternative.c +15 -6
index 9eca21b..59f1770 100644
@@ -153,14 +153,23 @@ extern u8 __smp_alt_begin[], __smp_alt_end[];
 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 {
 	struct alt_instr *a;
-	u8 *instr;
+	u8 *instr, *replacement;
+	u8 replacementlen;
 	int diff;
 
 	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
 	for (a = start; a < end; a++) {
-		BUG_ON(a->replacementlen > a->instrlen);
-		if (!boot_cpu_has(a->cpuid))
+		if (boot_cpu_has(a->cpuid)) {
+			replacement = a->replacement;
+			replacementlen = a->replacementlen;
+		} else if ((a->replacementlen2 > 0) &&
+			   (boot_cpu_has(a->cpuid2))) {
+			replacement = a->replacement2;
+			replacementlen = a->replacementlen2;
+		} else
 			continue;
+
+		BUG_ON(replacementlen > a->instrlen);
 		instr = a->instr;
 #ifdef CONFIG_X86_64
 		/* vsyscall code is not mapped yet. resolve it manually. */
@@ -170,9 +179,9 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 				__FUNCTION__, a->instr, instr);
 		}
 #endif
-		memcpy(instr, a->replacement, a->replacementlen);
-		diff = a->instrlen - a->replacementlen;
-		nop_out(instr + a->replacementlen, diff);
+		memcpy(instr, replacement, replacementlen);
+		diff = a->instrlen - replacementlen;
+		nop_out(instr + replacementlen, diff);
 	}
 }
 
include/asm-i386/alternative.h +38 -2
index b8fa955..4a77e93 100644
@@ -10,11 +10,14 @@
 struct alt_instr {
 	u8 *instr; 		/* original instruction */
 	u8 *replacement;
+	u8 *replacement2;
 	u8  cpuid;		/* cpuid bit set for replacement */
+	u8  cpuid2;		/* cpuid bit set for replacement2 */
 	u8  instrlen;		/* length of original instruction */
 	u8  replacementlen; 	/* length of new instruction, <= instrlen */
-	u8  pad;
-};
+	u8  replacementlen2;
+	u8  pad[3];
+}  __attribute__ ((packed));
 
 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
 
@@ -36,6 +39,12 @@ static inline void alternatives_smp_switch(int smp) {}
 #endif
 
 /*
+ * use this macro(s) if you need more than one output parameter
+ * in alternative_io_*
+ */
+#define ASM_OUTPUT2(a, b) a, b
+
+/*
  * Alternative instructions for different CPU types or capabilities.
  *
  * This allows to use optimized instructions even on generic binary
@@ -53,9 +62,12 @@ static inline void alternatives_smp_switch(int smp) {}
 		      "  .align 4\n"					\
 		      "  .long 661b\n"            /* label */		\
 		      "  .long 663f\n"		  /* new instruction */	\
+		      "  .long 0x00\n"					\
 		      "  .byte %c0\n"             /* feature bit */	\
+		      "  .byte 0x00\n"					\
 		      "  .byte 662b-661b\n"       /* sourcelen */	\
 		      "  .byte 664f-663f\n"       /* replacementlen */	\
+		      "  .byte 0x00\n"					\
 		      ".previous\n"					\
 		      ".section .altinstr_replacement,\"ax\"\n"		\
 		      "663:\n\t" newinstr "\n664:\n"   /* replacement */\
@@ -77,14 +89,38 @@ static inline void alternatives_smp_switch(int smp) {}
 		      "  .align 4\n"					\
 		      "  .long 661b\n"            /* label */		\
 		      "  .long 663f\n"		  /* new instruction */ \
+		      "  .long 0x00\n"					\
 		      "  .byte %c0\n"             /* feature bit */	\
+		      "  .byte 0x00\n"					\
 		      "  .byte 662b-661b\n"       /* sourcelen */	\
 		      "  .byte 664f-663f\n"       /* replacementlen */ 	\
+		      "  .byte 0x00\n"					\
 		      ".previous\n"					\
 		      ".section .altinstr_replacement,\"ax\"\n"		\
 		      "663:\n\t" newinstr "\n664:\n"   /* replacement */\
 		      ".previous" :: "i" (feature), ##input)
 
+/* Like alternative_io, but supports 2 possible alternatives */
+#define alternative_io_two(oldinstr, newinstr, feat, newinstr2, feat2,\
+		output, input...) \
+	asm volatile ("661:\n\t" oldinstr "\n662:\n"                    \
+		      ".section .altinstructions,\"a\"\n"               \
+		      "  .align 4\n"                                    \
+		      "  .long 661b\n"            /* label */           \
+		      "  .long 663f\n"            /* new instruction */ \
+		      "  .long 665f\n"            /* new instruction 2 */\
+		      "  .byte %c[f]\n"        /* feature bit */        \
+		      "  .byte %c[f2]\n"          /* feature bit 2*/    \
+		      "  .byte 662b-661b\n"       /* sourcelen */       \
+		      "  .byte 664f-663f\n"       /* replacementlen */  \
+		      "  .byte 666f-665f\n"       /* replacementlen 2 */\
+		      ".previous\n"                                     \
+		      ".section .altinstr_replacement,\"ax\"\n"         \
+		      "663:\n\t" newinstr "\n664:\n"  /* replacement */ \
+		      "665:\n\t" newinstr2 "\n666:\n" /* replacement2 */ \
+		      ".previous" : output : [f] "i" (feat), \
+		      [f2] "i" (feat2),##input)
+
 /*
  * Alternative inline assembly for SMP.
  *
#2 lore raw

PATCH 2/4 x86_64: changes to x86_64 architecture for alternative instruction improvements

In reply to Joerg Roedel

From: Joerg Roedel <[email protected]>

In this patch updates the x86_64 architecture to work with the changes to alternative instructions in i386

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG
arch/x86_64/lib/clear_page.S +3 -0
index 9a10a78..ab525ee 100644
@@ -53,7 +53,10 @@ ENDPROC(clear_page)
 	.align 8
 	.quad clear_page
 	.quad 1b
+	.quad 0
 	.byte X86_FEATURE_REP_GOOD
+	.byte 0
 	.byte .Lclear_page_end - clear_page
 	.byte 2b - 1b
+	.byte 0
 	.previous
arch/x86_64/lib/copy_page.S +3 -0
index 727a5d4..b4d0329 100644
@@ -113,7 +113,10 @@ ENDPROC(copy_page)
 	.align 8
 	.quad copy_page
 	.quad 1b
+	.quad 0
 	.byte X86_FEATURE_REP_GOOD
+	.byte 0
 	.byte .Lcopy_page_end - copy_page
 	.byte 2b - 1b
+	.byte 0
 	.previous
arch/x86_64/lib/copy_user.S +3 -0
index 70bebd3..d505df3 100644
@@ -27,9 +27,12 @@
 	.align 8
 	.quad  0b
 	.quad  2b
+	.quad  0
 	.byte  \feature		     /* when feature is set */
+	.byte  0
 	.byte  5
 	.byte  5
+	.byte  0
 	.previous
 	.endm
 
arch/x86_64/lib/memcpy.S +3 -0
index 0ea0ddc..b1e1686 100644
@@ -123,7 +123,10 @@ ENDPROC(__memcpy)
 	.align 8
 	.quad memcpy
 	.quad 1b
+	.quad 0
 	.byte X86_FEATURE_REP_GOOD
+	.byte 0
 	.byte .Lfinal - memcpy
 	.byte 2b - 1b
+	.byte 0
 	.previous
arch/x86_64/lib/memset.S +3 -0
index 2c59481..566e179 100644
@@ -127,7 +127,10 @@ ENDPROC(__memset)
 	.align 8
 	.quad memset
 	.quad 1b
+	.quad 0
 	.byte X86_FEATURE_REP_GOOD
+	.byte 0
 	.byte .Lfinal - memset
 	.byte 2b - 1b
+	.byte 0
 	.previous
include/asm-x86_64/alternative.h +40 -2

89 lines not on this page · raw message

#3 lore raw

PATCH 3/4 i386: add the X86_FEATURE_SYNC_RDTSC flag

In reply to Joerg Roedel

From: Joerg Roedel <[email protected]>

This patch adds the X86_FEATURE_SYNC_RDTSC to the i386 architecture. This is very helpfull to simplify the get_cycles_sync() function and remove the #ifdefs from it.

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG
arch/i386/kernel/cpu/amd.c +2 -0
index 41cfea5..11f5730 100644
@@ -241,6 +241,8 @@ static void __cpuinit init_amd(struct cpuinfo_x86 *c)
 
 	if (cpuid_eax(0x80000000) >= 0x80000006)
 		num_cache_leaves = 3;
+
+	clear_bit(X86_FEATURE_SYNC_RDTSC, c->x86_capability);
 }
 
 static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 * c, unsigned int size)
arch/i386/kernel/cpu/intel.c +4 -1
index 56fe265..403a495 100644
@@ -188,8 +188,11 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
 	}
 #endif
 
-	if (c->x86 == 15)
+	if (c->x86 == 15) {
 		set_bit(X86_FEATURE_P4, c->x86_capability);
+		set_bit(X86_FEATURE_SYNC_RDTSC, c->x86_capability);
+	} else
+		clear_bit(X86_FEATURE_SYNC_RDTSC, c->x86_capability);
 	if (c->x86 == 6) 
 		set_bit(X86_FEATURE_P3, c->x86_capability);
 	if ((c->x86 == 0xf && c->x86_model >= 0x03) ||
include/asm-i386/cpufeature.h +1 -0
index 3f92b94..a9f1f01 100644
@@ -75,6 +76,7 @@
 #define X86_FEATURE_ARCH_PERFMON (3*32+11) /* Intel Architectural PerfMon */
 #define X86_FEATURE_PEBS	(3*32+12)  /* Precise-Event Based Sampling */
 #define X86_FEATURE_BTS		(3*32+13)  /* Branch Trace Store */
+#define X86_FEATURE_SYNC_RDTSC  (3*32+14)  /* RDTSC is serializing */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3	(4*32+ 0) /* Streaming SIMD Extensions-3 */
#4 lore raw

PATCH 4/4 optimize and simplify get_cycles_sync()

In reply to Joerg Roedel

From: Joerg Roedel <[email protected]>

This patch simplifies the get_cycles_sync() function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG
include/asm-i386/cpufeature.h +1 -0
index 3f92b94..a9f1f01 100644
@@ -49,6 +49,7 @@
 #define X86_FEATURE_MP		(1*32+19) /* MP Capable. */
 #define X86_FEATURE_NX		(1*32+20) /* Execute Disable */
 #define X86_FEATURE_MMXEXT	(1*32+22) /* AMD MMX extensions */
+#define X86_FEATURE_RDTSCP      (1*32+27) /* RDTSCP */
 #define X86_FEATURE_LM		(1*32+29) /* Long Mode (x86-64) */
 #define X86_FEATURE_3DNOWEXT	(1*32+30) /* AMD 3DNow! extensions */
 #define X86_FEATURE_3DNOW	(1*32+31) /* 3DNow! */
include/asm-x86_64/tsc.h +7 -14
index 9a0a368..05df3f6 100644
@@ -34,22 +34,15 @@ static inline cycles_t get_cycles(void)
 /* Like get_cycles, but make sure the CPU is synchronized. */
 static __always_inline cycles_t get_cycles_sync(void)
 {
-	unsigned long long ret;
-#ifdef X86_FEATURE_SYNC_RDTSC
-	unsigned eax;
+	unsigned int a, d;
 
-	/*
-	 * Don't do an additional sync on CPUs where we know
-	 * RDTSC is already synchronous:
-	 */
-	alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
-			  "=a" (eax), "0" (1) : "ebx","ecx","edx","memory");
-#else
-	sync_core();
-#endif
-	rdtscll(ret);
+	alternative_io_two("cpuid\nrdtsc",
+			   "rdtsc", X86_FEATURE_SYNC_RDTSC,
+			   "rdtscp", X86_FEATURE_RDTSCP,
+			   ASM_OUTPUT2("=a" (a), "=d" (d)),
+			   "0" (1) : "ecx", "memory");
 
-	return ret;
+	return ((unsigned long long)a) | (((unsigned long long)d)<<32);
 }
 
 extern void tsc_init(void);
#5 lore raw

Re: [PATCH 4/4] optimize and simplify get_cycles_sync()

Quoted 7 lines · On Wed, 28 Feb 2007 15:25:54 +0100 "Joerg Roedel" <[email protected]> wrote:

From: Joerg Roedel <[email protected]>

This patch simplifies the get_cycles_sync() function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.

Problem:

http://test.kernel.org/functional/index.html

lots of builds fell over because their binutils versions don't understand rdtscp.

I don't know how recently rdtscp support was added to binutils, but this is likely to be a problem. Perhaps we can change this patch to embed the hex code for that opcode instead?

#6 lore raw

Re: [PATCH 4/4] optimize and simplify get_cycles_sync()

Quoted 13 lines · Andrew Morton
Quoted 7 lines · On Wed, 28 Feb 2007 15:25:54 +0100 "Joerg Roedel" <[email protected]> wrote:

From: Joerg Roedel <[email protected]>

This patch simplifies the get_cycles_sync() function by removing the #ifdefs from it. Further it introduces an optimization for AMD processors. There the RDTSCP instruction is used instead of CPUID;RDTSC which is helpfull if the kernel runs as a KVM guest. Running as a guest makes CPUID very expensive because it causes an intercept of the guest.

Problem:

Quoted 8 lines

http://test.kernel.org/functional/index.html

lots of builds fell over because their binutils versions don't understand rdtscp.

I don't know how recently rdtscp support was added to binutils, but this is likely to be a problem. Perhaps we can change this patch to embed the hex code for that opcode instead?

Ok, I will submit an updated version soon. Thanks for pointing this out.

Joerg

Signature · 3 lines
Joerg Roedel
Operating System Research Center
AMD Saxony LLC & Co. KG