AMD F15h cache aliasing fixes

GIT PULL New Borislav Petkov 1 participant 5 messages Nov 4, 2011

Subsystems (5)

Stable branch Documentation x86 exec and binfmt +1 more subsystems

lore

Share & tools

lore

Borislav Petkov

author

2011-11-04 11:11 UTC #0 lore raw

Hi Greg,

as previously discussed, here are the 4 patches needed for addressing the issue in $Subject. I'm sending them as mails but I have them also in a git-pullable form in case you'd like to verify them:

git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp.git l1i-3.0.x

Here are the patches on the branch above along with SHA1s (from the bottom up):

bf27c8742db1e2c01bff6e8e2cce3b14d5d6cca5 x86-32, amd: Move va_align definition to unbreak 32-bit build 57d8c29f231049d7b10a497c8afa42dba4672a70 x86, amd: Move BSP code to cpu_dev helper 28b1d99bcde97b468585bdf6c94fa51931d2a1aa x86: Add a BSP cpu_dev helper 5c303905f98ab4e378488c919cc60483d59002ff x86, amd: Avoid cache aliasing penalties on AMD family 15h

Please apply,
thanks.

Signature · 8 lines

Regards/Gruss,
Boris.

Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
GM: Alberto Bozzo
Reg: Dornach, Landkreis Muenchen
HRB Nr. 43632 WEEE Registernr: 129 19551

Borislav Petkov

author

2011-11-04 11:15 UTC #1 lore raw

PATCH 1/4 x86, amd: Avoid cache aliasing penalties on AMD family 15h

From: Borislav Petkov <[email protected]>

Upstream commit: dfb09f9b7ab03fd367740e541a5caf830ed56726

This patch provides performance tuning for the "Bulldozer" CPU. With its shared instruction cache there is a chance of generating an excessive number of cache cross-invalidates when running specific workloads on the cores of a compute module.

This excessive amount of cross-invalidations can be observed if cache lines backed by shared physical memory alias in bits [14:12] of their virtual addresses, as those bits are used for the index generation.

This patch addresses the issue by clearing all the bits in the [14:12] slice of the file mapping's virtual address at generation time, thus forcing those bits the same for all mappings of a single shared library across processes and, in doing so, avoids instruction cache aliases.

It also adds the command line option "align_va_addr=(32|64|on|off)" with which virtual address alignment can be enabled for 32-bit or 64-bit x86 individually, or both, or be completely disabled.

This change leaves virtual region address allocation on other families and/or vendors unaffected.

Signed-off-by Borislav Petkov <[email protected]>

Link http://lkml.kernel.org/r/[email protected] open here

Signed-off-by H. Peter Anvin <[email protected]>

---
 Documentation/kernel-parameters.txt |   13 ++++++
 arch/x86/include/asm/elf.h          |   31 +++++++++++++
 arch/x86/kernel/cpu/amd.c           |   13 ++++++
 arch/x86/kernel/sys_x86_64.c        |   81 +++++++++++++++++++++++++++++++++-
 arch/x86/mm/mmap.c                  |   15 ------
 arch/x86/vdso/vma.c                 |    9 ++++
 6 files changed, 144 insertions(+), 18 deletions(-)

Documentation/kernel-parameters.txt +13 -0

index aa47be7..af73c03 100644
@@ -299,6 +299,19 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
 			behaviour to be specified.  Bit 0 enables warnings,
 			bit 1 enables fixups, and bit 2 sends a segfault.
 
+	align_va_addr=	[X86-64]
+			Align virtual addresses by clearing slice [14:12] when
+			allocating a VMA at process creation time. This option
+			gives you up to 3% performance improvement on AMD F15h
+			machines (where it is enabled by default) for a
+			CPU-intensive style benchmark, and it can vary highly in
+			a microbenchmark depending on workload and compiler.
+
+			1: only for 32-bit processes
+			2: only for 64-bit processes
+			on: enable for both 32- and 64-bit processes
+			off: disable for both 32- and 64-bit processes
+
 	amd_iommu=	[HW,X86-84]
 			Pass parameters to the AMD IOMMU driver in the system.
 			Possible values are:

arch/x86/include/asm/elf.h +31 -0

index f2ad216..5f962df 100644
@@ -4,6 +4,7 @@
 /*
  * ELF register definitions..
  */
+#include <linux/thread_info.h>
 
 #include <asm/ptrace.h>
 #include <asm/user.h>

@@ -320,4 +321,34 @@ extern int syscall32_setup_pages(struct linux_binprm *, int exstack);
 extern unsigned long arch_randomize_brk(struct mm_struct *mm);
 #define arch_randomize_brk arch_randomize_brk
 
+/*
+ * True on X86_32 or when emulating IA32 on X86_64
+ */
+static inline int mmap_is_ia32(void)
+{
+#ifdef CONFIG_X86_32
+	return 1;
+#endif
+#ifdef CONFIG_IA32_EMULATION
+	if (test_thread_flag(TIF_IA32))
+		return 1;
+#endif
+	return 0;
+}
+
+/* The first two values are special, do not change. See align_addr() */
+enum align_flags {
+	ALIGN_VA_32	= BIT(0),
+	ALIGN_VA_64	= BIT(1),
+	ALIGN_VDSO	= BIT(2),
+	ALIGN_TOPDOWN	= BIT(3),
+};
+
+struct va_alignment {
+	int flags;
+	unsigned long mask;
+} ____cacheline_aligned;
+
+extern struct va_alignment va_align;
+extern unsigned long align_addr(unsigned long, struct file *, enum align_flags);
 #endif /* _ASM_X86_ELF_H */

arch/x86/kernel/cpu/amd.c +13 -0

index b13ed39..b0234bc 100644
@@ -458,6 +458,19 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 					"with P0 frequency!\n");
 		}
 	}
+
+	if (c->x86 == 0x15) {
+		unsigned long upperbit;
+		u32 cpuid, assoc;
+
+		cpuid	 = cpuid_edx(0x80000005);
+		assoc	 = cpuid >> 16 & 0xff;
+		upperbit = ((cpuid >> 24) << 10) / assoc;
+
+		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
+		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
+
+	}
 }
 
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)

arch/x86/kernel/sys_x86_64.c +78 -3

112 lines not on this page · raw message

arch/x86/mm/mmap.c +0 ~~-15~~

index 1dab519..d4c0736 100644
@@ -51,21 +51,6 @@ static unsigned int stack_maxrandom_size(void)
 #define MIN_GAP (128*1024*1024UL + stack_maxrandom_size())
 #define MAX_GAP (TASK_SIZE/6*5)
 
-/*
- * True on X86_32 or when emulating IA32 on X86_64
- */
-static int mmap_is_ia32(void)
-{
-#ifdef CONFIG_X86_32
-	return 1;
-#endif
-#ifdef CONFIG_IA32_EMULATION
-	if (test_thread_flag(TIF_IA32))
-		return 1;
-#endif
-	return 0;
-}
-
 static int mmap_is_legacy(void)
 {
 	if (current->personality & ADDR_COMPAT_LAYOUT)

arch/x86/vdso/vma.c +9 -0

index 7abd2be..caa42ce 100644
@@ -69,6 +69,15 @@ static unsigned long vdso_addr(unsigned long start, unsigned len)
 	addr = start + (offset << PAGE_SHIFT);
 	if (addr >= end)
 		addr = end;
+
+	/*
+	 * page-align it here so that get_unmapped_area doesn't
+	 * align it wrongfully again to the next page. addr can come in 4K
+	 * unaligned here as a result of stack start randomization.
+	 */
+	addr = PAGE_ALIGN(addr);
+	addr = align_addr(addr, NULL, ALIGN_VDSO);
+
 	return addr;
 }

Signature · 1 line

1.7.8.rc0

Borislav Petkov

author

2011-11-04 11:15 UTC #2 lore raw

PATCH 2/4 x86: Add a BSP cpu_dev helper

Upstream commit: a110b5ec7371592eac856ac5c22dc7b518952d44

Add a function ptr to struct cpu_dev which is destined to be run only once on the BSP during boot.

Signed-off-by Borislav Petkov <[email protected]>

Link http://lkml.kernel.org/r/20110805180116.GB26217@aftab open here

Signed-off-by H. Peter Anvin <[email protected]>

---
 arch/x86/kernel/cpu/common.c |    3 +++
 arch/x86/kernel/cpu/cpu.h    |    1 +
 2 files changed, 4 insertions(+), 0 deletions(-)

arch/x86/kernel/cpu/common.c +3 -0

index 22a073d..8ed394a 100644
@@ -681,6 +681,9 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	filter_cpuid_features(c, false);
 
 	setup_smep(c);
+
+	if (this_cpu->c_bsp_init)
+		this_cpu->c_bsp_init(c);
 }
 
 void __init early_cpu_init(void)

arch/x86/kernel/cpu/cpu.h +1 -0

index e765633..1b22dcc 100644
@@ -18,6 +18,7 @@ struct cpu_dev {
 	struct		cpu_model_info c_models[4];
 
 	void            (*c_early_init)(struct cpuinfo_x86 *);
+	void		(*c_bsp_init)(struct cpuinfo_x86 *);
 	void		(*c_init)(struct cpuinfo_x86 *);
 	void		(*c_identify)(struct cpuinfo_x86 *);
 	unsigned int	(*c_size_cache)(struct cpuinfo_x86 *, unsigned int);

Signature · 1 line

1.7.8.rc0

Borislav Petkov

author

2011-11-04 11:15 UTC #3 lore raw

PATCH 3/4 x86, amd: Move BSP code to cpu_dev helper

In reply to Borislav Petkov

Upstream commit: 8fa8b035085e7320c15875c1f6b03b290ca2dd66

Move code which is run once on the BSP during boot into the cpu_dev helper.

[ hpa: removed bogus cpu_has -> static_cpu_has conversion ]

Signed-off-by Borislav Petkov <[email protected]>

Link http://lkml.kernel.org/r/20110805180409.GC26217@aftab open here

Signed-off-by H. Peter Anvin <[email protected]>

---
 arch/x86/kernel/cpu/amd.c |   59 ++++++++++++++++++++++-----------------------
 1 files changed, 29 insertions(+), 30 deletions(-)

arch/x86/kernel/cpu/amd.c +29 ~~-30~~

index b0234bc..b6e3e87 100644
@@ -410,6 +410,34 @@ static void __cpuinit early_init_amd_mc(struct cpuinfo_x86 *c)
 #endif
 }
 
+static void __cpuinit bsp_init_amd(struct cpuinfo_x86 *c)
+{
+	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
+
+		if (c->x86 > 0x10 ||
+		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
+			u64 val;
+
+			rdmsrl(MSR_K7_HWCR, val);
+			if (!(val & BIT(24)))
+				printk(KERN_WARNING FW_BUG "TSC doesn't count "
+					"with P0 frequency!\n");
+		}
+	}
+
+	if (c->x86 == 0x15) {
+		unsigned long upperbit;
+		u32 cpuid, assoc;
+
+		cpuid	 = cpuid_edx(0x80000005);
+		assoc	 = cpuid >> 16 & 0xff;
+		upperbit = ((cpuid >> 24) << 10) / assoc;
+
+		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
+		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
+	}
+}
+
 static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 {
 	early_init_amd_mc(c);

@@ -441,36 +469,6 @@ static void __cpuinit early_init_amd(struct cpuinfo_x86 *c)
 			set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
 	}
 #endif
-
-	/* We need to do the following only once */
-	if (c != &boot_cpu_data)
-		return;
-
-	if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
-
-		if (c->x86 > 0x10 ||
-		    (c->x86 == 0x10 && c->x86_model >= 0x2)) {
-			u64 val;
-
-			rdmsrl(MSR_K7_HWCR, val);
-			if (!(val & BIT(24)))
-				printk(KERN_WARNING FW_BUG "TSC doesn't count "
-					"with P0 frequency!\n");
-		}
-	}
-
-	if (c->x86 == 0x15) {
-		unsigned long upperbit;
-		u32 cpuid, assoc;
-
-		cpuid	 = cpuid_edx(0x80000005);
-		assoc	 = cpuid >> 16 & 0xff;
-		upperbit = ((cpuid >> 24) << 10) / assoc;
-
-		va_align.mask	  = (upperbit - 1) & PAGE_MASK;
-		va_align.flags    = ALIGN_VA_32 | ALIGN_VA_64;
-
-	}
 }
 
 static void __cpuinit init_amd(struct cpuinfo_x86 *c)

@@ -692,6 +690,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
 	.c_size_cache	= amd_size_cache,
 #endif
 	.c_early_init   = early_init_amd,
+	.c_bsp_init	= bsp_init_amd,
 	.c_init		= init_amd,
 	.c_x86_vendor	= X86_VENDOR_AMD,
 };

Signature · 1 line

1.7.8.rc0

Borislav Petkov

author

2011-11-04 11:15 UTC #4 lore raw

PATCH 4/4 x86-32, amd: Move va_align definition to unbreak 32-bit build

In reply to Borislav Petkov

From: Borislav Petkov <[email protected]>

Upstream commit: 9387f774d61b01ab71bade85e6d0bfab0b3419bd

hpa reported that dfb09f9b7ab03fd367740e541a5caf830ed56726 breaks 32-bit builds with the following error message:

/home/hpa/kernel/linux-tip.cpu/arch/x86/kernel/cpu/amd.c:437: undefined reference to va_align' /home/hpa/kernel/linux-tip.cpu/arch/x86/kernel/cpu/amd.c:436: undefined reference to va_align'

This is due to the fact that va_align is a global in a 64-bit only compilation unit. Move it to mmap.c where it is visible to both subarches.

Signed-off-by Borislav Petkov <[email protected]>

Link http://lkml.kernel.org/r/[email protected] open here

Signed-off-by H. Peter Anvin <[email protected]>

---
 arch/x86/kernel/sys_x86_64.c |    4 ----
 arch/x86/mm/mmap.c           |    5 ++++-
 2 files changed, 4 insertions(+), 5 deletions(-)

arch/x86/kernel/sys_x86_64.c +0 -4

index aaa8d09..fe7d2da 100644
@@ -18,10 +18,6 @@
 #include <asm/ia32.h>
 #include <asm/syscalls.h>
 
-struct __read_mostly va_alignment va_align = {
-	.flags = -1,
-};
-
 /*
  * Align a virtual address to avoid aliasing in the I$ on AMD F15h.
  *

arch/x86/mm/mmap.c +4 -1

index d4c0736..4b5ba85 100644
@@ -31,6 +31,10 @@
 #include <linux/sched.h>
 #include <asm/elf.h>
 
+struct __read_mostly va_alignment va_align = {
+	.flags = -1,
+};
+
 static unsigned int stack_maxrandom_size(void)
 {
 	unsigned int max = 0;

@@ -42,7 +46,6 @@ static unsigned int stack_maxrandom_size(void)
 	return max;
 }
 
-
 /*
  * Top of mmap area (just below the process stack).
  *

Signature · 1 line

1.7.8.rc0