Skip to contentSkip to conversation

CPA: Don't flush caches on CPUs that support self-snoop

PATCH New Andi Kleen 1 participant 3 messages Feb 1, 2008

Subsystems (2)
lore
Share & tools
lore

Andi Kleen

author
#0 lore raw

PATCH 1/3 CPA: Don't flush caches on CPUs that support self-snoop

The specification of SS in the public manuals is a little unclear, but I got confirmation from Intel that SS implies that there is no cache flush needed on caching attribute changes.

---
 arch/x86/mm/pageattr.c       |    5 ++++-
 include/asm-x86/cpufeature.h |    1 +
 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/mm/pageattr.c +4 -1
Index: linux/arch/x86/mm/pageattr.c
===================================================================
@@ -60,7 +60,7 @@ static void __cpa_flush_all(void *arg)
 	 */
 	__flush_tlb_all();
 
-	if (boot_cpu_data.x86_model >= 4)
+	if (!cpu_has_ss && boot_cpu_data.x86_model >= 4)
 		wbinvd();
 }
 
@@ -91,6 +91,9 @@ static void cpa_flush_range(unsigned lon
 
 	on_each_cpu(__cpa_flush_range, NULL, 1, 1);
 
+	if (cpu_has_ss)
+		return;
+
 	/*
 	 * We only need to flush on one CPU,
 	 * clflush is a MESI-coherent instruction that
include/asm-x86/cpufeature.h +1 -0
Index: linux/include/asm-x86/cpufeature.h
===================================================================
@@ -153,6 +153,7 @@
 #define cpu_has_mtrr		boot_cpu_has(X86_FEATURE_MTRR)
 #define cpu_has_mmx		boot_cpu_has(X86_FEATURE_MMX)
 #define cpu_has_fxsr		boot_cpu_has(X86_FEATURE_FXSR)
+#define cpu_has_ss		boot_cpu_has(X86_FEATURE_SELFSNOOP)
 #define cpu_has_xmm		boot_cpu_has(X86_FEATURE_XMM)
 #define cpu_has_xmm2		boot_cpu_has(X86_FEATURE_XMM2)
 #define cpu_has_xmm3		boot_cpu_has(X86_FEATURE_XMM3)
author
#1 lore raw

PATCH 2/3 CPA: Only flush the cache if the caching attributes have changed.

We only need to flush the caches in cpa() if the the caching attributes have changed. Otherwise only flush the TLBs.

This checks the PAT bits too although they are currently not used by the kernel.

---
 arch/x86/mm/pageattr.c    |   28 +++++++++++++++++++---------
 include/asm-x86/pgtable.h |    4 ++++
 2 files changed, 23 insertions(+), 9 deletions(-)
arch/x86/mm/pageattr.c +20 -10
Index: linux/arch/x86/mm/pageattr.c
===================================================================
@@ -54,21 +54,22 @@ void clflush_cache_range(void *vaddr, un
 
 static void __cpa_flush_all(void *arg)
 {
+	unsigned long cache = (unsigned long)arg;
 	/*
 	 * Flush all to work around Errata in early athlons regarding
 	 * large page flushing.
 	 */
 	__flush_tlb_all();
 
-	if (!cpu_has_ss && boot_cpu_data.x86_model >= 4)
+	if (cache && !cpu_has_ss && boot_cpu_data.x86_model >= 4)
 		wbinvd();
 }
 
-static void cpa_flush_all(void)
+static void cpa_flush_all(int cache)
 {
 	BUG_ON(irqs_disabled());
 
-	on_each_cpu(__cpa_flush_all, NULL, 1, 1);
+	on_each_cpu(__cpa_flush_all, (void *)(unsigned long)cache, 1, 1);
 }
 
 static void __cpa_flush_range(void *arg)
@@ -81,7 +82,7 @@ static void __cpa_flush_range(void *arg)
 	__flush_tlb_all();
 }
 
-static void cpa_flush_range(unsigned long start, int numpages)
+static void cpa_flush_range(unsigned long start, int numpages, int cache)
 {
 	unsigned int i, level;
 	unsigned long addr;
@@ -91,7 +92,7 @@ static void cpa_flush_range(unsigned lon
 
 	on_each_cpu(__cpa_flush_range, NULL, 1, 1);
 
-	if (cpu_has_ss)
+	if (!cache || cpu_has_ss)
 		return;
 
 	/*
@@ -395,7 +396,8 @@ static int __change_page_attr_set_clr(un
 }
 
 static int change_page_attr_set_clr(unsigned long addr, int numpages,
-				    pgprot_t mask_set, pgprot_t mask_clr)
+				    pgprot_t mask_set, pgprot_t mask_clr,
+				    int cache)
 {
 	int ret = __change_page_attr_set_clr(addr, numpages, mask_set,
 					     mask_clr);
@@ -407,23 +409,31 @@ static int change_page_attr_set_clr(unsi
 	 * wbindv):
 	 */
 	if (!ret && cpu_has_clflush)
-		cpa_flush_range(addr, numpages);
+		cpa_flush_range(addr, numpages, cache);
 	else
-		cpa_flush_all();
+		cpa_flush_all(cache);
 
 	return ret;
 }
 
+static int cache_attr(pgprot_t attr)
+{
+	if (pgprot_val(attr) & (_PAGE_PAT|_PAGE_PAT_LARGE|_PAGE_PWT|_PAGE_PCD))
+		return 1;
+	return 0;
+}
+
 static inline int change_page_attr_set(unsigned long addr, int numpages,
 				       pgprot_t mask)
 {
-	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0));
+	return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0),
+					cache_attr(mask));
 }
 
 static inline int change_page_attr_clear(unsigned long addr, int numpages,
 					 pgprot_t mask)
 {
-	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask);
+	return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0);
 }
 
 int set_memory_uc(unsigned long addr, int numpages)
include/asm-x86/pgtable.h +4 -0
Index: linux/include/asm-x86/pgtable.h
===================================================================
@@ -13,10 +13,12 @@
 #define _PAGE_BIT_DIRTY		6
 #define _PAGE_BIT_FILE		6
 #define _PAGE_BIT_PSE		7	/* 4 MB (or 2MB) page */
+#define _PAGE_BIT_PAT		7	/* on 4KB pages */
 #define _PAGE_BIT_GLOBAL	8	/* Global TLB entry PPro+ */
 #define _PAGE_BIT_UNUSED1	9	/* available for programmer */
 #define _PAGE_BIT_UNUSED2	10
 #define _PAGE_BIT_UNUSED3	11
+#define _PAGE_BIT_PAT_LARGE	12	/* On 2MB or 1GB pages */
 #define _PAGE_BIT_NX           63       /* No execute: only valid after cpuid check */
 
 /*
@@ -36,6 +38,8 @@
 #define _PAGE_UNUSED1	(_AC(1, L)<<_PAGE_BIT_UNUSED1)
 #define _PAGE_UNUSED2	(_AC(1, L)<<_PAGE_BIT_UNUSED2)
 #define _PAGE_UNUSED3	(_AC(1, L)<<_PAGE_BIT_UNUSED3)
+#define _PAGE_PAT	(_AC(1, L)<<_PAGE_BIT_PAT)
+#define _PAGE_PAT_LARGE (_AC(1, L)<<_PAGE_BIT_PAT_LARGE)
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 #define _PAGE_NX	(_AC(1, ULL) << _PAGE_BIT_NX)
author
#2 lore raw

PATCH 3/3 CPA: Remove outdated comment in kernel_map_pages and add warning about recursion

In reply to Andi Kleen
---
 arch/x86/mm/pageattr.c |    5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)
arch/x86/mm/pageattr.c +1 -4
Index: linux/arch/x86/mm/pageattr.c
===================================================================
@@ -556,6 +556,7 @@ static int __set_pages_np(struct page *p
 					__pgprot(_PAGE_PRESENT));
 }
 
+/* This function recurses with unknown maximum depth */
 void kernel_map_pages(struct page *page, int numpages, int enable)
 {
 	if (PageHighMem(page))
@@ -571,10 +572,6 @@ void kernel_map_pages(struct page *page,
 	if (!debug_pagealloc_enabled)
 		return;
 
-	/*
-	 * The return value is ignored - the calls cannot fail,
-	 * large pages are disabled at boot time:
-	 */
 	if (enable)
 		__set_pages_p(page, numpages);
 	else