Skip to contentSkip to conversation

ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

RFC PATCH New Catalin Marinas 2 participants 5 messages Nov 5, 2009 Arm (32-bit)

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.

---
 arch/arm/include/asm/cachetype.h |   16 ++++++++++++++++
 arch/arm/mm/fault-armv.c         |    5 ++---
 arch/arm/mm/flush.c              |    8 +++-----
 3 files changed, 21 insertions(+), 8 deletions(-)
arch/arm/include/asm/cachetype.h +16 -0
index d3a4c2c..3d5d1c9 100644
@@ -1,6 +1,8 @@
 #ifndef __ASM_ARM_CACHETYPE_H
 #define __ASM_ARM_CACHETYPE_H
 
+#include <asm/cputype.h>
+
 #define CACHEID_VIVT			(1 << 0)
 #define CACHEID_VIPT_NONALIASING	(1 << 1)
 #define CACHEID_VIPT_ALIASING		(1 << 2)
@@ -49,4 +51,18 @@ static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask)
 	       (~__CACHEID_NEVER & __CACHEID_ARCH_MIN & mask & cacheid);
 }
 
+/*
+ * Cache maintenance operations hardware broadcasting.
+ */
+#ifndef CONFIG_SMP
+#define cache_ops_hw_broadcast()	1
+#elif __LINUX_ARM_ARCH__ <= 6
+#define cache_ops_hw_broadcast()	0
+#else
+static inline int cache_ops_hw_broadcast(void)
+{
+	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) >= 1;
+}
+#endif
+
 #endif
arch/arm/mm/fault-armv.c +2 -3
index d0d17b6..214c3ed 100644
@@ -153,10 +153,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 
 	page = pfn_to_page(pfn);
 	mapping = page_mapping(page);
-#ifndef CONFIG_SMP
-	if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
+	if (cache_ops_hw_broadcast() &&
+	    test_and_clear_bit(PG_dcache_dirty, &page->flags))
 		__flush_dcache_page(mapping, page);
-#endif
 	if (mapping) {
 		if (cache_is_vivt())
 			make_coherent(mapping, vma, addr, pfn);
arch/arm/mm/flush.c +3 -5
index 7f294f3..4e63786 100644
@@ -198,12 +198,10 @@ void flush_dcache_page(struct page *page)
 {
 	struct address_space *mapping = page_mapping(page);
 
-#ifndef CONFIG_SMP
-	if (!PageHighMem(page) && mapping && !mapping_mapped(mapping))
+	if (cache_ops_hw_broadcast() &&
+	    !PageHighMem(page) && mapping && !mapping_mapped(mapping))
 		set_bit(PG_dcache_dirty, &page->flags);
-	else
-#endif
-	{
+	else {
 		__flush_dcache_page(mapping, page);
 		if (mapping && cache_is_vivt())
 			__flush_dcache_aliases(mapping, page);
#1 lore raw

RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

Quoted 21 lines · Catalin Marinas
arch/arm/include/asm/cachetype.h +6 -0
index d3a4c2c..3d5d1c9 100644
@@ -1,6 +1,8 @@
 #ifndef __ASM_ARM_CACHETYPE_H
 #define __ASM_ARM_CACHETYPE_H
 
+#include <asm/cputype.h>
+
 #define CACHEID_VIVT			(1 << 0)
 #define CACHEID_VIPT_NONALIASING	(1 << 1)
 #define CACHEID_VIPT_ALIASING		(1 << 2)
@@ -49,4 +51,18 @@ static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask)
 	       (~__CACHEID_NEVER & __CACHEID_ARCH_MIN & mask & cacheid);
 }
 
+/*
+ * Cache maintenance operations hardware broadcasting.
+ */
+#ifndef CONFIG_SMP

arch/arm/include/asm/cachetype.h static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask) · line 67

Quoted 10 lines
arch/arm/include/asm/cachetype.h +10 -0
+#define cache_ops_hw_broadcast()	1
+#elif __LINUX_ARM_ARCH__ <= 6
+#define cache_ops_hw_broadcast()	0
+#else
+static inline int cache_ops_hw_broadcast(void)
+{
+	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) >= 1;
+}
+#endif
+

Please place this in asm/smp_plat.h, and let's have it in the same sense as the TLB ops version as well for consistency (so we don't have to individually remember how each of these functions work and their sense.) IOW (this is cut'n'pasted, so is tab broken):

arch/arm/include/asm/smp_plat.h +5 -0
index 59303e2..e621530 100644
@@ -13,4 +13,9 @@ static inline int tlb_ops_need_broadcast(void)
        return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
 }
 
+static inline int cache_ops_need_broadcast(void)
+{
+       return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 1;
+}
+
 #endif
#2 lore raw

RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

Quoted 6 lines · Russell King - ARM Linux
Quoted 5 lines · Catalin Marinas
arch/arm/include/asm/cachetype.h +0 -0
index d3a4c2c..3d5d1c9 100644
@@ -1,6 +1,8 @@

[...]

Quoted 6 lines
Quoted 6 lines
arch/arm/include/asm/cachetype.h +6 -0
+#ifndef CONFIG_SMP
+#define cache_ops_hw_broadcast()	1
+#elif __LINUX_ARM_ARCH__ <= 6
+#define cache_ops_hw_broadcast()	0
+#else
+static inline int cache_ops_hw_broadcast(void)
Quoted 8 lines

arch/arm/include/asm/cachetype.h

Quoted 4 lines
arch/arm/include/asm/cachetype.h +4 -0
+{
+	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) >= 1;
+}
+#endif

Please place this in asm/smp_plat.h, and let's have it in the same sense as the TLB ops version as well for consistency (so we don't have to individually remember how each of these functions work and their sense.)

I was looking at this for a slightly different perspective since we don't do in-software cache broadcasting. But I'm ok with change the name to be the same as the tlb one.

If we place this in smp_plat.h, we need to include this header in fault-armv.c and flush.c and also add the non-SMP cache_ops_need_broadcast() variant (or use #ifdefs at the calling site though I don't think it would look nice).

ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

From: Catalin Marinas <[email protected]>

ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.

---
 arch/arm/include/asm/smp_plat.h |    9 +++++++++
 arch/arm/mm/fault-armv.c        |    6 +++---
 arch/arm/mm/flush.c             |    9 ++++-----
 3 files changed, 16 insertions(+), 8 deletions(-)
arch/arm/include/asm/smp_plat.h +9 -0
index 59303e2..e587167 100644
@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void)
 	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
 }
 
+#ifndef CONFIG_SMP
+#define cache_ops_need_broadcast()	0
+#else
+static inline int cache_ops_need_broadcast(void)
+{
+	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 1;
+}
+#endif
+
 #endif
arch/arm/mm/fault-armv.c +3 -3
index d0d17b6..a7f5c69 100644
@@ -22,6 +22,7 @@
 #include <asm/cachetype.h>
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
+#include <asm/smp_plat.h>
 
 static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
 
@@ -153,10 +154,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 
 	page = pfn_to_page(pfn);
 	mapping = page_mapping(page);
-#ifndef CONFIG_SMP
-	if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
+	if (!cache_ops_need_broadcast() &&
+	    test_and_clear_bit(PG_dcache_dirty, &page->flags))
 		__flush_dcache_page(mapping, page);
-#endif
 	if (mapping) {
 		if (cache_is_vivt())
 			make_coherent(mapping, vma, addr, pfn);
arch/arm/mm/flush.c +4 -5
index 7f294f3..2d3325d 100644
@@ -15,6 +15,7 @@
 #include <asm/cachetype.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
+#include <asm/smp_plat.h>
 
 #include "mm.h"
 
@@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page)
 {
 	struct address_space *mapping = page_mapping(page);
 
-#ifndef CONFIG_SMP
-	if (!PageHighMem(page) && mapping && !mapping_mapped(mapping))
+	if (!cache_ops_need_broadcast() &&
+	    !PageHighMem(page) && mapping && !mapping_mapped(mapping))
 		set_bit(PG_dcache_dirty, &page->flags);
-	else
-#endif
-	{
+	else {
 		__flush_dcache_page(mapping, page);
 		if (mapping && cache_is_vivt())
 			__flush_dcache_aliases(mapping, page);
Signature · 1 line
Catalin
#3 lore raw

RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

arch/arm/mm/fault-armv.c void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) · line 158

Quoted 4 lines · Catalin Marinas
arch/arm/mm/fault-armv.c +2 -2
-#ifndef CONFIG_SMP
-	if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
+	if (!cache_ops_need_broadcast() &&
+	    test_and_clear_bit(PG_dcache_dirty, &page->flags))

We really don't need the additional check here - there's only one place where PG_dcache_dirty gets set, and that's flush_dcache_page(). If we avoid setting the bit for hardware which doesn't broadcast, we won't call __flush_dcache_page().

#4 lore raw

RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

Quoted 10 lines · Russell King - ARM Linux

Review

Quoted 4 lines · Catalin Marinas
(quoted diff) +2 -2
-#ifndef CONFIG_SMP
-	if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
+	if (!cache_ops_need_broadcast() &&
+	    test_and_clear_bit(PG_dcache_dirty, &page->flags))

We really don't need the additional check here - there's only one place where PG_dcache_dirty gets set, and that's flush_dcache_page(). If we avoid setting the bit for hardware which doesn't broadcast, we won't call __flush_dcache_page().

OK. This was meant as slight optimisation for the ARMv6 MP case but I removed that from the cache_ops_need_broadcast() definition (you never know, someone may build an ARMv6 MP with cache broadcasting in hw).

Here's the updated patch:

ARMv7: Use lazy cache flushing if hardware broadcasts cache operations

From: Catalin Marinas <[email protected]>

ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.

---
 arch/arm/include/asm/smp_plat.h |    9 +++++++++
 arch/arm/mm/fault-armv.c        |    2 --
 arch/arm/mm/flush.c             |    9 ++++-----
 3 files changed, 13 insertions(+), 7 deletions(-)
arch/arm/include/asm/smp_plat.h +9 -0
index 59303e2..e587167 100644
@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void)
 	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
 }
 
+#ifndef CONFIG_SMP
+#define cache_ops_need_broadcast()	0
+#else
+static inline int cache_ops_need_broadcast(void)
+{
+	return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 1;
+}
+#endif
+
 #endif
arch/arm/mm/fault-armv.c +0 -2
index d0d17b6..bb60117 100644
@@ -153,10 +153,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 
 	page = pfn_to_page(pfn);
 	mapping = page_mapping(page);
-#ifndef CONFIG_SMP
 	if (test_and_clear_bit(PG_dcache_dirty, &page->flags))
 		__flush_dcache_page(mapping, page);
-#endif
 	if (mapping) {
 		if (cache_is_vivt())
 			make_coherent(mapping, vma, addr, pfn);
arch/arm/mm/flush.c +4 -5
index 7f294f3..2d3325d 100644
@@ -15,6 +15,7 @@
 #include <asm/cachetype.h>
 #include <asm/system.h>
 #include <asm/tlbflush.h>
+#include <asm/smp_plat.h>
 
 #include "mm.h"
 
@@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page)
 {
 	struct address_space *mapping = page_mapping(page);
 
-#ifndef CONFIG_SMP
-	if (!PageHighMem(page) && mapping && !mapping_mapped(mapping))
+	if (!cache_ops_need_broadcast() &&
+	    !PageHighMem(page) && mapping && !mapping_mapped(mapping))
 		set_bit(PG_dcache_dirty, &page->flags);
-	else
-#endif
-	{
+	else {
 		__flush_dcache_page(mapping, page);
 		if (mapping && cache_is_vivt())
 			__flush_dcache_aliases(mapping, page);
Signature · 1 line
Catalin