ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
--- arch/arm/include/asm/cachetype.h | 16 ++++++++++++++++ arch/arm/mm/fault-armv.c | 5 ++--- arch/arm/mm/flush.c | 8 +++----- 3 files changed, 21 insertions(+), 8 deletions(-)
arch/arm/include/asm/cachetype.h +16 -0
@@ -1,6 +1,8 @@#ifndef __ASM_ARM_CACHETYPE_H#define __ASM_ARM_CACHETYPE_H +#include <asm/cputype.h> +#define CACHEID_VIVT (1 << 0)#define CACHEID_VIPT_NONALIASING (1 << 1)#define CACHEID_VIPT_ALIASING (1 << 2)
@@ -49,4 +51,18 @@ static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask) (~__CACHEID_NEVER & __CACHEID_ARCH_MIN & mask & cacheid); } +/* +* Cache maintenance operations hardware broadcasting. +*/ +#ifndef CONFIG_SMP +#define cache_ops_hw_broadcast () 1 +#elif __LINUX_ARM_ARCH__ <=6 +#define cache_ops_hw_broadcast () 0 +#else +static inline int cache_ops_hw_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) >=1 ; +} +#endif +#endif
arch/arm/mm/fault-armv.c +2 -3
@@ -153,10 +153,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) page =pfn_to_page (pfn); mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (test_and_clear_bit (PG_dcache_dirty, &page->flags)) +if (cache_ops_hw_broadcast () && +test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent (mapping, vma, addr, pfn);
arch/arm/mm/flush.c +3 -5
@@ -198,12 +198,10 @@ void flush_dcache_page(struct page *page) {struct address_space *mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (cache_ops_hw_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
Quoted 21 lines · Catalin Marinas
arch/arm/include/asm/cachetype.h +6
-0@@ -1,6 +1,8 @@#ifndef __ASM_ARM_CACHETYPE_H#define __ASM_ARM_CACHETYPE_H +#include <asm/cputype.h> +#define CACHEID_VIVT (1 << 0)#define CACHEID_VIPT_NONALIASING (1 << 1)#define CACHEID_VIPT_ALIASING (1 << 2)@@ -49,4 +51,18 @@ static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask) (~__CACHEID_NEVER & __CACHEID_ARCH_MIN & mask & cacheid); } +/* +* Cache maintenance operations hardware broadcasting. +*/ +#ifndef CONFIG_SMP
arch/arm/include/asm/cachetype.h static inline unsigned int __attribute__((pure)) cacheid_is(unsigned int mask) · line 67
Quoted 10 lines
arch/arm/include/asm/cachetype.h +10
-0+#define cache_ops_hw_broadcast () 1 +#elif __LINUX_ARM_ARCH__ <=6 +#define cache_ops_hw_broadcast () 0 +#else +static inline int cache_ops_hw_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) >=1 ; +} +#endif +
Please place this in asm/smp_plat.h, and let's have it in the same sense as the TLB ops version as well for consistency (so we don't have to individually remember how each of these functions work and their sense.) IOW (this is cut'n'pasted, so is tab broken):
arch/arm/include/asm/smp_plat.h +5 -0
@@ -13,4 +13,9 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +static inline int cache_ops_need_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; +} +#endif
RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
Quoted 6 lines · Russell King - ARM Linux
Quoted 5 lines · Catalin Marinas
arch/arm/include/asm/cachetype.h +0
-0@@ -1,6 +1,8 @@
[...]
Quoted 6 lines
Quoted 6 lines
arch/arm/include/asm/cachetype.h +6
-0+#ifndef CONFIG_SMP +#define cache_ops_hw_broadcast () 1 +#elif __LINUX_ARM_ARCH__ <=6 +#define cache_ops_hw_broadcast () 0 +#else +static inline int cache_ops_hw_broadcast (void )
Quoted 8 lines
arch/arm/include/asm/cachetype.h
Quoted 4 lines
arch/arm/include/asm/cachetype.h +4
-0+{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) >=1 ; +} +#endif Please place this in
asm/smp_plat.h, and let's have it in the same sense as the TLB ops version as well for consistency (so we don't have to individually remember how each of these functions work and their sense.)
I was looking at this for a slightly different perspective since we don't do in-software cache broadcasting. But I'm ok with change the name to be the same as the tlb one.
If we place this in smp_plat.h, we need to include this header in fault-armv.c and flush.c and also add the non-SMP cache_ops_need_broadcast() variant (or use #ifdefs at the calling site though I don't think it would look nice).
ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
From: Catalin Marinas <[email protected]>
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
--- arch/arm/include/asm/smp_plat.h | 9 +++++++++ arch/arm/mm/fault-armv.c | 6 +++--- arch/arm/mm/flush.c | 9 ++++----- 3 files changed, 16 insertions(+), 8 deletions(-)
arch/arm/include/asm/smp_plat.h +9 -0
@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +#ifndef CONFIG_SMP +#define cache_ops_need_broadcast () 0 +#else +static inline int cache_ops_need_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; +} +#endif +#endif
arch/arm/mm/fault-armv.c +3 -3
@@ -22,6 +22,7 @@#include <asm/cachetype.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> static unsigned long shared_pte_mask =L_PTE_MT_BUFFERABLE ;
@@ -153,10 +154,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) page =pfn_to_page (pfn); mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (test_and_clear_bit (PG_dcache_dirty, &page->flags)) +if (!cache_ops_need_broadcast () && +test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent (mapping, vma, addr, pfn);
arch/arm/mm/flush.c +4 -5
@@ -15,6 +15,7 @@#include <asm/cachetype.h> #include <asm/system.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> #include "mm.h"
@@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page) {struct address_space *mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (!cache_ops_need_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
Signature · 1 line
Catalin
RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
arch/arm/mm/fault-armv.c void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) · line 158
Quoted 4 lines · Catalin Marinas
arch/arm/mm/fault-armv.c +2
-2-#ifndef CONFIG_SMP -if (test_and_clear_bit (PG_dcache_dirty, &page->flags)) +if (!cache_ops_need_broadcast () && +test_and_clear_bit (PG_dcache_dirty, &page->flags))
We really don't need the additional check here - there's only one place where PG_dcache_dirty gets set, and that's flush_dcache_page(). If we avoid setting the bit for hardware which doesn't broadcast, we won't call __flush_dcache_page().
RFC PATCH 1/1 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
Quoted 10 lines · Russell King - ARM Linux
Review
Quoted 4 lines · Catalin Marinas
(quoted diff) +2
-2-#ifndef CONFIG_SMP - if (test_and_clear_bit(PG_dcache_dirty, &page->flags)) + if (!cache_ops_need_broadcast() && + test_and_clear_bit(PG_dcache_dirty, &page->flags))We really don't need the additional check here - there's only one place where
PG_dcache_dirtygets set, and that'sflush_dcache_page(). If we avoid setting the bit for hardware which doesn't broadcast, we won't call__flush_dcache_page().
OK. This was meant as slight optimisation for the ARMv6 MP case but I removed that from the cache_ops_need_broadcast() definition (you never know, someone may build an ARMv6 MP with cache broadcasting in hw).
Here's the updated patch:
ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
From: Catalin Marinas <[email protected]>
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
--- arch/arm/include/asm/smp_plat.h | 9 +++++++++ arch/arm/mm/fault-armv.c | 2 -- arch/arm/mm/flush.c | 9 ++++----- 3 files changed, 13 insertions(+), 7 deletions(-)
arch/arm/include/asm/smp_plat.h +9 -0
@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +#ifndef CONFIG_SMP +#define cache_ops_need_broadcast () 0 +#else +static inline int cache_ops_need_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; +} +#endif +#endif
arch/arm/mm/fault-armv.c +0 -2
@@ -153,10 +153,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) page =pfn_to_page (pfn); mapping =page_mapping (page); -#ifndef CONFIG_SMP if (test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent (mapping, vma, addr, pfn);
arch/arm/mm/flush.c +4 -5
@@ -15,6 +15,7 @@#include <asm/cachetype.h> #include <asm/system.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> #include "mm.h"
@@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page) {struct address_space *mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (!cache_ops_need_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
Signature · 1 line
Catalin