arm64: change PoC D-cache flush to PoU
v3 cover arm64: change PoC D-cache flush to PoU
For keeping I and D coherent, dcache flush till PoU(Point of Unification) should be sufficient instead of doing till PoC(Point of coherence). In SoC with more levels of cache, there could be a performance hit in doing flush till PoC as __flush_dcache_area does both flush and invalidate. Introduced new API __clean_dcache_area_pou which does only clean till PoU.
Also deferred dcache flush in __cpu_copy_user_page to __sync_icache_dcache.
changes since v2 [2]:
Incorporated Mark Rutland's review comments of
* fixing comments
* creating a helper function for __sync_icache_dcache.
changes since v1 [1]:
Incorporated Mark Rutland's review comments of
* renaming __flush_dcache_area_pou to __clean_dcache_area_pou
* using inner shareable domain for dsb in __clean_dcache_area_pou
* having a common macro for __flush_dcache_area and __clean_dcache_area_pou.
Thanks,
Ashok
[1] http://lists.infradead.org/pipermail/linux-arm-kernel/2015-December/393527.html
[2] http://lists.infradead.org/pipermail/linux-arm-kernel/2015-December/393837.html
Ashok Kumar (2):
arm64: Defer dcache flush in __cpu_copy_user_page
arm64: Use PoU cache instr for I/D coherency
arch/arm64/include/asm/cacheflush.h | 1 + arch/arm64/mm/cache.S | 50 +++++++++++++++++++++++++++++-------- arch/arm64/mm/copypage.c | 3 ++- arch/arm64/mm/flush.c | 33 +++++++++++++----------- 4 files changed, 60 insertions(+), 27 deletions(-)
Signature · 1 line
2.1.0
PATCH v3 1/2 arm64: Defer dcache flush in __cpu_copy_user_page
Defer dcache flushing to __sync_icache_dcache by calling flush_dcache_page which clears PG_dcache_clean flag.
--- arch/arm64/mm/copypage.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/arm64/mm/copypage.c +2 -1
@@ -24,8 +24,9 @@void __cpu_copy_user_page (void *kto,const void *kfrom,unsigned long vaddr) { +struct page *page =virt_to_page (kto);copy_page (kto, kfrom); -__flush_dcache_area (kto,PAGE_SIZE ); +flush_dcache_page (page); }EXPORT_SYMBOL_GPL (__cpu_copy_user_page);
Signature · 1 line
2.1.0
PATCH v3 2/2 arm64: Use PoU cache instr for I/D coherency
In reply to Ashok KumarIn systems with three levels of cache(PoU at L1 and PoC at L3), PoC cache flush instructions flushes L2 and L3 caches which could affect performance.
For cache flushes for I and D coherency, PoU should suffice. So changing all I and D coherency related cache flushes to PoU.
Introduced a new __clean_dcache_area_pou API for dcache flush till PoU and provided a common macro for __flush_dcache_area and __clean_dcache_area_pou.
--- arch/arm64/include/asm/cacheflush.h | 1 + arch/arm64/mm/cache.S | 50 +++++++++++++++++++++++++++++-------- arch/arm64/mm/flush.c | 33 +++++++++++++----------- 3 files changed, 58 insertions(+), 26 deletions(-)
arch/arm64/include/asm/cacheflush.h +1 -0
@@ -68,6 +68,7 @@extern void flush_cache_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void flush_icache_range (unsigned long start,unsigned long end);extern void __flush_dcache_area (void *addr,size_t len); +extern void __clean_dcache_area_pou (void *addr,size_t len);extern long __flush_cache_user_range (unsigned long start,unsigned long end);static inline void flush_cache_mm (struct mm_struct *mm)
arch/arm64/mm/cache.S +39 -11
67 lines not on this page · raw message
arch/arm64/mm/flush.c +18 -15
@@ -34,19 +34,24 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start,__flush_icache_all (); } +static void sync_icache_aliases (void *kaddr,unsigned long len) +{ +unsigned long addr = (unsigned long )kaddr; + +if (icache_is_aliasing ()) { +__clean_dcache_area_pou (kaddr, len); +__flush_icache_all (); + }else { +flush_icache_range (addr, addr + len); + } +} +static void flush_ptrace_access (struct vm_area_struct *vma,struct page *page,unsigned long uaddr,void *kaddr,unsigned long len) { -if (vma->vm_flags &VM_EXEC ) { -unsigned long addr = (unsigned long )kaddr; -if (icache_is_aliasing ()) { -__flush_dcache_area (kaddr, len); -__flush_icache_all (); - }else { -flush_icache_range (addr, addr + len); - } - } +if (vma->vm_flags &VM_EXEC ) +sync_icache_aliases (kaddr, len); }/*
@@ -74,13 +79,11 @@ void __sync_icache_dcache(pte_t pte, unsigned long addr)if (!page_mapping (page))return ; -if (!test_and_set_bit (PG_dcache_clean, &page->flags)) { -__flush_dcache_area (page_address (page), -PAGE_SIZE <<compound_order (page)); +if (!test_and_set_bit (PG_dcache_clean, &page->flags)) +sync_icache_aliases (page_address (page), +PAGE_SIZE <<compound_order (page)); +else if (icache_is_aivivt ())__flush_icache_all (); - }else if (icache_is_aivivt ()) { -__flush_icache_all (); - } }/*
Signature · 1 line
2.1.0
Catalin Marinas
maintainerPATCH v3 2/2 arm64: Use PoU cache instr for I/D coherency
Quoted 6 lines · Ashok Kumar
arch/arm64/mm/flush.c +0
-2@@ -74,13 +79,11 @@ void __sync_icache_dcache(pte_t pte, unsigned long addr)if (!page_mapping (page))return ; -if (!test_and_set_bit (PG_dcache_clean, &page->flags)) { - __flush_dcache_area(page_address(page),
arch/arm64/mm/flush.c void __sync_icache_dcache(pte_t pte, unsigned long addr) · line 87
Quoted 10 lines
arch/arm64/mm/flush.c +4
-4- PAGE_SIZE << compound_order(page)); +if (!test_and_set_bit (PG_dcache_clean, &page->flags)) +sync_icache_aliases (page_address (page), +PAGE_SIZE <<compound_order (page)); +else if (icache_is_aivivt ())__flush_icache_all (); - }else if (icache_is_aivivt ()) { -__flush_icache_all (); - } }
You changed the original code path slightly here. We had a __flush_icache_all() even for non-aliasing VIPT but it now does the I-cache invalidation per page. It may be an improvement, I can't tell without benchmarks but you should at least mention this in the commit log so that we remember in the future. Apart from this:
PATCH v3 2/2 arm64: Use PoU cache instr for I/D coherency
In reply to Ashok KumarQuoted 48 lines · Ashok Kumar
In systems with three levels of cache(PoU at L1 and PoC at L3), PoC cache flush instructions flushes L2 and L3 caches which could affect performance.
For cache flushes for I and D coherency, PoU should suffice. So changing all I and D coherency related cache flushes to PoU.Introduced a new
__clean_dcache_area_pouAPI for dcache flush till PoU and provided a common macro for__flush_dcache_areaand__clean_dcache_area_pou.Reviewed-by Mark Rutland <[email protected]>Signed-off-by Ashok Kumar <[email protected]>--- arch/arm64/include/asm/cacheflush.h | 1 + arch/arm64/mm/cache.S | 50 +++++++++++++++++++++++++++++-------- arch/arm64/mm/flush.c | 33 +++++++++++++----------- 3 files changed, 58 insertions(+), 26 deletions(-)arch/arm64/include/asm/cacheflush.h +1
-0@@ -68,6 +68,7 @@extern void flush_cache_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void flush_icache_range (unsigned long start,unsigned long end);extern void __flush_dcache_area (void *addr,size_t len); +extern void __clean_dcache_area_pou (void *addr,size_t len);extern long __flush_cache_user_range (unsigned long start,unsigned long end);static inline void flush_cache_mm (struct mm_struct *mm)arch/arm64/mm/cache.S +10
-0@@ -79,28 +79,56 @@ ENDPROC(flush_icache_range) ENDPROC(__flush_cache_user_range) /* + * Macro to perform a data cache maintenance for the interval + * [kaddr, kaddr + size) + * + * op: operation passed to dc instruction + * domain: domain used in dsb instruciton + * kaddr: starting virtual address of the region + * size: size of the region + * Corrupts: kaddr, size, tmp1, tmp2 + */ + .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
arch/arm64/mm/cache.S ENDPROC(flush_icache_range) · line 101
Quoted 10 lines
arch/arm64/mm/cache.S +10
-0+ dcache_line_size \tmp1, \tmp2 + add \size, \kaddr, \size + sub \tmp2, \tmp1, #1 + bic \kaddr, \kaddr, \tmp2 +1: dc \op, \kaddr + add \kaddr, \kaddr, \tmp1 + cmp \kaddr, \size + b.lo 1b + dsb \domain + .endm
Minor comment, but can you stick this in proc-macros.S and change that label from 1: to something like 9998 please?
Other than that, looks good. I can take the next version for 4.5.
Cheers,
Will