arm64: mm: Optimise tlb flush logic where we have >4K granule

The tlb maintainence functions: __cpu_flush_user_tlb_range and __cpu_flush_kern_tlb_range do not take into consideration the page granule when looping through the address range, and repeatedly flush tlb entries for the same page when operating with 64K pages.

This patch re-works the logic s.t. we instead advance the loop by
1 << (PAGE_SHIFT - 12), so avoid repeating ourselves.

Also the routines have been converted from assembler to static inline functions to aid with legibility and potential compiler optimisations.

The isb() has been removed from flush_tlb_kernel_range(.) as it is only needed when changing the execute permission of a mapping. If one needs to set an area of the kernel as execute/non-execute an isb() must be inserted after the call to flush_tlb_kernel_range.

---
Changed in V3: isb() taken out of flush_tlb_kernel_range, as it is only really needed when changing execute permissions. This has been documented in the commit log.

---
 arch/arm64/include/asm/tlbflush.h | 30 ++++++++++++++---
 arch/arm64/mm/Makefile            |  2 +-
 arch/arm64/mm/tlb.S               | 71 ---------------------------------------
 3 files changed, 26 insertions(+), 77 deletions(-)
 delete mode 100644 arch/arm64/mm/tlb.S

index 8b48203..7881d7d 100644
@@ -98,11 +98,31 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
 	dsb();
 }
 
-/*
- * Convert calls to our calling convention.
- */
-#define flush_tlb_range(vma,start,end)	__cpu_flush_user_tlb_range(start,end,vma)
-#define flush_tlb_kernel_range(s,e)	__cpu_flush_kern_tlb_range(s,e)
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+					unsigned long start, unsigned long end)
+{
+	unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
+	unsigned long addr;
+	start = asid | (start >> 12);
+	end = asid | (end >> 12);
+
+	dsb(ishst);
+	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
+		asm("tlbi vae1is, %0" : : "r"(addr));
+	dsb(ish);
+}
+
+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+	unsigned long addr;
+	start >>= 12;
+	end >>= 12;
+
+	dsb(ishst);
+	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
+		asm("tlbi vaae1is, %0" : : "r"(addr));
+	dsb(ish);
+}
 
 /*
  * On AArch64, the cache coherency is handled via the set_pte_at() function.

index b51d364..3ecb56c 100644
@@ -1,5 +1,5 @@
 obj-y				:= dma-mapping.o extable.o fault.o init.o \
 				   cache.o copypage.o flush.o \
 				   ioremap.o mmap.o pgd.o mmu.o \
-				   context.o tlb.o proc.o
+				   context.o proc.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o

1.8.1.4

Applied. Thanks.

Catalin