ARC MMU code updates
cover ARC MMU code updates
Hi,
This is set of patches almost lost in one of my older branches. I decided to clean them and post given the work on newer MMU.
Thx,
-Vineet
Vineet Gupta (6):
ARCv2: mm: TLB Miss optim: SMP builds can cache pgd pointer in mmu
scratch reg
ARCv2: mm: TLB Miss optim: Use double world load/stores LDD/STD
ARC: mm: TLB Miss optim: avoid re-reading ECR
ARC: mm: tlb flush optim: Make TLBWriteNI fallback to TLBWrite if not
available
ARC: mm: tlb flush optim: elide repeated uTLB invalidate in loop
ARC: mm: tlb flush optim: elide redundant uTLB invalidates for MMUv3
arch/arc/include/asm/entry-compact.h | 4 +- arch/arc/include/asm/mmu.h | 6 +++ arch/arc/include/asm/mmu_context.h | 2 +- arch/arc/include/asm/pgtable.h | 2 +- arch/arc/mm/tlb.c | 81 +++++++++++----------------- arch/arc/mm/tlbex.S | 18 ++++--- 6 files changed, 51 insertions(+), 62 deletions(-)
Signature · 1 line
2.20.1
PATCH 1/6 ARCv2: mm: TLB Miss optim: SMP builds can cache pgd pointer in mmu scratch reg
ARC700 exception (and intr handling) didn't have auto stack switching thus had to rely on stashing a reg temporarily (to free it up) at a known place in memory, allowing to code up the low level stack switching. This however was not re-entrant in SMP which thus had to repurpose the per-cpu MMU SCRATCH DATA register otherwise used to "cache" the task pdg pointer (vs. reading it from mm struct)
The newer HS cores do have auto-stack switching and thus even SMP builds can use the MMU SCRATCH reg as originally intended.
This patch fixes the restriction to ARC700 SMP builds only
--- arch/arc/include/asm/entry-compact.h | 4 ++-- arch/arc/include/asm/mmu.h | 4 ++++ arch/arc/include/asm/mmu_context.h | 2 +- arch/arc/include/asm/pgtable.h | 2 +- arch/arc/mm/tlb.c | 2 +- arch/arc/mm/tlbex.S | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-)
arch/arc/include/asm/entry-compact.h +2 -2
@@ -130,7 +130,7 @@* to be saved again on kernel mode stack, as part of pt_regs. *-------------------------------------------------------------*/ .macro PROLOG_FREEUP_REG reg, mem -#ifdef CONFIG_SMP +#ifndef ARC_USE_SCRATCH_REG sr \reg, [ARC_REG_SCRATCH_DATA0] #else st \reg, [\mem]
@@ -138,7 +138,7 @@ .endm .macro PROLOG_RESTORE_REG reg, mem -#ifdef CONFIG_SMP +#ifndef ARC_USE_SCRATCH_REG lr \reg, [ARC_REG_SCRATCH_DATA0] #else ld \reg, [\mem]
arch/arc/include/asm/mmu.h +4 -0
@@ -40,6 +40,10 @@#define ARC_REG_SCRATCH_DATA0 0x46c#endif +#if defined(CONFIG_ISA_ARCV2 ) || !defined(CONFIG_SMP ) +#define ARC_USE_SCRATCH_REG +#endif +/* Bits in MMU PID register */ #define __TLB_ENABLE (1 << 31)#define __PROG_ENABLE (1 << 30)
arch/arc/include/asm/mmu_context.h +1 -1
@@ -144,7 +144,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,*/ cpumask_set_cpu (cpu,mm_cpumask (next)); -#ifndef CONFIG_SMP +#ifdef ARC_USE_SCRATCH_REG /* PGD cached in MMU reg to avoid 3 mem lookups: task->mm->pgd */ write_aux_reg (ARC_REG_SCRATCH_DATA0 , next->pgd);#endif
arch/arc/include/asm/pgtable.h +1 -1
@@ -351,7 +351,7 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,* Thus use this macro only when you are certain that "current" is current * e.g. when dealing with signal frame setup code etc */ -#ifndef CONFIG_SMP +#ifdef ARC_USE_SCRATCH_REG #define pgd_offset_fast (mm, addr) \ ({ \ pgd_t *pgd_base = (pgd_t *) read_aux_reg(ARC_REG_SCRATCH_DATA0); \
arch/arc/mm/tlb.c +1 -1
@@ -868,7 +868,7 @@ void arc_mmu_init(void)write_aux_reg (ARC_REG_PID ,MMU_ENABLE );/* In smp we use this reg for interrupt 1 scratch */ -#ifndef CONFIG_SMP +#ifdef ARC_USE_SCRATCH_REG /* swapper_pg_dir is the pgd for the kernel, used by vmalloc */ write_aux_reg (ARC_REG_SCRATCH_DATA0 , swapper_pg_dir);#endif
arch/arc/mm/tlbex.S +1 -1
@@ -193,7 +193,7 @@ ex_saved_reg1: lr r2, [efa] -#ifndef CONFIG_SMP +#ifdef ARC_USE_SCRATCH_REG lr r1, [ARC_REG_SCRATCH_DATA0] ; current pgd #else GET_CURR_TASK_ON_CPU r1
Signature · 1 line
2.20.1
PATCH 2/6 ARCv2: mm: TLB Miss optim: Use double world load/stores LDD/STD
In reply to Vineet Gupta--- arch/arc/mm/tlbex.S | 10 ++++++++++ 1 file changed, 10 insertions(+)
arch/arc/mm/tlbex.S +10 -0
@@ -122,17 +122,27 @@ ex_saved_reg1: #else /* ARCv2 */ .macro TLBMISS_FREEUP_REGS +#ifdef CONFIG_ARC_HAS_LL64 + std r0, [sp, -16] + std r2, [sp, -8] +#else PUSH r0 PUSH r1 PUSH r2 PUSH r3 +#endif .endm .macro TLBMISS_RESTORE_REGS +#ifdef CONFIG_ARC_HAS_LL64 + ldd r0, [sp, -16] + ldd r2, [sp, -8] +#else POP r3 POP r2 POP r1 POP r0 +#endif .endm #endif
Signature · 1 line
2.20.1
PATCH 3/6 ARC: mm: TLB Miss optim: avoid re-reading ECR
In reply to Vineet GuptaFor setting PTE Dirty bit, reuse the prior test for ST miss.
No need to reload ECR and test for ST cause code as the prev condition code is still valid (uncloberred)
--- arch/arc/mm/tlbex.S | 2 -- 1 file changed, 2 deletions(-)
arch/arc/mm/tlbex.S +0 -2
@@ -380,9 +380,7 @@ ENTRY(EV_TLBMissD) ;---------------------------------------------------------------- ; UPDATE_PTE: Let Linux VM know that page was accessed/dirty - lr r3, [ecr] or r0, r0, _PAGE_ACCESSED ; Accessed bit always - btst_s r3, ECR_C_BIT_DTLB_ST_MISS ; See if it was a Write Access ? or.nz r0, r0, _PAGE_DIRTY ; if Write, set Dirty bit as well st_s r0, [r1] ; Write back PTE
Signature · 1 line
2.20.1
PATCH 3/6 ARC: mm: TLB Miss optim: avoid re-reading ECR
Hi Vineet,
Quoted 22 lines
-----Original Message-----
From: Vineet Gupta <vgupta at synopsys.com>
Sent: Monday, September 16, 2019 2:32 PM
To: linux-snps-arc at lists.infradead.org
Cc: Alexey Brodkin <abrodkin at synopsys.com>; Vineet Gupta <vgupta at synopsys.com>
Subject: [PATCH 3/6] ARC: mm: TLB Miss optim: avoid re-reading ECRFor setting PTE Dirty bit, reuse the prior test for ST miss.
No need to reload ECR and test for ST cause code as the prev condition code is still valid (uncloberred)
Signed-off-by Vineet Gupta <vgupta at synopsys.com>--- arch/arc/mm/tlbex.S | 2 -- 1 file changed, 2 deletions(-)arch/arc/mm/tlbex.S +0
-0
arch/arc/mm/tlbex.S ENTRY(EV_TLBMissD) · line 382
Quoted 3 lines
arch/arc/mm/tlbex.S +0
-0@@ -380,9 +380,7 @@ ENTRY(EV_TLBMissD) ;---------------------------------------------------------------- ; UPDATE_PTE: Let Linux VM know that page was accessed/dirty
I'd suggest to put a BOLD comment here saying that we rely on previously set condition flag so that whoever reads or (even worse) modifies that or previous code keeps in mind that we shouldn't clobber a particular flag.
arch/arc/mm/tlbex.S ENTRY(EV_TLBMissD) · line 385
Quoted 5 lines
arch/arc/mm/tlbex.S +0
-2- lr r3, [ecr] or r0, r0, _PAGE_ACCESSED ; Accessed bit always - btst_s r3, ECR_C_BIT_DTLB_ST_MISS ; See if it was a Write Access ? or.nz r0, r0, _PAGE_DIRTY ; if Write, set Dirty bit as well st_s r0, [r1] ; Write back PTE
-Alexey
PATCH 3/6 ARC: mm: TLB Miss optim: avoid re-reading ECR
Quoted 7 lines · Alexey Brodkin
arch/arc/mm/tlbex.S ENTRY(EV_TLBMissD) · line 382
Quoted 3 lines
arch/arc/mm/tlbex.S +0
-0;---------------------------------------------------------------- ; UPDATE_PTE: Let Linux VM know that page was accessed/dirtyI'd suggest to put a BOLD comment here saying that we rely on previously set condition flag so that whoever reads or (even worse) modifies that or previous code keeps in mind that we shouldn't clobber a particular flag.
The flag setting code is only a few lines prior. It would be messy to annotate for flag checking instruction where the flags are clobbered. This is low level assembly code - not for faint hearted.
PATCH 4/6 ARC: mm: tlb flush optim: Make TLBWriteNI fallback to TLBWrite if not available
In reply to Vineet GuptaTLBWriteNI was introduced in MMUv2 (to not invalidate uTLBs in Fast Path
TLB Refill Handler). To avoid #ifdef'ery make it fallback to TLBWrite availabel on all MMUs. This will also help with next change
--- arch/arc/include/asm/mmu.h | 2 ++ arch/arc/mm/tlbex.S | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-)
arch/arc/include/asm/mmu.h +2 -0
@@ -67,6 +67,8 @@#if (CONFIG_ARC_MMU_VER >=2 )#define TLBWriteNI 0x5/* write JTLB without inv uTLBs */ #define TLBIVUTLB 0x6/* explicitly inv uTLBs */ +#else +#define TLBWriteNI TLBWrite/* Not present in hardware, fallback */ #endif #if (CONFIG_ARC_MMU_VER >=4 )
arch/arc/mm/tlbex.S +0 -4
@@ -292,11 +292,7 @@ ex_saved_reg1: sr TLBGetIndex, [ARC_REG_TLBCOMMAND] /* Commit the Write */ -#if (CONFIG_ARC_MMU_VER >= 2) /* introduced in v2 */ sr TLBWriteNI, [ARC_REG_TLBCOMMAND] -#else - sr TLBWrite, [ARC_REG_TLBCOMMAND] -#endif #else sr TLBInsertEntry, [ARC_REG_TLBCOMMAND]
Signature · 1 line
2.20.1
PATCH 5/6 ARC: mm: tlb flush optim: elide repeated uTLB invalidate in loop
In reply to Vineet GuptaThe unconditional full TLB flush (on say ASID rollover) iterates over each entry and uses TLBWrite to zero it out. TLBWrite by design also invalidates the uTLBs thus we end up invalidating it as many times as numbe rof entries (512 or 1k)
Optimize this by using a weaker TLBWriteNI cmd in loop, which doesn't tinker with uTLBs and an explicit one time IVUTLB, outside the loop to invalidate them all once.
And given the optimiztion, the IVUTLB is now needed on MMUv4 too where the uTLBs and JTLBs are otherwise coherent given the TLBInsertEntry / TLBDeleteEntry commands
--- arch/arc/mm/tlb.c | 74 +++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 45 deletions(-)
arch/arc/mm/tlb.c +29 -45
@@ -118,6 +118,33 @@ static inline void __tlb_entry_erase(void)write_aux_reg (ARC_REG_TLBCOMMAND , TLBWrite); } +static void utlb_invalidate (void ) +{ +#if (CONFIG_ARC_MMU_VER >=2 ) + +#if (CONFIG_ARC_MMU_VER ==2 ) +/* MMU v2 introduced the uTLB Flush command. +* There was however an obscure hardware bug, where uTLB flush would +* fail when a prior probe for J-TLB (both totally unrelated) would +* return lkup err - because the entry didn't exist in MMU. +* The Workround was to set Index reg with some valid value, prior to +* flush. This was fixed in MMU v3 +*/ +unsigned int idx; + +/* make sure INDEX Reg is valid */ + idx =read_aux_reg (ARC_REG_TLBINDEX ); + +/* If not write some dummy val */ +if (unlikely (idx &TLB_LKUP_ERR )) +write_aux_reg (ARC_REG_TLBINDEX ,0xa ); +#endif + +write_aux_reg (ARC_REG_TLBCOMMAND ,TLBIVUTLB ); +#endif + +} +#if (CONFIG_ARC_MMU_VER <4 )static inline unsigned int tlb_entry_lkup (unsigned long vaddr_n_asid)
@@ -149,44 +176,6 @@ static void tlb_entry_erase(unsigned int vaddr_n_asid) } } -/**************************************************************************** -* ARC700 MMU caches recently used J-TLB entries (RAM) as uTLBs (FLOPs) -* -* New IVUTLB cmd in MMU v2 explictly invalidates the uTLB -* -* utlb_invalidate ( ) -* -For v2 MMU calls Flush uTLB Cmd -* -For v1 MMU does nothing (except for Metal Fix v1 MMU) -* This is because in v1 TLBWrite itself invalidate uTLBs -***************************************************************************/ - -static void utlb_invalidate (void ) -{ -#if (CONFIG_ARC_MMU_VER >=2 ) - -#if (CONFIG_ARC_MMU_VER ==2 ) -/* MMU v2 introduced the uTLB Flush command. -* There was however an obscure hardware bug, where uTLB flush would -* fail when a prior probe for J-TLB (both totally unrelated) would -* return lkup err - because the entry didn't exist in MMU. -* The Workround was to set Index reg with some valid value, prior to -* flush. This was fixed in MMU v3 hence not needed any more -*/ -unsigned int idx; - -/* make sure INDEX Reg is valid */ - idx =read_aux_reg (ARC_REG_TLBINDEX ); - -/* If not write some dummy val */ -if (unlikely (idx &TLB_LKUP_ERR )) -write_aux_reg (ARC_REG_TLBINDEX ,0xa ); -#endif - -write_aux_reg (ARC_REG_TLBCOMMAND ,TLBIVUTLB ); -#endif - -} -static void tlb_entry_insert (unsigned int pd0,pte_t pd1) {unsigned int idx;
@@ -219,11 +208,6 @@ static void tlb_entry_insert(unsigned int pd0, pte_t pd1)#else /* CONFIG_ARC_MMU_VER >= 4) */ -static void utlb_invalidate (void ) -{ -/* No need since uTLB is always in sync with JTLB */ -} -static void tlb_entry_erase (unsigned int vaddr_n_asid) {write_aux_reg (ARC_REG_TLBPD0 , vaddr_n_asid | _PAGE_PRESENT);
@@ -267,7 +251,7 @@ noinline void local_flush_tlb_all(void)for (entry =0 ; entry < num_tlb; entry++) {/* write this entry to the TLB */ write_aux_reg (ARC_REG_TLBINDEX , entry); -write_aux_reg (ARC_REG_TLBCOMMAND , TLBWrite); +write_aux_reg (ARC_REG_TLBCOMMAND , TLBWriteNI); }if (IS_ENABLED (CONFIG_TRANSPARENT_HUGEPAGE )) {
@@ -278,7 +262,7 @@ noinline void local_flush_tlb_all(void)for (entry = stlb_idx; entry < stlb_idx +16 ; entry++) {write_aux_reg (ARC_REG_TLBINDEX , entry); -write_aux_reg (ARC_REG_TLBCOMMAND , TLBWrite); +write_aux_reg (ARC_REG_TLBCOMMAND , TLBWriteNI); } }
Signature · 1 line
2.20.1
PATCH 6/6 ARC: mm: tlb flush optim: elide redundant uTLB invalidates for MMUv3
In reply to Vineet GuptaFor MMUv3 (and prior) the flush_tlb_{range,mm,page} API use the MMU TLBWrite cmd which already nukes the entire uTLB, so NO need for additional IVUTLB cmd from utlb_invalidate() - hence this patch
local_flush_tlb_all() is special since it uses a weaker TLBWriteNI cmd (prec commit) to shoot down JTLB, hence we retain the explicit uTLB flush
--- arch/arc/mm/tlb.c | 5 ----- 1 file changed, 5 deletions(-)
arch/arc/mm/tlb.c +0 -5
@@ -339,8 +339,6 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, } } -utlb_invalidate (); -local_irq_restore (flags); }
@@ -369,8 +367,6 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) start +=PAGE_SIZE ; } -utlb_invalidate (); -local_irq_restore (flags); }
@@ -391,7 +387,6 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)if (asid_mm (vma->vm_mm, cpu) !=MM_CTXT_NO_ASID ) {tlb_entry_erase ((page &PAGE_MASK ) |hw_pid (vma->vm_mm, cpu)); -utlb_invalidate (); }local_irq_restore (flags);
Signature · 1 line
2.20.1