riscv: Fixup asid_allocator remaining issues
Subsystems (2)
v5 cover riscv: Fixup asid_allocator remaining issues
From: Guo Ren <[email protected]>
The patchset fixes the remaining problems of asid_allocator.
- Fixup _PAGE_GLOBAL for kernel virtual address mapping
- Optimize tlb_flush with asid & range
Changes since v4:
- Fixup double PAGE_SIZE add in local_flush_tlb_range_asid
- Add tlbflush: Optimize coding convention
- Optimize comment
Changes since v3:
- Optimize coding convention for
"riscv: Use use_asid_allocator flush TLB"
Changes since v2:
- Remove PAGE_UP/DOWN usage in tlbflush.h
- Optimize variable name
Changes since v1:
- Drop PAGE_UP wrong fixup
- Rebase on clean linux-5.13-rc2
- Add Reviewed-by
Guo Ren (3):
riscv: Use global mappings for kernel pages
riscv: Add ASID-based tlbflushing methods
riscv: tlbflush: Optimize coding convention
arch/riscv/include/asm/mmu_context.h | 2 ++ arch/riscv/include/asm/pgtable.h | 3 +- arch/riscv/include/asm/tlbflush.h | 22 ++++++++++++++ arch/riscv/mm/context.c | 2 +- arch/riscv/mm/tlbflush.c | 57 ++++++++++++++++++++++++++++-------- 5 files changed, 71 insertions(+), 15 deletions(-)
Signature · 1 line
2.7.4
PATCH v5 1/3 riscv: Use global mappings for kernel pages
From: Guo Ren <[email protected]>
We map kernel pages into all addresses space, so they can be marked as global. This allows hardware to avoid flushing the kernel mappings when moving between address spaces.
--- arch/riscv/include/asm/pgtable.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/riscv/include/asm/pgtable.h +2 -1
@@ -134,7 +134,8 @@ | _PAGE_WRITE \ | _PAGE_PRESENT \ | _PAGE_ACCESSED \ - | _PAGE_DIRTY) + | _PAGE_DIRTY \ + | _PAGE_GLOBAL)#define PAGE_KERNEL __pgprot(_PAGE_KERNEL)#define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE)
Signature · 1 line
2.7.4
PATCH v5 2/3 riscv: Add ASID-based tlbflushing methods
In reply to [email protected]From: Guo Ren <[email protected]>
Implement optimized version of the tlb flushing routines for systems using ASIDs. These are behind the use_asid_allocator static branch to not affect existing systems not using ASIDs.
--- arch/riscv/include/asm/mmu_context.h | 2 ++ arch/riscv/include/asm/tlbflush.h | 22 +++++++++++++++++ arch/riscv/mm/context.c | 2 +- arch/riscv/mm/tlbflush.c | 46 +++++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 4 deletions(-)
arch/riscv/include/asm/mmu_context.h +2 -0
@@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk, return 0; } +DECLARE_STATIC_KEY_FALSE (use_asid_allocator); +#include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */
arch/riscv/include/asm/tlbflush.h +22 -0
@@ -22,9 +22,31 @@ static inline void local_flush_tlb_page(unsigned long addr) {ALT_FLUSH_TLB_PAGE (__asm__ __volatile__ ("sfence.vma %0" : :"r" (addr) :"memory" )); } + +static inline void local_flush_tlb_all_asid (unsigned long asid) +{ + __asm__ __volatile__ ("sfence.vma x0, %0" + : + :"r" (asid) + :"memory" ); +} + +static inline void local_flush_tlb_range_asid (unsigned long start, +unsigned long size,unsigned long asid) +{ +unsigned long tmp, end =ALIGN (start + size,PAGE_SIZE ); + +for (tmp = start &PAGE_MASK ; tmp < end; tmp +=PAGE_SIZE ) { + __asm__ __volatile__ ("sfence.vma %0, %1" + : + :"r" (tmp),"r" (asid) + :"memory" ); + } +}#else /* CONFIG_MMU */ #define local_flush_tlb_all () do { } while (0)#define local_flush_tlb_page (addr) do { } while (0) +#define local_flush_tlb_range_asid (addr) do { } while (0)#endif /* CONFIG_MMU */ #if defined(CONFIG_SMP ) && defined(CONFIG_MMU )
arch/riscv/mm/context.c +1 -1
@@ -18,7 +18,7 @@#ifdef CONFIG_MMU -static DEFINE_STATIC_KEY_FALSE (use_asid_allocator); +DEFINE_STATIC_KEY_FALSE (use_asid_allocator);static unsigned long asid_bits;static unsigned long num_asids;
arch/riscv/mm/tlbflush.c +43 -3
67 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods
Quoted 5 lines · On Sun, May 30, 2021 at 04:49:25PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Implement optimized version of the tlb flushing routines for systems using ASIDs. These are behind the
use_asid_allocatorstatic branch to not affect existing systems not using ASIDs.
I still think the code duplication and exposing of new code in a global header here is a bad idea and would suggest the version I sent instead.
Re: [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods
Quoted 10 lines · Christoph Hellwig
Quoted 5 lines · On Sun, May 30, 2021 at 04:49:25PM +0000, [email protected] wrote:
From: Guo Ren <[email protected]>
Implement optimized version of the tlb flushing routines for systems using ASIDs. These are behind the
use_asid_allocatorstatic branch to not affect existing systems not using ASIDs.I still think the code duplication and exposing of new code in a global header here is a bad idea and would suggest the version I sent instead.
Your idea is in the third patch, and I also add you with Co-developed-by. Please have a look:
https://lore.kernel.org/linux-riscv/[email protected]/T/#u
[PATCH V5 3/3] riscv: tlbflush: Optimize coding convention
--
Best Regards
Guo Ren
PATCH v5 3/3 riscv: tlbflush: Optimize coding convention
In reply to [email protected]From: Guo Ren <[email protected]>
Passing the mm_struct as the first argument, as we can derive both the cpumask and asid from it instead of doing that in the callers.
But more importantly, the static branch check can be moved deeper into the code to avoid a lot of duplication.
Also add FIXME comment on the non-ASID code switches to a global flush once flushing more than a single page.
--- arch/riscv/mm/tlbflush.c | 91 ++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 50 deletions(-)
arch/riscv/mm/tlbflush.c +41 -50
@@ -12,56 +12,59 @@ void flush_tlb_all(void) }/* -* This function must not be called with cmask being null. +* This function must not be called with mm_cpumask(mm) being null. * Kernel may panic if cmask is NULL. */ -static void __sbi_tlb_flush_range (struct cpumask *cmask,unsigned long start, +static void __sbi_tlb_flush_range (struct mm_struct *mm, +unsigned long start,unsigned long size) { +struct cpumask *cmask =mm_cpumask (mm);struct cpumask hmask;unsigned int cpuid; +bool local;if (cpumask_empty (cmask))return ; cpuid =get_cpu (); -if (cpumask_any_but (cmask, cpuid) >= nr_cpu_ids) { -/* local cpu is the only cpu present in cpumask */ -if (size <=PAGE_SIZE ) -local_flush_tlb_page (start); -else -local_flush_tlb_all (); - }else { -riscv_cpuid_to_hartid_mask (cmask, &hmask); -sbi_remote_sfence_vma (cpumask_bits (&hmask), start, size); - } +/* +* check if the tlbflush needs to be sent to other CPUs, local +* cpu is the only cpu present in cpumask. +*/ + local = !(cpumask_any_but (cmask, cpuid) < nr_cpu_ids); -put_cpu (); -} - -static void __sbi_tlb_flush_range_asid (struct cpumask *cmask, -unsigned long start, -unsigned long size, -unsigned long asid) -{ -struct cpumask hmask; -unsigned int cpuid; - -if (cpumask_empty (cmask)) -return ; - - cpuid =get_cpu (); +if (static_branch_likely (&use_asid_allocator)) { +unsigned long asid =atomic_long_read (&mm->context.id); -if (cpumask_any_but (cmask, cpuid) >= nr_cpu_ids) { -if (size ==-1 ) -local_flush_tlb_all_asid (asid); -else -local_flush_tlb_range_asid (start, size, asid); +if (likely (local)) { +if (size ==-1 ) +local_flush_tlb_all_asid (asid); +else +local_flush_tlb_range_asid (start, size, asid); + }else { +riscv_cpuid_to_hartid_mask (cmask, &hmask); +sbi_remote_sfence_vma_asid (cpumask_bits (&hmask), + start, size, asid); + } }else { -riscv_cpuid_to_hartid_mask (cmask, &hmask); -sbi_remote_sfence_vma_asid (cpumask_bits (&hmask), - start, size, asid); +if (likely (local)) { +/* +* FIXME: The non-ASID code switches to a global flush +* once flushing more than a single page. It's made by +* commit 6efb16b1d551 (RISC-V: Issue a tlb page flush +* if possible). +*/ +if (size <=PAGE_SIZE ) +local_flush_tlb_page (start); +else +local_flush_tlb_all (); + }else { +riscv_cpuid_to_hartid_mask (cmask, &hmask); +sbi_remote_sfence_vma (cpumask_bits (&hmask), + start, size); + } }put_cpu ();
@@ -69,28 +72,16 @@ static void __sbi_tlb_flush_range_asid(struct cpumask *cmask,void flush_tlb_mm (struct mm_struct *mm) { -if (static_branch_unlikely (&use_asid_allocator)) -__sbi_tlb_flush_range_asid (mm_cpumask (mm),0 ,-1 , -atomic_long_read (&mm->context.id)); -else -__sbi_tlb_flush_range (mm_cpumask (mm),0 ,-1 ); +__sbi_tlb_flush_range (mm,0 ,-1 ); }void flush_tlb_page (struct vm_area_struct *vma,unsigned long addr) { -if (static_branch_unlikely (&use_asid_allocator)) -__sbi_tlb_flush_range_asid (mm_cpumask (vma->vm_mm), addr,PAGE_SIZE , -atomic_long_read (&vma->vm_mm->context.id)); -else -__sbi_tlb_flush_range (mm_cpumask (vma->vm_mm), addr,PAGE_SIZE ); +__sbi_tlb_flush_range (vma->vm_mm, addr,PAGE_SIZE ); }void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end) { -if (static_branch_unlikely (&use_asid_allocator)) -__sbi_tlb_flush_range_asid (mm_cpumask (vma->vm_mm), start, end - start, -atomic_long_read (&vma->vm_mm->context.id)); -else -__sbi_tlb_flush_range (mm_cpumask (vma->vm_mm), start, end - start); +__sbi_tlb_flush_range (vma->vm_mm, start, end - start); }
Signature · 1 line
2.7.4