riscv: mm: Some optimizations for tlb flush
cover riscv: mm: Some optimizations for tlb flush
Some optimizations for tlb flush on RISC-V smp:
1. Apply Svinval in update_mmu_cache() to avoid flushing irrelevant tlb entries.
2. Clear bit of current cpu in mm_cpumask after local_flush_tlb_all_asid() to avoid potential IPIs in the future.
We saw the number of IPI reduced from ~98k to 268 on mmapstress01 benchmark.
Some false positive spacing error happens during patch checking. Thus I CCed maintainers of checkpatch.pl as well.
Xu Lu (2):
riscv: mm: Apply svinval in update_mmu_cache()
riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid
arch/riscv/include/asm/pgtable.h | 16 +++++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++ arch/riscv/mm/tlbflush.c | 64 ++++++++++++------------------- 3 files changed, 63 insertions(+), 40 deletions(-)
Signature · 1 line
2.20.1
PATCH RESEND 1/2 riscv: mm: Apply svinval in update_mmu_cache()
Only flush tlb of the specified mm, and apply svinval if available.
--- arch/riscv/include/asm/pgtable.h | 16 +++++++++++++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++ arch/riscv/mm/tlbflush.c | 23 ----------------------- 3 files changed, 38 insertions(+), 24 deletions(-)
arch/riscv/include/asm/pgtable.h +15 -1
@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + int i; +unsigned long asid =get_mm_asid (vma->vm_mm); + asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) : : : : svvptc); + asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1) + : : : : svinval); +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
@@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); +return ; + +svinval: +local_sfence_w_inval (); +for (i =0 ; i < nr; i++) +local_sinval_vma (address + nr *PAGE_SIZE , asid); +local_sfence_inval_ir (); +return ; svvptc:;/*
arch/riscv/include/asm/tlbflush.h +23 -0
@@ -15,6 +15,29 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1) #ifdef CONFIG_MMU +static inline unsigned long get_mm_asid(struct mm_struct *mm) +{ + return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; +} + +static inline void local_sfence_inval_ir(void) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval(void) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma(unsigned long vma, unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -23
@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below.
@@ -110,11 +92,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size,
Signature · 1 line
2.20.1
Paul Walmsley
maintainerRe: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
Hi,
Quoted 3 lines · Xu Lu
Only flush tlb of the specified mm, and apply svinval if available.
Signed-off-by Xu Lu <[email protected]>
I wound up splitting this into two separate patches, rather than one patch, since there seem to be two orthogonal changes. The first change restricts the sfence.vma to a particular ASID (below). The second change involves the Svinval path, and is sent in a subsequent message. What do you think?
- Paul
From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH 1/2] riscv: mm: flush TLB by ASID in update_mmu_cache_range()
Only flush the TLB of the specified mm (via local_flush_tlb_page_asid)
--- arch/riscv/include/asm/pgtable.h | 7 +++++-- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 10 insertions(+), 7 deletions(-)
arch/riscv/include/asm/pgtable.h +5 -2
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)
arch/riscv/include/asm/tlbflush.h +5 -0
@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );
arch/riscv/mm/tlbflush.c +0 -5
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size,
Signature · 1 line
2.53.0
Re: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
Hi Paul,
Yes, it makes more sense. Go ahead please.
Best regards.
Xu Lu
Quoted 99 lines · Paul Walmsley
Hi,
Quoted 3 lines · Xu Lu
Only flush tlb of the specified mm, and apply svinval if available.
Signed-off-by Xu Lu <[email protected]>I wound up splitting this into two separate patches, rather than one patch, since there seem to be two orthogonal changes. The first change restricts the sfence.vma to a particular ASID (below). The second change involves the Svinval path, and is sent in a subsequent message. What do you think?
- Paul
From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH 1/2] riscv: mm: flush TLB by ASID inupdate_mmu_cache_range()Only flush the TLB of the specified mm (via
local_flush_tlb_page_asid)Signed-off-by Xu Lu <[email protected]>[[email protected]: split the non-Svinval code in the original into this patch; update description]Signed-off-by Paul Walmsley <[email protected]>--- arch/riscv/include/asm/pgtable.h | 7 +++++-- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 10 insertions(+), 7 deletions(-)arch/riscv/include/asm/pgtable.h +5
-2@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)arch/riscv/include/asm/tlbflush.h +5
-0@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );arch/riscv/mm/tlbflush.c +0
-6@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size, --2.53.0
Paul Walmsley
maintainerRe: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
Quoted 1 line · Xu Lu
Yes, it makes more sense. Go ahead please.
Great, thanks, I queued the two split patches for v7.3.
- Paul
Paul Walmsley
maintainerRe: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
In reply to Xu LuQuoted 8 lines · Xu Lu
Only flush tlb of the specified mm, and apply svinval if available.
Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 16 +++++++++++++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++ arch/riscv/mm/tlbflush.c | 23 ----------------------- 3 files changed, 38 insertions(+), 24 deletions(-)
And here's the second patch, separated out from your original, which adds the Svinval path. Thoughts?
- Paul
From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH] riscv: mm: Use Svinval if present in update_mmu_cache_range()
If the Svinval extension is present, use it in update_mmu_cache_range(), rather than using the standard sfence.vma. Svinval should be faster.
--- arch/riscv/include/asm/pgtable.h | 16 ++++++++++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 34 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +16 -0
@@ -562,6 +562,17 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)#define pgd_ERROR (e) \ pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e)) +static inline void __update_mmu_cache_range_svinval (struct vm_area_struct *vma, +unsigned long address,unsigned int nr) +{ +int i; +unsigned long asid =get_mm_asid (vma->vm_mm); + +local_sfence_w_inval (); +for (i =0 ; i < nr; i++) +local_sinval_vma (address + nr *PAGE_SIZE , asid); +local_sfence_inval_ir (); +}/* Commit new configuration to MMU hardware */ static inline void update_mmu_cache_range(struct vm_fault *vmf,
@@ -578,6 +589,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))return ; +if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +__update_mmu_cache_range_svinval (vma, address, nr); +return ; + } +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below.
Signature · 1 line
2.53.0
Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
In reply to Xu LuHi,
Quoted 32 lines · Xu Lu
Only flush tlb of the specified mm, and apply svinval if available.
Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 16 +++++++++++++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++ arch/riscv/mm/tlbflush.c | 23 ----------------------- 3 files changed, 38 insertions(+), 24 deletions(-)arch/riscv/include/asm/pgtable.h +6
-0@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + int i; +unsigned long asid =get_mm_asid (vma->vm_mm); + asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) : : : : svvptc); + asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1) + : : : : svinval); +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */
arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 522
Quoted 10 lines
arch/riscv/include/asm/pgtable.h +7
-1while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); +return ; + +svinval: +local_sfence_w_inval (); +for (i =0 ; i < nr; i++) +local_sinval_vma (address + nr *PAGE_SIZE , asid);
Probably should be
+ for (i = 0; i < nr; i++)
+ local_sinval_vma(address + i * PAGE_SIZE, asid);
instead. My BPI-f3 hangs when entering the initramfs with the original patch. Reverting also fixes it.
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0+local_sfence_inval_ir (); +return ; svvptc:;/*
Regards,
Klara Modin
...
Re: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
Hi Klara,
Nice catch! Thanks for your review. I will fix it and resend the patch series later.
Best regards,
Xu Lu
Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()
In reply to Klara ModinQuoted 48 lines · Klara Modin
Hi,
Quoted 32 lines · Xu Lu
Only flush tlb of the specified mm, and apply svinval if available.
Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 16 +++++++++++++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++ arch/riscv/mm/tlbflush.c | 23 ----------------------- 3 files changed, 38 insertions(+), 24 deletions(-)arch/riscv/include/asm/pgtable.h +6
-0@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + int i; +unsigned long asid =get_mm_asid (vma->vm_mm); + asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) : : : : svvptc); + asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1) + : : : : svinval); +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a @@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 522
Quoted 10 lines
arch/riscv/include/asm/pgtable.h +7
-1while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); +return ; + +svinval: +local_sfence_w_inval (); +for (i =0 ; i < nr; i++) +local_sinval_vma (address + nr *PAGE_SIZE , asid);Probably should be
Quoted 5 lines
+ for (i = 0; i < nr; i++)
+local_sinval_vma(address + i * PAGE_SIZE, asid);instead. My BPI-f3 hangs when entering the initramfs with the original patch. Reverting also fixes it.
I'm also noticing this. linux-next is unable to enter userspace due to this on Spacemit K1. Hopefully we can get a fix merged in soon.
Regards
Anirudh Srinivasan
Quoted 11 lines
Quoted 6 lines
arch/riscv/include/asm/pgtable.h +2
-0+local_sfence_inval_ir (); +return ; svvptc:;/*
Regards,
Klara Modin...
PATCH RESEND 2/2 riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid
In reply to Xu LuClear corresponding bit of current cpu in mm_cpumask after executing local_flush_tlb_all_asid().
This reduces the number of IPI due to tlb flush:
* ltp - mmapstress01
Before: ~98k
After: 268
--- arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-)
arch/riscv/mm/tlbflush.c +25 -16
@@ -17,7 +17,8 @@*/ unsigned long tlb_flush_all_threshold__read_mostly =64 ; -static void local_flush_tlb_range_threshold_asid (unsigned long start, +static void local_flush_tlb_range_threshold_asid (struct mm_struct *mm, +unsigned long start,unsigned long size,unsigned long stride,unsigned long asid)
@@ -27,6 +28,8 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,if (nr_ptes_in_range > tlb_flush_all_threshold) {local_flush_tlb_all_asid (asid); +if (mm && mm != current->active_mm) +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm));return ; }
@@ -46,21 +49,28 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start, } } -static inline void local_flush_tlb_range_asid (unsigned long start, -unsigned long size,unsigned long stride,unsigned long asid) +static inline void local_flush_tlb_range_mm (struct mm_struct *mm, +unsigned long start, +unsigned long size, +unsigned long stride) { -if (size <= stride) +unsigned long asid =get_mm_asid (mm); + +if (size <= stride) {local_flush_tlb_page_asid (start, asid); -else if (size ==FLUSH_TLB_MAX_SIZE ) + }else if (size ==FLUSH_TLB_MAX_SIZE ) {local_flush_tlb_all_asid (asid); -else -local_flush_tlb_range_threshold_asid (start, size, stride, asid); +if (mm && mm != current->active_mm) +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm)); + }else { +local_flush_tlb_range_threshold_asid (mm, start, size, stride, asid); + } }/* Flush a range of kernel pages without broadcasting */ void local_flush_tlb_kernel_range (unsigned long start,unsigned long end) { -local_flush_tlb_range_asid (start, end - start,PAGE_SIZE ,FLUSH_TLB_NO_ASID ); +local_flush_tlb_range_mm (NULL , start, end - start,PAGE_SIZE ); }static void __ipi_flush_tlb_all (void *info)
@@ -79,17 +89,17 @@ void flush_tlb_all(void) }struct flush_tlb_range_data { -unsigned long asid; +struct mm_struct *mm;unsigned long start;unsigned long size;unsigned long stride; }; -static void __ipi_flush_tlb_range_asid (void *info) +static void __ipi_flush_tlb_range_mm (void *info) {struct flush_tlb_range_data *d = info; -local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); +local_flush_tlb_range_mm (d->mm, d->start, d->size, d->stride); } static void __flush_tlb_range(struct mm_struct *mm,
@@ -97,7 +107,6 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long size, unsigned long stride) { - unsigned long asid = get_mm_asid(mm); unsigned int cpu;if (cpumask_empty (cmask))
@@ -107,17 +116,17 @@ static void __flush_tlb_range(struct mm_struct *mm,/* Check if the TLB flush needs to be sent to other CPUs. */ if (cpumask_any_but (cmask, cpu) >= nr_cpu_ids) { -local_flush_tlb_range_asid (start, size, stride, asid); +local_flush_tlb_range_mm (mm, start, size, stride); }else if (riscv_use_sbi_for_rfence ()) { -sbi_remote_sfence_vma_asid (cmask, start, size, asid); +sbi_remote_sfence_vma_asid (cmask, start, size,get_mm_asid (mm)); }else {struct flush_tlb_range_data ftd; - ftd.asid = asid; + ftd.mm = mm; ftd.start = start; ftd.size = size; ftd.stride = stride; -on_each_cpu_mask (cmask, __ipi_flush_tlb_range_asid, &ftd,1 ); +on_each_cpu_mask (cmask, __ipi_flush_tlb_range_mm, &ftd,1 ); }put_cpu ();
Signature · 1 line
2.20.1