Skip to contentSkip to conversation

riscv: mm: Some optimizations for tlb flush

PATCH RESEND Superseded Xu Lu 4 participants 10 messages Sep 1, 2025 – Jul 17 RISC-V

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Xu Lu

author
#0 lore raw

cover riscv: mm: Some optimizations for tlb flush

Some optimizations for tlb flush on RISC-V smp:
1. Apply Svinval in update_mmu_cache() to avoid flushing irrelevant tlb entries.
2. Clear bit of current cpu in mm_cpumask after local_flush_tlb_all_asid() to avoid potential IPIs in the future.

We saw the number of IPI reduced from ~98k to 268 on mmapstress01 benchmark.

Some false positive spacing error happens during patch checking. Thus I CCed maintainers of checkpatch.pl as well.

Xu Lu (2):
riscv: mm: Apply svinval in update_mmu_cache()
riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid

 arch/riscv/include/asm/pgtable.h  | 16 +++++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++
 arch/riscv/mm/tlbflush.c          | 64 ++++++++++++-------------------
 3 files changed, 63 insertions(+), 40 deletions(-)
Signature · 1 line
2.20.1
author
#1 lore raw

PATCH RESEND 1/2 riscv: mm: Apply svinval in update_mmu_cache()

Only flush tlb of the specified mm, and apply svinval if available.

---
 arch/riscv/include/asm/pgtable.h  | 16 +++++++++++++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 23 -----------------------
 3 files changed, 38 insertions(+), 24 deletions(-)
arch/riscv/include/asm/pgtable.h +15 -1
index 91697fbf1f901..165cd02d51629 100644
@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	int i;
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
 	asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1)
 		 : : : : svvptc);
 
+	asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1)
+		 : : : : svinval);
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
@@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
+	return;
+
+svinval:
+	local_sfence_w_inval();
+	for (i = 0; i < nr; i++)
+		local_sinval_vma(address + nr * PAGE_SIZE, asid);
+	local_sfence_inval_ir();
+	return;
 
 svvptc:;
 	/*
arch/riscv/include/asm/tlbflush.h +23 -0
index eed0abc405143..9636d07fe9eed 100644
@@ -15,6 +15,29 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -23
index 8404530ec00f9..962db300a1665 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
@@ -110,11 +92,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
 			      const struct cpumask *cmask,
 			      unsigned long start, unsigned long size,
Signature · 1 line
2.20.1
maintainer
#2 lore raw

Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

Hi,

Quoted 3 lines · Xu Lu

Only flush tlb of the specified mm, and apply svinval if available.

I wound up splitting this into two separate patches, rather than one patch, since there seem to be two orthogonal changes. The first change restricts the sfence.vma to a particular ASID (below). The second change involves the Svinval path, and is sent in a subsequent message. What do you think?

- Paul

From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH 1/2] riscv: mm: flush TLB by ASID in update_mmu_cache_range()

Only flush the TLB of the specified mm (via local_flush_tlb_page_asid)

[[email protected]: split the non-Svinval code in the original into this patch; update description]
---
 arch/riscv/include/asm/pgtable.h  | 7 +++++--
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 10 insertions(+), 7 deletions(-)
arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	unsigned long asid;
+
 	/*
 	 * Svvptc guarantees that the new valid pte will be visible within
 	 * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * Relying on flush_tlb_fix_spurious_fault would suffice, but
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
+	asid = get_mm_asid(vma->vm_mm);
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
 	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -5
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
 			      const struct cpumask *cmask,
 			      unsigned long start, unsigned long size,
Signature · 1 line
2.53.0
author
#3 lore raw

Re: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

Hi Paul,

Yes, it makes more sense. Go ahead please.

Best regards.
Xu Lu

Quoted 99 lines · Paul Walmsley

Hi,

Quoted 3 lines · Xu Lu

Only flush tlb of the specified mm, and apply svinval if available.

I wound up splitting this into two separate patches, rather than one patch, since there seem to be two orthogonal changes. The first change restricts the sfence.vma to a particular ASID (below). The second change involves the Svinval path, and is sent in a subsequent message. What do you think?

- Paul

From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH 1/2] riscv: mm: flush TLB by ASID in update_mmu_cache_range()

Only flush the TLB of the specified mm (via local_flush_tlb_page_asid)

[[email protected]: split the non-Svinval code in the original into this patch; update description]
---
 arch/riscv/include/asm/pgtable.h  | 7 +++++--
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 10 insertions(+), 7 deletions(-)
arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
                struct vm_area_struct *vma, unsigned long address,
                pte_t *ptep, unsigned int nr)
 {
+       unsigned long asid;
+
        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
         */
+       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
-               local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+               local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+       return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
        __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -6
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
        local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-       return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
                              const struct cpumask *cmask,
                              unsigned long start, unsigned long size,
--

2.53.0

maintainer
#4 lore raw

Re: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

Quoted 1 line · Xu Lu

Yes, it makes more sense. Go ahead please.

Great, thanks, I queued the two split patches for v7.3.

- Paul

maintainer
#5 lore raw

Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

In reply to Xu Lu
Quoted 8 lines · Xu Lu

Only flush tlb of the specified mm, and apply svinval if available.

---
 arch/riscv/include/asm/pgtable.h  | 16 +++++++++++++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 23 -----------------------
 3 files changed, 38 insertions(+), 24 deletions(-)

And here's the second patch, separated out from your original, which adds the Svinval path. Thoughts?

- Paul

From: Xu Lu <[email protected]>
Date: Mon, 1 Sep 2025 19:41:40 +0800
Subject: [PATCH] riscv: mm: Use Svinval if present in update_mmu_cache_range()

If the Svinval extension is present, use it in update_mmu_cache_range(), rather than using the standard sfence.vma. Svinval should be faster.

[[email protected]: separate the non-Svinval path into an earlier patch; update to apply and improve the description]
---
 arch/riscv/include/asm/pgtable.h  | 16 ++++++++++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 34 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +16 -0
index 755495a542cc..899d38614bb3 100644
@@ -562,6 +562,17 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 #define pgd_ERROR(e) \
 	pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e))
 
+static inline void __update_mmu_cache_range_svinval(struct vm_area_struct *vma,
+		unsigned long address, unsigned int nr)
+{
+	int i;
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
+	local_sfence_w_inval();
+	for (i = 0; i < nr; i++)
+		local_sinval_vma(address + nr * PAGE_SIZE, asid);
+	local_sfence_inval_ir();
+}
 
 /* Commit new configuration to MMU hardware */
 static inline void update_mmu_cache_range(struct vm_fault *vmf,
@@ -578,6 +589,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		__update_mmu_cache_range_svinval(vma, address, nr);
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
Signature · 1 line
2.53.0
#6 lore raw

Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

In reply to Xu Lu

Hi,

Quoted 32 lines · Xu Lu

Only flush tlb of the specified mm, and apply svinval if available.

---
 arch/riscv/include/asm/pgtable.h  | 16 +++++++++++++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 23 -----------------------
 3 files changed, 38 insertions(+), 24 deletions(-)
arch/riscv/include/asm/pgtable.h +6 -0
index 91697fbf1f901..165cd02d51629 100644
@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	int i;
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
 	asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1)
 		 : : : : svvptc);
 
+	asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1)
+		 : : : : svinval);
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
@@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 522

Quoted 10 lines
arch/riscv/include/asm/pgtable.h +7 -1
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
+	return;
+
+svinval:
+	local_sfence_w_inval();
 
+	for (i = 0; i < nr; i++)
+		local_sinval_vma(address + nr * PAGE_SIZE, asid);

Probably should be

+ for (i = 0; i < nr; i++)
+ local_sinval_vma(address + i * PAGE_SIZE, asid);

instead. My BPI-f3 hangs when entering the initramfs with the original patch. Reverting also fixes it.

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
+	local_sfence_inval_ir();
+	return;
 
 svvptc:;

/*

Regards,
Klara Modin

...

author
#7 lore raw

Re: [External] Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

Hi Klara,

Nice catch! Thanks for your review. I will fix it and resend the patch series later.

Best regards,
Xu Lu

#8 lore raw

Re: [PATCH RESEND 1/2] riscv: mm: Apply svinval in update_mmu_cache()

In reply to Klara Modin
Quoted 48 lines · Klara Modin

Hi,

Quoted 32 lines · Xu Lu

Only flush tlb of the specified mm, and apply svinval if available.

---
 arch/riscv/include/asm/pgtable.h  | 16 +++++++++++++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 23 -----------------------
 3 files changed, 38 insertions(+), 24 deletions(-)
arch/riscv/include/asm/pgtable.h +6 -0
index 91697fbf1f901..165cd02d51629 100644
@@ -495,9 +495,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	int i;
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
 	asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1)
 		 : : : : svvptc);
 
+	asm goto(ALTERNATIVE("nop", "j %l[svinval]", 0, RISCV_ISA_EXT_SVINVAL, 1)
+		 : : : : svinval);
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
@@ -506,7 +512,15 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 522

Quoted 10 lines
arch/riscv/include/asm/pgtable.h +7 -1
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
+	return;
+
+svinval:
+	local_sfence_w_inval();
 
+	for (i = 0; i < nr; i++)
+		local_sinval_vma(address + nr * PAGE_SIZE, asid);

Probably should be

Quoted 5 lines

+ for (i = 0; i < nr; i++)
+ local_sinval_vma(address + i * PAGE_SIZE, asid);

instead. My BPI-f3 hangs when entering the initramfs with the original patch. Reverting also fixes it.

I'm also noticing this. linux-next is unable to enter userspace due to this on Spacemit K1. Hopefully we can get a fix merged in soon.

Regards
Anirudh Srinivasan

Quoted 11 lines
Quoted 6 lines
arch/riscv/include/asm/pgtable.h +2 -0
+	local_sfence_inval_ir();
+	return;
 
 svvptc:;

/*

Regards,
Klara Modin

...

author
#9 lore raw

PATCH RESEND 2/2 riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid

In reply to Xu Lu

Clear corresponding bit of current cpu in mm_cpumask after executing local_flush_tlb_all_asid().

This reduces the number of IPI due to tlb flush:

* ltp - mmapstress01
Before: ~98k
After: 268

---
 arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)
arch/riscv/mm/tlbflush.c +25 -16
index 962db300a1665..571358f385879 100644
@@ -17,7 +17,8 @@
  */
 unsigned long tlb_flush_all_threshold __read_mostly = 64;
 
-static void local_flush_tlb_range_threshold_asid(unsigned long start,
+static void local_flush_tlb_range_threshold_asid(struct mm_struct *mm,
+						 unsigned long start,
 						 unsigned long size,
 						 unsigned long stride,
 						 unsigned long asid)
@@ -27,6 +28,8 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,
 
 	if (nr_ptes_in_range > tlb_flush_all_threshold) {
 		local_flush_tlb_all_asid(asid);
+		if (mm && mm != current->active_mm)
+			cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
 		return;
 	}
 
@@ -46,21 +49,28 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,
 	}
 }
 
-static inline void local_flush_tlb_range_asid(unsigned long start,
-		unsigned long size, unsigned long stride, unsigned long asid)
+static inline void local_flush_tlb_range_mm(struct mm_struct *mm,
+					    unsigned long start,
+					    unsigned long size,
+					    unsigned long stride)
 {
-	if (size <= stride)
+	unsigned long asid = get_mm_asid(mm);
+
+	if (size <= stride) {
 		local_flush_tlb_page_asid(start, asid);
-	else if (size == FLUSH_TLB_MAX_SIZE)
+	} else if (size == FLUSH_TLB_MAX_SIZE) {
 		local_flush_tlb_all_asid(asid);
-	else
-		local_flush_tlb_range_threshold_asid(start, size, stride, asid);
+		if (mm && mm != current->active_mm)
+			cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
+	} else {
+		local_flush_tlb_range_threshold_asid(mm, start, size, stride, asid);
+	}
 }
 
 /* Flush a range of kernel pages without broadcasting */
 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
-	local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
+	local_flush_tlb_range_mm(NULL, start, end - start, PAGE_SIZE);
 }
 
 static void __ipi_flush_tlb_all(void *info)
@@ -79,17 +89,17 @@ void flush_tlb_all(void)
 }
 
 struct flush_tlb_range_data {
-	unsigned long asid;
+	struct mm_struct *mm;
 	unsigned long start;
 	unsigned long size;
 	unsigned long stride;
 };
 
-static void __ipi_flush_tlb_range_asid(void *info)
+static void __ipi_flush_tlb_range_mm(void *info)
 {
 	struct flush_tlb_range_data *d = info;
 
-	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
+	local_flush_tlb_range_mm(d->mm, d->start, d->size, d->stride);
 }
 
 static void __flush_tlb_range(struct mm_struct *mm,
@@ -97,7 +107,6 @@ static void __flush_tlb_range(struct mm_struct *mm,
 			      unsigned long start, unsigned long size,
 			      unsigned long stride)
 {
-	unsigned long asid = get_mm_asid(mm);
 	unsigned int cpu;
 
 	if (cpumask_empty(cmask))
@@ -107,17 +116,17 @@ static void __flush_tlb_range(struct mm_struct *mm,
 
 	/* Check if the TLB flush needs to be sent to other CPUs. */
 	if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) {
-		local_flush_tlb_range_asid(start, size, stride, asid);
+		local_flush_tlb_range_mm(mm, start, size, stride);
 	} else if (riscv_use_sbi_for_rfence()) {
-		sbi_remote_sfence_vma_asid(cmask, start, size, asid);
+		sbi_remote_sfence_vma_asid(cmask, start, size, get_mm_asid(mm));
 	} else {
 		struct flush_tlb_range_data ftd;
 
-		ftd.asid = asid;
+		ftd.mm = mm;
 		ftd.start = start;
 		ftd.size = size;
 		ftd.stride = stride;
-		on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1);
+		on_each_cpu_mask(cmask, __ipi_flush_tlb_range_mm, &ftd, 1);
 	}
 
 	put_cpu();
Signature · 1 line
2.20.1