Skip to contentSkip to conversation

riscv: mm: Some optimizations for tlb flush

PATCH In review Xu Lu 4 participants 13 messages Jul 15 – Jul 17 RISC-V

lore
Share & tools
lore

Xu Lu

author
#0 lore raw

v2 cover riscv: mm: Some optimizations for tlb flush

Some optimizations for tlb flush on RISC-V smp:
1. Apply Svinval in update_mmu_cache() to avoid flushing irrelevant tlb entries.
2. Clear bit of current cpu in mm_cpumask after local_flush_tlb_all_asid() to avoid potential IPIs in the future.

Some false positive spacing error happens during patch checking. Thus I CCed maintainers of checkpatch.pl as well.

Changes in V2:
1. Split the modification on update_mmu_cache() into two commits. Thanks to Paul.
2. Fix address errors in the svinval processing path in V1. Thanks to Klara.

Xu Lu (3):
riscv: mm: Use ASID in update_mmu_cache()
riscv: mm: Apply Svinval in update_mmu_cache()
riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid

 arch/riscv/include/asm/pgtable.h  | 12 +++++-
 arch/riscv/include/asm/tlbflush.h | 23 +++++++++++
 arch/riscv/mm/tlbflush.c          | 64 ++++++++++++-------------------
 3 files changed, 59 insertions(+), 40 deletions(-)
Signature · 1 line
2.39.5
author
#1 lore raw

PATCH v2 1/3 riscv: mm: Use ASID in update_mmu_cache()

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +3 -1
index 5d5756bda82e..9926556099ae 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+
 	/*
 	 * Svvptc guarantees that the new valid pte will be visible within
 	 * a bounded timeframe, so when the uarch does not cache invalid
@@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 
 }
 #define update_mmu_cache(vma, addr, ptep) \
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -5
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
 			      const struct cpumask *cmask,
 			      unsigned long start, unsigned long size,
Signature · 1 line
2.39.5
#2 lore raw

Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()

Hi,

Quoted 14 lines · Xu Lu

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 5d5756bda82e..9926556099ae 100644

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	unsigned long asid = get_mm_asid(vma->vm_mm);
+

It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:

arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	unsigned long asid;
+
 	/*
 	 * Svvptc guarantees that the new valid pte will be visible within
 	 * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * Relying on flush_tlb_fix_spurious_fault would suffice, but
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
+	asid = get_mm_asid(vma->vm_mm);
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
 	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 

This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.

Regards,
Klara Modin

Quoted 47 lines
arch/riscv/include/asm/pgtable.h +1 -1
 	/*
 	 * Svvptc guarantees that the new valid pte will be visible within
 	 * a bounded timeframe, so when the uarch does not cache invalid
@@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 
 }
 #define update_mmu_cache(vma, addr, ptep) \
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -5
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
 			      const struct cpumask *cmask,
 			      unsigned long start, unsigned long size,
Signature · 1 line
2.39.5
author
#3 lore raw

Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()

Hi Paul,

Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?

Best regards,
Xu Lu

Quoted 111 lines · Klara Modin

Hi,

Quoted 14 lines · Xu Lu

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 5d5756bda82e..9926556099ae 100644

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
              struct vm_area_struct *vma, unsigned long address,
              pte_t *ptep, unsigned int nr)
 {
+     unsigned long asid = get_mm_asid(vma->vm_mm);
+

It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:

arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
                struct vm_area_struct *vma, unsigned long address,
                pte_t *ptep, unsigned int nr)
 {
+       unsigned long asid;
+
        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
         */
+       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
-               local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+               local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 

This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.

Regards,
Klara Modin

Quoted 47 lines
arch/riscv/include/asm/pgtable.h +1 -1
      /*
       * Svvptc guarantees that the new valid pte will be visible within
       * a bounded timeframe, so when the uarch does not cache invalid
@@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
       * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
       */
      while (nr--)
-             local_flush_tlb_page(address + nr * PAGE_SIZE);
+             local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 
 }
 #define update_mmu_cache(vma, addr, ptep) \
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+     return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
      __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -6
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
      local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-     return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
                            const struct cpumask *cmask,
                            unsigned long start, unsigned long size,
--

2.39.5

author
#4 lore raw

Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()

Hi Paul,

A gentle ping.

I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?

Best regards,
Xu Lu

Quoted 122 lines · Xu Lu

Hi Paul,

Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?

Best regards,
Xu Lu

Quoted 111 lines · Klara Modin

Hi,

Quoted 14 lines · Xu Lu

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 5d5756bda82e..9926556099ae 100644

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
              struct vm_area_struct *vma, unsigned long address,
              pte_t *ptep, unsigned int nr)
 {
+     unsigned long asid = get_mm_asid(vma->vm_mm);
+

It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:

arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
                struct vm_area_struct *vma, unsigned long address,
                pte_t *ptep, unsigned int nr)
 {
+       unsigned long asid;
+
        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
         */
+       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
-               local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+               local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 

This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.

Regards,
Klara Modin

Quoted 47 lines
arch/riscv/include/asm/pgtable.h +1 -1
      /*
       * Svvptc guarantees that the new valid pte will be visible within
       * a bounded timeframe, so when the uarch does not cache invalid
@@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
       * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
       */
      while (nr--)
-             local_flush_tlb_page(address + nr * PAGE_SIZE);
+             local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 
 }
 #define update_mmu_cache(vma, addr, ptep) \
arch/riscv/include/asm/tlbflush.h +5 -0
index eed0abc40514..7c2cd5cc92d3 100644
@@ -15,6 +15,11 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+     return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
 static inline void local_flush_tlb_all(void)
 {
      __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -6
index 8404530ec00f..73c226f719c7 100644
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
      local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-     return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
                            const struct cpumask *cmask,
                            unsigned long start, unsigned long size,
--

2.39.5

#5 lore raw

Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()

Quoted 7 lines · Xu Lu

Hi Paul,

A gentle ping.

I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?

Paul has updated the for-next branch at git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git so it should sort itself out to either today's or monday's next version (depending on if the tree has already been pulled or not). Even if it is not pulled until monday I think it will be fine, this is what linux-next is for after all.

Regards,
Klara Modin

Quoted 79 lines

Best regards,
Xu Lu

Quoted 74 lines · Xu Lu

Hi Paul,

Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?

Best regards,
Xu Lu

Quoted 63 lines · Klara Modin

Hi,

Quoted 14 lines · Xu Lu

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 5d5756bda82e..9926556099ae 100644

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
              struct vm_area_struct *vma, unsigned long address,
              pte_t *ptep, unsigned int nr)
 {
+     unsigned long asid = get_mm_asid(vma->vm_mm);
+

It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:

arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
                struct vm_area_struct *vma, unsigned long address,
                pte_t *ptep, unsigned int nr)
 {
+       unsigned long asid;
+
        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
         */
+       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
-               local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+               local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 

This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.

Regards,
Klara Modin

...

author
#6 lore raw

Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()

Thanks, Klara. Thanks, Paul.

Best regards,
Xu Lu

Quoted 101 lines · Klara Modin
Quoted 7 lines · Xu Lu

Hi Paul,

A gentle ping.

I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?

Paul has updated the for-next branch at git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git so it should sort itself out to either today's or monday's next version (depending on if the tree has already been pulled or not). Even if it is not pulled until monday I think it will be fine, this is what linux-next is for after all.

Regards,
Klara Modin

Quoted 79 lines

Best regards,
Xu Lu

Quoted 74 lines · Xu Lu

Hi Paul,

Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?

Best regards,
Xu Lu

Quoted 63 lines · Klara Modin

Hi,

Quoted 14 lines · Xu Lu

Only flush TLB entries for the specified mm in update_mmu_cache_range().

---
 arch/riscv/include/asm/pgtable.h  | 4 +++-
 arch/riscv/include/asm/tlbflush.h | 5 +++++
 arch/riscv/mm/tlbflush.c          | 5 -----
 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 5d5756bda82e..9926556099ae 100644

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572

Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2 -0
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
              struct vm_area_struct *vma, unsigned long address,
              pte_t *ptep, unsigned int nr)
 {
+     unsigned long asid = get_mm_asid(vma->vm_mm);
+

It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:

arch/riscv/include/asm/pgtable.h +5 -2
index 5d5756bda82e..755495a542cc 100644
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
                struct vm_area_struct *vma, unsigned long address,
                pte_t *ptep, unsigned int nr)
 {
+       unsigned long asid;
+
        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
         */
+       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
-               local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+               local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 

This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.

Regards,
Klara Modin

...

author
#7 lore raw

PATCH v2 2/3 riscv: mm: Apply Svinval in update_mmu_cache()

In reply to Xu Lu

Use Svinval in update_mmu_cache_range() when the extension is available.

---
 arch/riscv/include/asm/pgtable.h  |  8 ++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +8 -0
index 9926556099ae..823805cc465a 100644
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);
+		local_sfence_inval_ir();
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
Signature · 1 line
2.39.5
Tested-by on v2 2/3
#8 lore raw

Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()

Quoted 3 lines · Xu Lu

Use Svinval in update_mmu_cache_range() when the extension is available.

This version works fine for me.

Thanks,

Quoted 86 lines
---
 arch/riscv/include/asm/pgtable.h  |  8 ++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +8 -0
index 9926556099ae..823805cc465a 100644
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);
+		local_sfence_inval_ir();
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
Signature · 1 line
2.39.5
#9 lore raw

Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()

Quoted 5 lines · Klara Modin
Quoted 3 lines · Xu Lu

Use Svinval in update_mmu_cache_range() when the extension is available.

This version works fine for me.

Lucky You! Here BPiF3 gets frozen once userspace has started.

Quoted 17 lines

Thanks,

Quoted 14 lines
---
 arch/riscv/include/asm/pgtable.h  |  8 ++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 9926556099ae..823805cc465a 100644
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
Quoted 4 lines

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 584

Quoted 4 lines
arch/riscv/include/asm/pgtable.h +4 -0
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);

'asid' is not initialized here. The following fixup is needed:

arch/riscv/include/asm/pgtable.h +0 -0
index 3c17ad0786ab..e283649e1dee 100644
@@ -570,32 +570,32 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,

        /*
         * Svvptc guarantees that the new valid pte will be visible within
         * a bounded timeframe, so when the uarch does not cache invalid
         * entries, we don't have to do anything.
         */
        if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return;

+       asid = get_mm_asid(vma->vm_mm);
        if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { local_sfence_w_inval();
                while (nr--)
                        local_sinval_vma(address + nr * PAGE_SIZE, asid); local_sfence_inval_ir();
                return;
        }

        /*
         * The kernel assumes that TLBs don't cache invalid entries, but
         * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
         * cache flush; it is necessary even after writing invalid entries.
         * Relying on flush_tlb_fix_spurious_fault would suffice, but
         * the extra traps reduce performance.  So, eagerly SFENCE.VMA. */
-       asid = get_mm_asid(vma->vm_mm);
        while (nr--)
                local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); }

 #define update_mmu_cache(vma, addr, ptep) \
        update_mmu_cache_range(NULL, vma, addr, ptep, 1)

 #define update_mmu_tlb_range(vma, addr, ptep, nr) \

Quoted 60 lines
Quoted 60 lines
arch/riscv/include/asm/pgtable.h +4 -0
+		local_sfence_inval_ir();
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -16
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
Quoted 8 lines
Quoted 8 lines
arch/riscv/mm/tlbflush.c +0 -2
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
Signature · 1 line
2.39.5

Best regards

Signature · 2 lines
Marek Szyprowski, PhD
Samsung R&D Institute Poland
#10 lore raw

Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()

Quoted 8 lines · Marek Szyprowski
Quoted 5 lines · Klara Modin
Quoted 3 lines · Xu Lu

Use Svinval in update_mmu_cache_range() when the extension is available.

This version works fine for me.

Lucky You! Here BPiF3 gets frozen once userspace has started.

The old version of patch 1 of the series was used when the series was applied for next, whereas I reverted the old series and applied the entirety of the second version when I tested. I commented this on patch 1.

Regards,
Klara Modin

Quoted 140 lines
Quoted 19 lines

Thanks,

Quoted 14 lines
---
 arch/riscv/include/asm/pgtable.h  |  8 ++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 9926556099ae..823805cc465a 100644
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
Quoted 4 lines

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 584

Quoted 4 lines
arch/riscv/include/asm/pgtable.h +4 -0
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);

'asid' is not initialized here. The following fixup is needed:

arch/riscv/include/asm/pgtable.h +1 -1
index 3c17ad0786ab..e283649e1dee 100644
@@ -570,32 +570,32 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
         /*
          * Svvptc guarantees that the new valid pte will be visible within
          * a bounded timeframe, so when the uarch does not cache invalid
          * entries, we don't have to do anything.
          */
         if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
                 return;
 
+       asid = get_mm_asid(vma->vm_mm);
         if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
                 local_sfence_w_inval();
                 while (nr--)
                         local_sinval_vma(address + nr * PAGE_SIZE, asid);
                 local_sfence_inval_ir();
                 return;
         }
 
         /*
          * The kernel assumes that TLBs don't cache invalid entries, but
          * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
          * cache flush; it is necessary even after writing invalid entries.
          * Relying on flush_tlb_fix_spurious_fault would suffice, but
          * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
          */
-       asid = get_mm_asid(vma->vm_mm);
         while (nr--)
                 local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
  }
 
  #define update_mmu_cache(vma, addr, ptep) \
         update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 
  #define update_mmu_tlb_range(vma, addr, ptep, nr) \
Quoted 63 lines
Quoted 63 lines
arch/riscv/include/asm/pgtable.h +4 -0
+		local_sfence_inval_ir();
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -16
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
Quoted 8 lines
Quoted 8 lines
arch/riscv/mm/tlbflush.c +0 -2
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
Signature · 1 line
2.39.5

Best regards

Signature · 2 lines
Marek Szyprowski, PhD
Samsung R&D Institute Poland
#11 lore raw

Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()

In reply to Xu Lu

Hi Xu Lu,

Quoted 15 lines · Xu Lu

Use Svinval in update_mmu_cache_range() when the extension is available.

---
 arch/riscv/include/asm/pgtable.h  |  8 ++++++++
 arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++
 arch/riscv/mm/tlbflush.c          | 18 ------------------
 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +0 -0
index 9926556099ae..823805cc465a 100644
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))

arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 588

Quoted 10 lines
arch/riscv/include/asm/pgtable.h +8 -0
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);
+		local_sfence_inval_ir();
+		return;
+	}
+

This duplicates the logic in local_flush_tlb_range_threshold_asid(). If you exported local_flush_tlb_range_mm(), you could use that here without rearranging any of the other functions, and it also be able to reuse the threshold logic, which is important on platforms that set local_flush_tlb_range_mm = 1 to always do full-address-space flushes.

Regards,
Samuel

Quoted 61 lines
arch/riscv/include/asm/pgtable.h +0 -0
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
index 7c2cd5cc92d3..9636d07fe9ee 100644
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)
 	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
 }
 
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
index 73c226f719c7..962db300a166 100644
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
author
#12 lore raw

PATCH v2 3/3 riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid

In reply to Xu Lu

Clear corresponding bit of current cpu in mm_cpumask after executing local_flush_tlb_all_asid().

---
 arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)
arch/riscv/mm/tlbflush.c +25 -16
index 962db300a166..571358f38587 100644
@@ -17,7 +17,8 @@
  */
 unsigned long tlb_flush_all_threshold __read_mostly = 64;
 
-static void local_flush_tlb_range_threshold_asid(unsigned long start,
+static void local_flush_tlb_range_threshold_asid(struct mm_struct *mm,
+						 unsigned long start,
 						 unsigned long size,
 						 unsigned long stride,
 						 unsigned long asid)
@@ -27,6 +28,8 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,
 
 	if (nr_ptes_in_range > tlb_flush_all_threshold) {
 		local_flush_tlb_all_asid(asid);
+		if (mm && mm != current->active_mm)
+			cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
 		return;
 	}
 
@@ -46,21 +49,28 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,
 	}
 }
 
-static inline void local_flush_tlb_range_asid(unsigned long start,
-		unsigned long size, unsigned long stride, unsigned long asid)
+static inline void local_flush_tlb_range_mm(struct mm_struct *mm,
+					    unsigned long start,
+					    unsigned long size,
+					    unsigned long stride)
 {
-	if (size <= stride)
+	unsigned long asid = get_mm_asid(mm);
+
+	if (size <= stride) {
 		local_flush_tlb_page_asid(start, asid);
-	else if (size == FLUSH_TLB_MAX_SIZE)
+	} else if (size == FLUSH_TLB_MAX_SIZE) {
 		local_flush_tlb_all_asid(asid);
-	else
-		local_flush_tlb_range_threshold_asid(start, size, stride, asid);
+		if (mm && mm != current->active_mm)
+			cpumask_clear_cpu(raw_smp_processor_id(), mm_cpumask(mm));
+	} else {
+		local_flush_tlb_range_threshold_asid(mm, start, size, stride, asid);
+	}
 }
 
 /* Flush a range of kernel pages without broadcasting */
 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
-	local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
+	local_flush_tlb_range_mm(NULL, start, end - start, PAGE_SIZE);
 }
 
 static void __ipi_flush_tlb_all(void *info)
@@ -79,17 +89,17 @@ void flush_tlb_all(void)
 }
 
 struct flush_tlb_range_data {
-	unsigned long asid;
+	struct mm_struct *mm;
 	unsigned long start;
 	unsigned long size;
 	unsigned long stride;
 };
 
-static void __ipi_flush_tlb_range_asid(void *info)
+static void __ipi_flush_tlb_range_mm(void *info)
 {
 	struct flush_tlb_range_data *d = info;
 
-	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
+	local_flush_tlb_range_mm(d->mm, d->start, d->size, d->stride);
 }
 
 static void __flush_tlb_range(struct mm_struct *mm,
@@ -97,7 +107,6 @@ static void __flush_tlb_range(struct mm_struct *mm,
 			      unsigned long start, unsigned long size,
 			      unsigned long stride)
 {
-	unsigned long asid = get_mm_asid(mm);
 	unsigned int cpu;
 
 	if (cpumask_empty(cmask))
@@ -107,17 +116,17 @@ static void __flush_tlb_range(struct mm_struct *mm,
 
 	/* Check if the TLB flush needs to be sent to other CPUs. */
 	if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) {
-		local_flush_tlb_range_asid(start, size, stride, asid);
+		local_flush_tlb_range_mm(mm, start, size, stride);
 	} else if (riscv_use_sbi_for_rfence()) {
-		sbi_remote_sfence_vma_asid(cmask, start, size, asid);
+		sbi_remote_sfence_vma_asid(cmask, start, size, get_mm_asid(mm));
 	} else {
 		struct flush_tlb_range_data ftd;
 
-		ftd.asid = asid;
+		ftd.mm = mm;
 		ftd.start = start;
 		ftd.size = size;
 		ftd.stride = stride;
-		on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1);
+		on_each_cpu_mask(cmask, __ipi_flush_tlb_range_mm, &ftd, 1);
 	}
 
 	put_cpu();
Signature · 1 line
2.39.5