riscv: mm: Some optimizations for tlb flush
v2 cover riscv: mm: Some optimizations for tlb flush
Some optimizations for tlb flush on RISC-V smp:
1. Apply Svinval in update_mmu_cache() to avoid flushing irrelevant tlb entries.
2. Clear bit of current cpu in mm_cpumask after local_flush_tlb_all_asid() to avoid potential IPIs in the future.
Some false positive spacing error happens during patch checking. Thus I CCed maintainers of checkpatch.pl as well.
Changes in V2:
1. Split the modification on update_mmu_cache() into two commits. Thanks to Paul.
2. Fix address errors in the svinval processing path in V1. Thanks to Klara.
Xu Lu (3):
riscv: mm: Use ASID in update_mmu_cache()
riscv: mm: Apply Svinval in update_mmu_cache()
riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid
arch/riscv/include/asm/pgtable.h | 12 +++++- arch/riscv/include/asm/tlbflush.h | 23 +++++++++++ arch/riscv/mm/tlbflush.c | 64 ++++++++++++------------------- 3 files changed, 59 insertions(+), 40 deletions(-)
Signature · 1 line
2.39.5
PATCH v2 1/3 riscv: mm: Use ASID in update_mmu_cache()
Only flush TLB entries for the specified mm in update_mmu_cache_range().
--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)
arch/riscv/include/asm/pgtable.h +3 -1
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid
@@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); }#define update_mmu_cache (vma, addr, ptep) \
arch/riscv/include/asm/tlbflush.h +5 -0
@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );
arch/riscv/mm/tlbflush.c +0 -5
@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size,
Signature · 1 line
2.39.5
Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()
Hi,
Quoted 14 lines · Xu Lu
Only flush TLB entries for the specified mm in
update_mmu_cache_range().Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0
arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +
It seems the old version of this patch is still used (at least in next-202650716, 15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:
arch/riscv/include/asm/pgtable.h +5 -2
@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid
@@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)
This means that patch 2 of this series will use the uninitialised asid for local_svinval_vma() and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.
Regards,
Klara Modin
Quoted 47 lines
arch/riscv/include/asm/pgtable.h +1
-1/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); }#define update_mmu_cache (vma, addr, ptep) \arch/riscv/include/asm/tlbflush.h +5
-0@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );arch/riscv/mm/tlbflush.c +0
-5@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size,Signature · 1 line
2.39.5
Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()
Hi Paul,
Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?
Best regards,
Xu Lu
Quoted 111 lines · Klara Modin
Hi,
Quoted 14 lines · Xu Lu
Only flush TLB entries for the specified mm in
update_mmu_cache_range().Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +It seems the old version of this patch is still used (at least in next-202650716,
15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:arch/riscv/include/asm/pgtable.h +5
-2@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)This means that patch 2 of this series will use the uninitialised asid for
local_svinval_vma()and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.Regards,
Klara ModinQuoted 47 lines
arch/riscv/include/asm/pgtable.h +1
-1/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); }#define update_mmu_cache (vma, addr, ptep) \arch/riscv/include/asm/tlbflush.h +5
-0@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );arch/riscv/mm/tlbflush.c +0
-6@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size, --2.39.5
Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()
Hi Paul,
A gentle ping.
I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?
Best regards,
Xu Lu
Quoted 122 lines · Xu Lu
Hi Paul,
Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?
Best regards,
Xu LuQuoted 111 lines · Klara Modin
Hi,
Quoted 14 lines · Xu Lu
Only flush TLB entries for the specified mm in
update_mmu_cache_range().Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +It seems the old version of this patch is still used (at least in next-202650716,
15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:arch/riscv/include/asm/pgtable.h +5
-2@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)This means that patch 2 of this series will use the uninitialised asid for
local_svinval_vma()and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.Regards,
Klara ModinQuoted 47 lines
arch/riscv/include/asm/pgtable.h +1
-1/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -584,7 +586,7 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* the extra traps reduce performance. So, eagerly SFENCE.VMA. */ while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); }#define update_mmu_cache (vma, addr, ptep) \arch/riscv/include/asm/tlbflush.h +5
-0@@ -15,6 +15,11 @@#define FLUSH_TLB_NO_ASID ((unsigned long)-1)#ifdef CONFIG_MMU +static inline unsigned long get_mm_asid (struct mm_struct *mm) +{ +return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; +} +static inline void local_flush_tlb_all (void ) { __asm__ __volatile__ ("sfence.vma" : : :"memory" );arch/riscv/mm/tlbflush.c +0
-6@@ -110,11 +110,6 @@ static void __ipi_flush_tlb_range_asid(void *info)local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); } -static inline unsigned long get_mm_asid (struct mm_struct *mm) -{ -return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; -} - static void __flush_tlb_range(struct mm_struct *mm, const struct cpumask *cmask, unsigned long start, unsigned long size, --2.39.5
Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()
Quoted 7 lines · Xu Lu
Hi Paul,
A gentle ping.
I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?
Paul has updated the for-next branch at git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git so it should sort itself out to either today's or monday's next version (depending on if the tree has already been pulled or not). Even if it is not pulled until monday I think it will be fine, this is what linux-next is for after all.
Regards,
Klara Modin
Quoted 79 lines
Best regards,
Xu LuQuoted 74 lines · Xu Lu
Hi Paul,
Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?
Best regards,
Xu LuQuoted 63 lines · Klara Modin
Hi,
Quoted 14 lines · Xu Lu
Only flush TLB entries for the specified mm in
update_mmu_cache_range().Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +It seems the old version of this patch is still used (at least in next-202650716,
15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:arch/riscv/include/asm/pgtable.h +5
-2@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)This means that patch 2 of this series will use the uninitialised asid for
local_svinval_vma()and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.Regards,
Klara Modin
...
Re: [External] Re: [PATCH v2 1/3] riscv: mm: Use ASID in update_mmu_cache()
Thanks, Klara. Thanks, Paul.
Best regards,
Xu Lu
Quoted 101 lines · Klara Modin
Quoted 7 lines · Xu Lu
Hi Paul,
A gentle ping.
I have no idea what I can do regarding linux-next. Could you please help revert the patch from version 1 and cherry-pick all the patches from version 2 instead? Or what can I do for this?
Paul has updated the for-next branch at git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git so it should sort itself out to either today's or monday's next version (depending on if the tree has already been pulled or not). Even if it is not pulled until monday I think it will be fine, this is what linux-next is for after all.
Regards,
Klara ModinQuoted 79 lines
Best regards,
Xu LuQuoted 74 lines · Xu Lu
Hi Paul,
Klara found that linux-next cherry-picks the new version of patch 2 while picking the old version of patch 1, which leads to an uninitialized asid. Could you please help correct this?
Best regards,
Xu LuQuoted 63 lines · Klara Modin
Hi,
Quoted 14 lines · Xu Lu
Only flush TLB entries for the specified mm in
update_mmu_cache_range().Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 4 +++- arch/riscv/include/asm/tlbflush.h | 5 +++++ arch/riscv/mm/tlbflush.c | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 572
Quoted 5 lines
arch/riscv/include/asm/pgtable.h +2
-0@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid = get_mm_asid(vma->vm_mm); +It seems the old version of this patch is still used (at least in next-202650716,
15ce1d7c4ddfe0dff00fcaee4ccaeef3efbc62c6), which means asid is not initialised here. We instead have:arch/riscv/include/asm/pgtable.h +5
-2@@ -568,6 +568,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { + unsigned long asid; +/* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid @@ -583,10 +585,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,* Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ + asid =get_mm_asid (vma->vm_mm);while (nr--) -local_flush_tlb_page (address + nr *PAGE_SIZE ); - +local_flush_tlb_page_asid (address + nr *PAGE_SIZE , asid); } +#define update_mmu_cache (vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1)This means that patch 2 of this series will use the uninitialised asid for
local_svinval_vma()and I get the same behaviour as before (hang on initramfs). If I revert the two patches of this series which are in next and apply this again it's fine.Regards,
Klara Modin...
PATCH v2 2/3 riscv: mm: Apply Svinval in update_mmu_cache()
In reply to Xu LuUse Svinval in update_mmu_cache_range() when the extension is available.
--- arch/riscv/include/asm/pgtable.h | 8 ++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 26 insertions(+), 18 deletions(-)
arch/riscv/include/asm/pgtable.h +8 -0
@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))return ; +if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +local_sfence_w_inval (); +while (nr--) +local_sinval_vma (address + nr *PAGE_SIZE , asid); +local_sfence_inval_ir (); +return ; + } +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
arch/riscv/include/asm/tlbflush.h +18 -0
@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");
arch/riscv/mm/tlbflush.c +0 -18
@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below.
Signature · 1 line
2.39.5
Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()
Quoted 3 lines · Xu Lu
Use Svinval in
update_mmu_cache_range()when the extension is available.Signed-off-by Xu Lu <[email protected]>
This version works fine for me.
Thanks,
Quoted 86 lines
--- arch/riscv/include/asm/pgtable.h | 8 ++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 26 insertions(+), 18 deletions(-)arch/riscv/include/asm/pgtable.h +8
-0@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))return ; +if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +local_sfence_w_inval (); +while (nr--) +local_sinval_vma (address + nr *PAGE_SIZE , asid); +local_sfence_inval_ir (); +return ; + } +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a arch/riscv/include/asm/tlbflush.h +18
-0@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");arch/riscv/mm/tlbflush.c +0
-18@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. Signature · 1 line
2.39.5
Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()
Quoted 5 lines · Klara Modin
Quoted 3 lines · Xu Lu
Use Svinval in
update_mmu_cache_range()when the extension is available.Signed-off-by Xu Lu <[email protected]>This version works fine for me.
Lucky You! Here BPiF3 gets frozen once userspace has started.
Quoted 17 lines
Thanks,
Tested-by Klara Modin <[email protected]>Quoted 14 lines
--- arch/riscv/include/asm/pgtable.h | 8 ++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 26 insertions(+), 18 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))return ;
Quoted 4 lines
arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 584
Quoted 4 lines
arch/riscv/include/asm/pgtable.h +4
-0+if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +local_sfence_w_inval (); +while (nr--) +local_sinval_vma (address + nr *PAGE_SIZE , asid);
'asid' is not initialized here. The following fixup is needed:
arch/riscv/include/asm/pgtable.h +0 -0
@@ -570,32 +570,32 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
/*
* Svvptc guarantees that the new valid pte will be visible within
* a bounded timeframe, so when the uarch does not cache invalid
* entries, we don't have to do anything.
*/
if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return;
+ asid = get_mm_asid(vma->vm_mm);
if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { local_sfence_w_inval();
while (nr--)
local_sinval_vma(address + nr * PAGE_SIZE, asid); local_sfence_inval_ir();
return;
}
/*
* The kernel assumes that TLBs don't cache invalid entries, but
* in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
* cache flush; it is necessary even after writing invalid entries.
* Relying on flush_tlb_fix_spurious_fault would suffice, but
* the extra traps reduce performance. So, eagerly SFENCE.VMA. */
- asid = get_mm_asid(vma->vm_mm);
while (nr--)
local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); }
#define update_mmu_cache(vma, addr, ptep) \
update_mmu_cache_range(NULL, vma, addr, ptep, 1)
#define update_mmu_tlb_range(vma, addr, ptep, nr) \
Quoted 60 lines
Quoted 60 lines
arch/riscv/include/asm/pgtable.h +4
-0+local_sfence_inval_ir (); +return ; + } +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a arch/riscv/include/asm/tlbflush.h +18
-0@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");arch/riscv/mm/tlbflush.c +0
-16@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
Quoted 8 lines
Quoted 8 lines
arch/riscv/mm/tlbflush.c +0
-2-} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. Signature · 1 line
2.39.5
Best regards
Signature · 2 lines
Marek Szyprowski, PhD Samsung R&D Institute Poland
Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()
Quoted 8 lines · Marek Szyprowski
Quoted 5 lines · Klara Modin
Quoted 3 lines · Xu Lu
Use Svinval in
update_mmu_cache_range()when the extension is available.Signed-off-by Xu Lu <[email protected]>This version works fine for me.
Lucky You! Here BPiF3 gets frozen once userspace has started.
The old version of patch 1 of the series was used when the series was applied for next, whereas I reverted the old series and applied the entirety of the second version when I tested. I commented this on patch 1.
Regards,
Klara Modin
Quoted 140 lines
Quoted 19 lines
Thanks,
Tested-by Klara Modin <[email protected]>Quoted 14 lines
--- arch/riscv/include/asm/pgtable.h | 8 ++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 26 insertions(+), 18 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))return ;Quoted 4 lines
arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 584
Quoted 4 lines
arch/riscv/include/asm/pgtable.h +4
-0+if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +local_sfence_w_inval (); +while (nr--) +local_sinval_vma (address + nr *PAGE_SIZE , asid);'asid' is not initialized here. The following fixup is needed:
arch/riscv/include/asm/pgtable.h +1
-1@@ -570,32 +570,32 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, /* * Svvptc guarantees that the new valid pte will be visible within * a bounded timeframe, so when the uarch does not cache invalid * entries, we don't have to do anything. */ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)) return; + asid = get_mm_asid(vma->vm_mm); if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) { local_sfence_w_inval(); while (nr--) local_sinval_vma(address + nr * PAGE_SIZE, asid); local_sfence_inval_ir(); return; } /* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a * cache flush; it is necessary even after writing invalid entries. * Relying on flush_tlb_fix_spurious_fault would suffice, but * the extra traps reduce performance. So, eagerly SFENCE.VMA. */ - asid = get_mm_asid(vma->vm_mm); while (nr--) local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid); } #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) #define update_mmu_tlb_range(vma, addr, ptep, nr) \Quoted 63 lines
Quoted 63 lines
arch/riscv/include/asm/pgtable.h +4
-0+local_sfence_inval_ir (); +return ; + } +/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a arch/riscv/include/asm/tlbflush.h +18
-0@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");arch/riscv/mm/tlbflush.c +0
-16@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");Quoted 8 lines
Quoted 8 lines
arch/riscv/mm/tlbflush.c +0
-2-} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below. Signature · 1 line
2.39.5Best regards
Signature · 2 lines
Marek Szyprowski, PhD Samsung R&D Institute Poland
Re: [PATCH v2 2/3] riscv: mm: Apply Svinval in update_mmu_cache()
In reply to Xu LuHi Xu Lu,
Quoted 15 lines · Xu Lu
Use Svinval in
update_mmu_cache_range()when the extension is available.Signed-off-by Xu Lu <[email protected]>--- arch/riscv/include/asm/pgtable.h | 8 ++++++++ arch/riscv/include/asm/tlbflush.h | 18 ++++++++++++++++++ arch/riscv/mm/tlbflush.c | 18 ------------------ 3 files changed, 26 insertions(+), 18 deletions(-)arch/riscv/include/asm/pgtable.h +0
-0@@ -578,6 +578,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVVPTC ))
arch/riscv/include/asm/pgtable.h static inline void update_mmu_cache_range(struct vm_fault *vmf, · line 588
Quoted 10 lines
arch/riscv/include/asm/pgtable.h +8
-0return ; +if (riscv_has_extension_unlikely (RISCV_ISA_EXT_SVINVAL )) { +local_sfence_w_inval (); +while (nr--) +local_sinval_vma (address + nr *PAGE_SIZE , asid); +local_sfence_inval_ir (); +return ; + } +
This duplicates the logic in local_flush_tlb_range_threshold_asid(). If you exported local_flush_tlb_range_mm(), you could use that here without rearranging any of the other functions, and it also be able to reuse the threshold logic, which is important on platforms that set local_flush_tlb_range_mm = 1 to always do full-address-space flushes.
Regards,
Samuel
Quoted 61 lines
arch/riscv/include/asm/pgtable.h +0
-0/* * The kernel assumes that TLBs don't cache invalid entries, but * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a arch/riscv/include/asm/tlbflush.h +18
-0@@ -20,6 +20,24 @@ static inline unsigned long get_mm_asid(struct mm_struct *mm)return mm ?cntx2asid (atomic_long_read (&mm->context.id)) :FLUSH_TLB_NO_ASID ; } +static inline void local_sfence_inval_ir (void ) +{ + asm volatile(SFENCE_INVAL_IR() ::: "memory"); +} + +static inline void local_sfence_w_inval (void ) +{ + asm volatile(SFENCE_W_INVAL() ::: "memory"); +} + +static inline void local_sinval_vma (unsigned long vma,unsigned long asid) +{ + if (asid != FLUSH_TLB_NO_ASID) + asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); + else + asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); +} + static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory");arch/riscv/mm/tlbflush.c +0
-18@@ -11,24 +11,6 @@#define has_svinval () riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) -static inline void local_sfence_inval_ir (void ) -{ - asm volatile(SFENCE_INVAL_IR() ::: "memory"); -} - -static inline void local_sfence_w_inval (void ) -{ - asm volatile(SFENCE_W_INVAL() ::: "memory"); -} - -static inline void local_sinval_vma(unsigned long vma, unsigned long asid) -{ - if (asid != FLUSH_TLB_NO_ASID) - asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); - else - asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); -} -/* * Flush entire TLB if number of entries to be flushed is greater * than the threshold below.
PATCH v2 3/3 riscv: mm: Clear cpu in mm_cpumask after local_flush_tlb_all_asid
In reply to Xu LuClear corresponding bit of current cpu in mm_cpumask after executing local_flush_tlb_all_asid().
--- arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-)
arch/riscv/mm/tlbflush.c +25 -16
@@ -17,7 +17,8 @@*/ unsigned long tlb_flush_all_threshold__read_mostly =64 ; -static void local_flush_tlb_range_threshold_asid (unsigned long start, +static void local_flush_tlb_range_threshold_asid (struct mm_struct *mm, +unsigned long start,unsigned long size,unsigned long stride,unsigned long asid)
@@ -27,6 +28,8 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start,if (nr_ptes_in_range > tlb_flush_all_threshold) {local_flush_tlb_all_asid (asid); +if (mm && mm != current->active_mm) +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm));return ; }
@@ -46,21 +49,28 @@ static void local_flush_tlb_range_threshold_asid(unsigned long start, } } -static inline void local_flush_tlb_range_asid (unsigned long start, -unsigned long size,unsigned long stride,unsigned long asid) +static inline void local_flush_tlb_range_mm (struct mm_struct *mm, +unsigned long start, +unsigned long size, +unsigned long stride) { -if (size <= stride) +unsigned long asid =get_mm_asid (mm); + +if (size <= stride) {local_flush_tlb_page_asid (start, asid); -else if (size ==FLUSH_TLB_MAX_SIZE ) + }else if (size ==FLUSH_TLB_MAX_SIZE ) {local_flush_tlb_all_asid (asid); -else -local_flush_tlb_range_threshold_asid (start, size, stride, asid); +if (mm && mm != current->active_mm) +cpumask_clear_cpu (raw_smp_processor_id (),mm_cpumask (mm)); + }else { +local_flush_tlb_range_threshold_asid (mm, start, size, stride, asid); + } }/* Flush a range of kernel pages without broadcasting */ void local_flush_tlb_kernel_range (unsigned long start,unsigned long end) { -local_flush_tlb_range_asid (start, end - start,PAGE_SIZE ,FLUSH_TLB_NO_ASID ); +local_flush_tlb_range_mm (NULL , start, end - start,PAGE_SIZE ); }static void __ipi_flush_tlb_all (void *info)
@@ -79,17 +89,17 @@ void flush_tlb_all(void) }struct flush_tlb_range_data { -unsigned long asid; +struct mm_struct *mm;unsigned long start;unsigned long size;unsigned long stride; }; -static void __ipi_flush_tlb_range_asid (void *info) +static void __ipi_flush_tlb_range_mm (void *info) {struct flush_tlb_range_data *d = info; -local_flush_tlb_range_asid (d->start, d->size, d->stride, d->asid); +local_flush_tlb_range_mm (d->mm, d->start, d->size, d->stride); } static void __flush_tlb_range(struct mm_struct *mm,
@@ -97,7 +107,6 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long size, unsigned long stride) { - unsigned long asid = get_mm_asid(mm); unsigned int cpu;if (cpumask_empty (cmask))
@@ -107,17 +116,17 @@ static void __flush_tlb_range(struct mm_struct *mm,/* Check if the TLB flush needs to be sent to other CPUs. */ if (cpumask_any_but (cmask, cpu) >= nr_cpu_ids) { -local_flush_tlb_range_asid (start, size, stride, asid); +local_flush_tlb_range_mm (mm, start, size, stride); }else if (riscv_use_sbi_for_rfence ()) { -sbi_remote_sfence_vma_asid (cmask, start, size, asid); +sbi_remote_sfence_vma_asid (cmask, start, size,get_mm_asid (mm)); }else {struct flush_tlb_range_data ftd; - ftd.asid = asid; + ftd.mm = mm; ftd.start = start; ftd.size = size; ftd.stride = stride; -on_each_cpu_mask (cmask, __ipi_flush_tlb_range_asid, &ftd,1 ); +on_each_cpu_mask (cmask, __ipi_flush_tlb_range_mm, &ftd,1 ); }put_cpu ();
Signature · 1 line
2.39.5