csky: Use generic asid function from arm
Subsystems (2)
[email protected]
maintainerv2 cover csky: Use generic asid function from arm
From: Guo Ren <[email protected]>
For C-SKY the first implementation is from mips and it's not implemented well for SMP in performance. Arm's asid allocator is right for us and we've tested it in our all CPUs:610/807/810/860 and it really reduce the tlb flush. We'll continue stress testing before merge into master.
Changes for V2:
- Add back set/clear cpu_mm.
- Update commit message to explain asid allocator.
Guo Ren (4):
csky: Revert mmu ASID mechanism
csky: Add new asid lib code from arm
csky: Use generic asid algorithm to implement switch_mm
csky: Improve tlb operation with help of asid
arch/arm64/lib/asid.c | 9 +- arch/csky/abiv1/inc/abi/ckmmu.h | 6 + arch/csky/abiv2/inc/abi/ckmmu.h | 10 ++ arch/csky/include/asm/asid.h | 78 ++++++++++++ arch/csky/include/asm/mmu.h | 2 +- arch/csky/include/asm/mmu_context.h | 114 ++--------------- arch/csky/include/asm/pgtable.h | 2 - arch/csky/kernel/smp.c | 2 - arch/csky/mm/Makefile | 2 + arch/csky/mm/asid.c | 188 ++++++++++++++++++++++++++++ arch/csky/mm/context.c | 46 +++++++ arch/csky/mm/init.c | 2 - arch/csky/mm/tlb.c | 238 ++++++++++++++---------------------- 13 files changed, 444 insertions(+), 255 deletions(-) create mode 100644 arch/csky/include/asm/asid.h create mode 100644 arch/csky/mm/asid.c create mode 100644 arch/csky/mm/context.c
Signature · 1 line
2.7.4
[email protected]
maintainerPATCH v2 1/4 csky: Revert mmu ASID mechanism
From: Guo Ren <[email protected]>
Current C-SKY ASID mechanism is from mips and it doesn't work well with multi-cores. ASID per core mechanism is not suitable for C-SKY SMP tlb maintain operations, eg: tlbi.vas need share the same asid in all processors and it'll invalid the tlb entry in all cores with the same asid.
This patch is prepare for new ASID mechanism.
--- arch/csky/include/asm/mmu.h | 1 - arch/csky/include/asm/mmu_context.h | 112 ++------------------- arch/csky/include/asm/pgtable.h | 2 - arch/csky/kernel/smp.c | 2 - arch/csky/mm/init.c | 2 - arch/csky/mm/tlb.c | 190 ++---------------------------------- 6 files changed, 14 insertions(+), 295 deletions(-)
arch/csky/include/asm/mmu.h +0 -1
@@ -5,7 +5,6 @@#define __ASM_CSKY_MMU_Htypedef struct { -unsigned long asid[NR_CPUS ];void *vdso; }mm_context_t ;
arch/csky/include/asm/mmu_context.h +7 -105
129 lines not on this page · raw message
arch/csky/include/asm/pgtable.h +0 -2
@@ -290,8 +290,6 @@ static inline pte_t *pte_offset(pmd_t *dir, unsigned long address)extern pgd_t swapper_pg_dir[PTRS_PER_PGD ];extern void paging_init (void ); -extern void show_jtlb_table (void ); -void update_mmu_cache (struct vm_area_struct *vma,unsigned long address,pte_t *pte);
arch/csky/kernel/smp.c +0 -2
@@ -212,8 +212,6 @@ void csky_start_secondary(void)TLBMISS_HANDLER_SETUP_PGD (swapper_pg_dir);TLBMISS_HANDLER_SETUP_PGD_KERNEL (swapper_pg_dir); -asid_cache (smp_processor_id ()) =ASID_FIRST_VERSION ; -#ifdef CONFIG_CPU_HAS_FPU init_fpu ();#endif
arch/csky/mm/init.c +0 -2
@@ -114,8 +114,6 @@ void __init pre_mmu_init(void)TLBMISS_HANDLER_SETUP_PGD (swapper_pg_dir);TLBMISS_HANDLER_SETUP_PGD_KERNEL (swapper_pg_dir); -asid_cache (smp_processor_id ()) =ASID_FIRST_VERSION ; -/* Setup page mask to 4k */ write_mmu_pagemask (0 ); }
arch/csky/mm/tlb.c +7 -183
216 lines not on this page · raw message
Signature · 1 line
2.7.4
[email protected]
maintainerPATCH v2 2/4 csky: Add new asid lib code from arm
In reply to [email protected]From: Guo Ren <[email protected]>
This patch only contains asid help code from arm for next patch to use.
The asid allocator use five level check to reduce the cost of switch_mm.
1. Check if the asid version is the same (it's general)
2. Check reserved_asid which is set in rollover flush_context() and key point is to keep the same bit position with the current asid version instead of input version.
3. Check if the position of bitmap is free then it could be set & used directly.
4. find_next_zero_bit() (a little performance cost)
5. flush_context (this is the worst cost with increase current asid version)
Check is level by level and cost is also higher with the next level. The reserved_asid and bitmap mechanism prevent unnecessary find_next_zero_bit().
The atomic 64 bit asid is also suitable for 32-bit system and it won't cost a lot in 1th 2th 3th level check.
The operation of set/clear mm_cpumask was removed in arm64 compared to arm32. It seems no side effect on current arm64 system, but from software meaning it's wrong. Although csky also needn't it, we add it back for csky.
The asid_per_ctxt is no use for csky and it reserves the lowest bits for other use, maybe: trust zone ? Ok, just keep it in csky copy.
Seems it also could be used by other archs and it's worth to move asid code to generic in future.
--- arch/arm64/lib/asid.c | 9 ++- arch/csky/include/asm/asid.h | 78 ++++++++++++++++++ arch/csky/mm/Makefile | 1 + arch/csky/mm/asid.c | 188 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 arch/csky/include/asm/asid.h create mode 100644 arch/csky/mm/asid.c
arch/arm64/lib/asid.c +6 -3
@@ -75,7 +75,8 @@ static bool check_update_reserved_asid(struct asid_info *info, u64 asid,return hit; } -static u64 new_context (struct asid_info *info,atomic64_t *pasid) +static u64 new_context(struct asid_info *info, atomic64_t *pasid + struct mm_struct *mm) {static u32 cur_idx =1 ;u64 asid =atomic64_read (pasid);
@@ -121,6 +122,7 @@ static u64 new_context(struct asid_info *info, atomic64_t *pasid) set_asid:__set_bit (asid, info->map); cur_idx = asid; +cpumask_clear (mm_cpumask (mm));return idx2asid (info, asid) | generation; }
@@ -132,7 +134,7 @@ static u64 new_context(struct asid_info *info, atomic64_t *pasid)* @cpu: current CPU ID. Must have been acquired through get_cpu() */ void asid_new_context (struct asid_info *info,atomic64_t *pasid, -unsigned int cpu) +unsigned int cpu,struct mm_struct *mm) {unsigned long flags;u64 asid;
@@ -141,7 +143,7 @@ void asid_new_context(struct asid_info *info, atomic64_t *pasid,/* Check that our ASID belongs to the current generation. */ asid =atomic64_read (pasid);if ((asid ^atomic64_read (&info->generation)) >> info->bits) { - asid =new_context (info, pasid); + asid =new_context (info, pasid, mm);atomic64_set (pasid, asid); }
@@ -149,6 +151,7 @@ void asid_new_context(struct asid_info *info, atomic64_t *pasid, info->flush_cpu_ctxt_cb ();atomic64_set (&active_asid (info, cpu), asid); +cpumask_set_cpu (cpu,mm_cpumask (mm));raw_spin_unlock_irqrestore (&info->lock, flags); }
arch/csky/include/asm/asid.h new file +78 -0
78 lines not on this page · raw message
arch/csky/mm/Makefile +1 -0
@@ -11,3 +11,4 @@ obj-y += init.o obj-y += ioremap.o obj-y += syscache.o obj-y += tlb.o +obj-y += asid.o
arch/csky/mm/asid.c new file +188 -0
188 lines not on this page · raw message
Signature · 1 line
2.7.4
Re: [PATCH V2 2/4] csky: Add new asid lib code from arm
Hi,
Quoted 5 lines · On 21/06/2019 10:39, [email protected] wrote:
Signed-off-by Guo Ren <[email protected]>Cc Arnd Bergmann <[email protected]>Cc Julien Grall <[email protected]>--- arch/arm64/lib/asid.c | 9 ++-
This change should be in a separate e-mail with the Arm64 maintainers in CC.
But you seem to have a copy of the allocator in csky now. So why do you need to modify arm64/lib/asid.c here?
Cheers,
Signature · 1 line
Julien Grall
Guo Ren
maintainerRe: [PATCH V2 2/4] csky: Add new asid lib code from arm
Sorry, I forgot delete arm's. It's mistake, no change arm64 file.
Quoted 19 lines · Julien Grall
Hi,
Quoted 5 lines · On 21/06/2019 10:39, [email protected] wrote:
Signed-off-by Guo Ren <[email protected]>Cc Arnd Bergmann <[email protected]>Cc Julien Grall <[email protected]>--- arch/arm64/lib/asid.c | 9 ++-This change should be in a separate e-mail with the Arm64 maintainers in CC.
But you seem to have a copy of the allocator in csky now. So why do you need to modify
arm64/lib/asid.chere?Cheers,
--
Julien Grall
Signature · 4 lines
Best Regards Guo Ren ML: https://lore.kernel.org/linux-csky/
[email protected]
maintainerPATCH v2 3/4 csky: Use generic asid algorithm to implement switch_mm
In reply to [email protected]From: Guo Ren <[email protected]>
Use linux generic asid/vmid algorithm to implement csky switch_mm function. The algorithm is from arm and it could work with SMP system. It'll help reduce tlb flush for switch_mm in task/vm switch.
--- arch/csky/abiv1/inc/abi/ckmmu.h | 6 +++++ arch/csky/abiv2/inc/abi/ckmmu.h | 10 ++++++++ arch/csky/include/asm/mmu.h | 1 + arch/csky/include/asm/mmu_context.h | 12 ++++++++-- arch/csky/mm/Makefile | 1 + arch/csky/mm/context.c | 46 +++++++++++++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 arch/csky/mm/context.c
arch/csky/abiv1/inc/abi/ckmmu.h +6 -0
@@ -78,6 +78,12 @@ static inline void tlb_invalid_all(void)cpwcr ("cpcr8" ,0x04000000 ); } + +static inline void local_tlb_invalid_all (void ) +{ +tlb_invalid_all (); +} +static inline void tlb_invalid_indexed (void ) {cpwcr ("cpcr8" ,0x02000000 );
arch/csky/abiv2/inc/abi/ckmmu.h +10 -0
@@ -85,6 +85,16 @@ static inline void tlb_invalid_all(void)#endif } +static inline void local_tlb_invalid_all (void ) +{ +#ifdef CONFIG_CPU_HAS_TLBI + asmvolatile ("tlbi.all\n" :::"memory" ); +sync_is (); +#else +tlb_invalid_all (); +#endif +} +static inline void tlb_invalid_indexed (void ) {mtcr ("cr<8, 15>" ,0x02000000 );
arch/csky/include/asm/mmu.h +1 -0
@@ -5,6 +5,7 @@#define __ASM_CSKY_MMU_Htypedef struct { +atomic64_t asid;void *vdso; }mm_context_t ;
arch/csky/include/asm/mmu_context.h +10 -2
@@ -20,20 +20,28 @@#define TLBMISS_HANDLER_SETUP_PGD_KERNEL (pgd) \ setup_pgd(__pa(pgd), true) -#define init_new_context (tsk,mm) 0 +#define ASID_MASK ((1 << CONFIG_CPU_ASID_BITS) - 1) +#define cpu_asid (mm) (atomic64_read(&mm->context.asid) & ASID_MASK) + +#define init_new_context (tsk,mm) ({ atomic64_set(&(mm)->context.asid, 0); 0; })#define activate_mm (prev,next) switch_mm(prev, next, current)#define destroy_context (mm) do {} while (0)#define enter_lazy_tlb (mm, tsk) do {} while (0)#define deactivate_mm (tsk, mm) do {} while (0) +void check_and_switch_context (struct mm_struct *mm,unsigned int cpu); +static inline void switch_mm (struct mm_struct *prev,struct mm_struct *next,struct task_struct *tsk) { +unsigned int cpu =smp_processor_id (); +if (prev != next) -tlb_invalid_all (); +check_and_switch_context (next, cpu);TLBMISS_HANDLER_SETUP_PGD (next->pgd); +write_mmu_entryhi (next->context.asid.counter); }#endif /* __ASM_CSKY_MMU_CONTEXT_H */
arch/csky/mm/Makefile +1 -0
@@ -12,3 +12,4 @@ obj-y += ioremap.o obj-y += syscache.o obj-y += tlb.o obj-y += asid.o +obj-y += context.o
arch/csky/mm/context.c new file +46 -0
46 lines not on this page · raw message
Signature · 1 line
2.7.4
[email protected]
maintainerPATCH v2 4/4 csky: Improve tlb operation with help of asid
In reply to [email protected]From: Guo Ren <[email protected]>
There are two generations of tlb operation instruction for C-SKY. First generation is use mcr register and it need software do more things, second generation is use specific instructions, eg:
tlbi.va, tlbi.vas, tlbi.alls
We implemented the following functions:
- flush_tlb_range (a range of entries)
- flush_tlb_page (one entry)
Above functions use asid from vma->mm to invalid tlb entries and we could use tlbi.vas instruction for newest generation csky cpu.
- flush_tlb_kernel_range
- flush_tlb_one
Above functions don't care asid and it invalid the tlb entries only with vpn and we could use tlbi.vaas instruction for newest generat- ion csky cpu.
--- arch/csky/mm/tlb.c | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 132 insertions(+), 4 deletions(-)
arch/csky/mm/tlb.c +132 -4
@@ -10,6 +10,13 @@#include <asm/pgtable.h> #include <asm/setup.h> +/* +* One C-SKY MMU TLB entry contain two PFN/page entry, ie: +* 1VPN -> 2PFN +*/ +#define TLB_ENTRY_SIZE (PAGE_SIZE * 2) +#define TLB_ENTRY_SIZE_MASK (PAGE_MASK << 1) +void flush_tlb_all (void ) {tlb_invalid_all ();
@@ -17,27 +24,148 @@ void flush_tlb_all(void)void flush_tlb_mm (struct mm_struct *mm) { +#ifdef CONFIG_CPU_HAS_TLBI + asmvolatile ("tlbi.asids %0" ::"r" (cpu_asid (mm))); +#else tlb_invalid_all (); +#endif } +/* +* MMU operation regs only could invalid tlb entry in jtlb and we +* need change asid field to invalid I-utlb & D-utlb. +*/ +#ifndef CONFIG_CPU_HAS_TLBI +#define restore_asid_inv_utlb (oldpid, newpid) \ +do { \ + if (oldpid == newpid) \ + write_mmu_entryhi(oldpid + 1); \ + write_mmu_entryhi(oldpid); \ +} while (0) +#endif +void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end) { -tlb_invalid_all (); +unsigned long newpid =cpu_asid (vma->vm_mm); + + start &=TLB_ENTRY_SIZE_MASK ; + end +=TLB_ENTRY_SIZE -1 ; + end &=TLB_ENTRY_SIZE_MASK ; + +#ifdef CONFIG_CPU_HAS_TLBI +while (start < end) { + asmvolatile ("tlbi.vas %0" ::"r" (start | newpid)); + start +=2 *PAGE_SIZE ; + } +sync_is (); +#else + { +unsigned long flags, oldpid; + +local_irq_save (flags); + oldpid =read_mmu_entryhi () &ASID_MASK ; +while (start < end) { +int idx; + +write_mmu_entryhi (start | newpid); + start +=2 *PAGE_SIZE ; +tlb_probe (); + idx =read_mmu_index (); +if (idx >=0 ) +tlb_invalid_indexed (); + } +restore_asid_inv_utlb (oldpid, newpid); +local_irq_restore (flags); + } +#endif }void flush_tlb_kernel_range (unsigned long start,unsigned long end) { -tlb_invalid_all (); + start &=TLB_ENTRY_SIZE_MASK ; + end +=TLB_ENTRY_SIZE -1 ; + end &=TLB_ENTRY_SIZE_MASK ; + +#ifdef CONFIG_CPU_HAS_TLBI +while (start < end) { + asmvolatile ("tlbi.vaas %0" ::"r" (start)); + start +=2 *PAGE_SIZE ; + } +sync_is (); +#else + { +unsigned long flags, oldpid; + +local_irq_save (flags); + oldpid =read_mmu_entryhi () &ASID_MASK ; +while (start < end) { +int idx; + +write_mmu_entryhi (start | oldpid); + start +=2 *PAGE_SIZE ; +tlb_probe (); + idx =read_mmu_index (); +if (idx >=0 ) +tlb_invalid_indexed (); + } +restore_asid_inv_utlb (oldpid, oldpid); +local_irq_restore (flags); + } +#endif }void flush_tlb_page (struct vm_area_struct *vma,unsigned long addr) { -tlb_invalid_all (); +int newpid =cpu_asid (vma->vm_mm); + + addr &=TLB_ENTRY_SIZE_MASK ; + +#ifdef CONFIG_CPU_HAS_TLBI + asmvolatile ("tlbi.vas %0" ::"r" (addr | newpid)); +sync_is (); +#else + { +int oldpid, idx; +unsigned long flags; + +local_irq_save (flags); + oldpid =read_mmu_entryhi () &ASID_MASK ; +write_mmu_entryhi (addr | newpid); +tlb_probe (); + idx =read_mmu_index (); +if (idx >=0 ) +tlb_invalid_indexed (); + +restore_asid_inv_utlb (oldpid, newpid); +local_irq_restore (flags); + } +#endif }void flush_tlb_one (unsigned long addr) { -tlb_invalid_all (); + addr &=TLB_ENTRY_SIZE_MASK ; + +#ifdef CONFIG_CPU_HAS_TLBI + asmvolatile ("tlbi.vaas %0" ::"r" (addr)); +sync_is (); +#else + { +int oldpid, idx; +unsigned long flags; + +local_irq_save (flags); + oldpid =read_mmu_entryhi () &ASID_MASK ; +write_mmu_entryhi (addr | oldpid); +tlb_probe (); + idx =read_mmu_index (); +if (idx >=0 ) +tlb_invalid_indexed (); + +restore_asid_inv_utlb (oldpid, oldpid); +local_irq_restore (flags); + } +#endif }EXPORT_SYMBOL (flush_tlb_one);
Signature · 1 line
2.7.4