arm64/mm: save memory access in check_and_switch_context() fast switch path
PATCH v2 1/1 arm64/mm: save memory access in check_and_switch_context() fast switch path
On arm64, smp_processor_id() reads a per-cpu cpu_number variable, using the per-cpu offset stored in the tpidr_el1 system register. In some cases we generate a per-cpu address with a sequence like:
cpu_ptr = &per_cpu(ptr, smp_processor_id());
Which potentially incurs a cache miss for both cpu_number and the in-memory __per_cpu_offset array. This can be written more optimally as:
cpu_ptr = this_cpu_ptr(ptr);
Which only needs the offset from tpidr_el1, and does not need to load from memory.
Signed-off-by Pingfan Liu <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Will Deacon <[email protected]>
Cc Steve Capper <[email protected]>
Cc Mark Rutland <[email protected]>
Cc Vladimir Murzin <[email protected]>
Cc Jean-Philippe Brucker <[email protected]>
To: [email protected]
---
v1 -> v2:
improve commit log
--- arch/arm64/include/asm/mmu_context.h | 6 ++---- arch/arm64/mm/context.c | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-)
arch/arm64/include/asm/mmu_context.h +2 -4
@@ -175,7 +175,7 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp)* take CPU migration into account. */ #define destroy_context (mm) do { } while(0) -void check_and_switch_context (struct mm_struct *mm,unsigned int cpu); +void check_and_switch_context (struct mm_struct *mm);#define init_new_context (tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; })
@@ -214,8 +214,6 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)static inline void __switch_mm (struct mm_struct *next) { -unsigned int cpu =smp_processor_id (); -/* * init_mm.pgd does not contain any user mappings and it is always * active for kernel addresses in TTBR1. Just set the reserved TTBR0.
@@ -225,7 +223,7 @@ static inline void __switch_mm(struct mm_struct *next)return ; } -check_and_switch_context (next, cpu); +check_and_switch_context (next); } static inline void
arch/arm64/mm/context.c +6 -4
@@ -198,9 +198,10 @@ static u64 new_context(struct mm_struct *mm)return idx2asid (asid) | generation; } -void check_and_switch_context (struct mm_struct *mm,unsigned int cpu) +void check_and_switch_context (struct mm_struct *mm) {unsigned long flags; +unsigned int cpu;u64 asid, old_active_asid;if (system_supports_cnp ())
@@ -222,9 +223,9 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)* relaxed xchg in flush_context will treat us as reserved * because atomic RmWs are totally ordered for a given location. */ - old_active_asid =atomic64_read (&per_cpu (active_asids, cpu)); + old_active_asid =atomic64_read (this_cpu_ptr (&active_asids));if (old_active_asid &&asid_gen_match (asid) && -atomic64_cmpxchg_relaxed (&per_cpu (active_asids, cpu), +atomic64_cmpxchg_relaxed (this_cpu_ptr (&active_asids), old_active_asid, asid)) goto switch_mm_fastpath;
@@ -236,10 +237,11 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)atomic64_set (&mm->context.id, asid); } + cpu =smp_processor_id ();if (cpumask_test_and_clear_cpu (cpu, &tlb_flush_pending))local_flush_tlb_all (); -atomic64_set (&per_cpu (active_asids, cpu), asid); +atomic64_set (this_cpu_ptr (&active_asids), asid);raw_spin_unlock_irqrestore (&cpu_asid_lock, flags); switch_mm_fastpath:
Signature · 7 lines
2.7.5 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel