ARM: tlb: add branch predictor maintenance operations
PATCH 1/2 ARM: tlb: add branch predictor maintenance operations
The ARM architecture requires explicit branch predictor maintenance when updating an instruction stream for a given virtual address. In reality, this isn't so much of a burden because the branch predictor is flushed during the cache maintenance required to make the new instructions visible to the I-side of the processor.
However, there are still some cases where explicit flushing is required, so add a local_bp_flush_all operation to deal with this.
---
I can't make my mind up about whether or not this go to -stable... on the one hand it *is* part of a bug fix but, then again, it's not something that's knowingly caused any problems so far.
arch/arm/include/asm/tlbflush.h | 34 ++++++++++++++++++++++++++++------ arch/arm/kernel/smp_tlb.c | 12 ++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-)
arch/arm/include/asm/tlbflush.h +28 -6
@@ -34,10 +34,13 @@#define TLB_V6_D_ASID (1 << 17)#define TLB_V6_I_ASID (1 << 18) +#define TLB_V6_BP (1 << 19) +/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */ -#define TLB_V7_UIS_PAGE (1 << 19) -#define TLB_V7_UIS_FULL (1 << 20) -#define TLB_V7_UIS_ASID (1 << 21) +#define TLB_V7_UIS_PAGE (1 << 20) +#define TLB_V7_UIS_FULL (1 << 21) +#define TLB_V7_UIS_ASID (1 << 22) +#define TLB_V7_UIS_BP (1 << 23)#define TLB_BARRIER (1 << 28)#define TLB_L2CLEAN_FR (1 << 29)/* Feroceon */
@@ -150,7 +153,8 @@#define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ TLB_V6_I_FULL | TLB_V6_D_FULL | \ TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ - TLB_V6_I_ASID | TLB_V6_D_ASID) + TLB_V6_I_ASID | TLB_V6_D_ASID | \ + TLB_V6_BP)#ifdef CONFIG_CPU_TLB_V6 # define v6wbi_possible_flags v6wbi_tlb_flags
@@ -166,9 +170,11 @@#endif #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ - TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID) + TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | \ + TLB_V7_UIS_ASID | TLB_V7_UIS_BP)#define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BARRIER | \ - TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID) + TLB_V6_U_FULL | TLB_V6_U_PAGE | \ + TLB_V6_U_ASID | TLB_V6_BP)#ifdef CONFIG_CPU_TLB_V7
@@ -430,6 +436,20 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) } } +static inline void local_flush_bp_all (void ) +{ +const int zero =0 ; +const unsigned int __tlb_flag = __cpu_tlb_flags; + +if (tlb_flag (TLB_V7_UIS_BP )) + asm("mcr p15, 0, %0, c7, c1, 6" : :"r" (zero)); +else if (tlb_flag (TLB_V6_BP )) + asm("mcr p15, 0, %0, c7, c5, 6" : :"r" (zero)); + +if (tlb_flag (TLB_BARRIER )) +isb (); +} +/* * flush_pmd_entry *
@@ -480,6 +500,7 @@ static inline void clean_pmd_entry(void *pmd)#define flush_tlb_kernel_page local_flush_tlb_kernel_page#define flush_tlb_range local_flush_tlb_range#define flush_tlb_kernel_range local_flush_tlb_kernel_range +#define flush_bp_all local_flush_bp_all#else extern void flush_tlb_all (void );extern void flush_tlb_mm (struct mm_struct *mm);
@@ -487,6 +508,7 @@ extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr);extern void flush_tlb_kernel_page (unsigned long kaddr);extern void flush_tlb_range (struct vm_area_struct *vma,unsigned long start,unsigned long end);extern void flush_tlb_kernel_range (unsigned long start,unsigned long end); +extern void flush_bp_all (void );#endif /*
arch/arm/kernel/smp_tlb.c +12 -0
@@ -64,6 +64,11 @@ static inline void ipi_flush_tlb_kernel_range(void *arg)local_flush_tlb_kernel_range (ta->ta_start, ta->ta_end); } +static inline void ipi_flush_bp_all (void *ignored) +{ +local_flush_bp_all (); +} +void flush_tlb_all (void ) {if (tlb_ops_need_broadcast ())
@@ -127,3 +132,10 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)local_flush_tlb_kernel_range (start, end); } +void flush_bp_all (void ) +{ +if (tlb_ops_need_broadcast ()) +on_each_cpu (ipi_flush_bp_all,NULL ,1 ); +else +local_flush_bp_all (); +}
Signature · 1 line
1.8.0
PATCH 2/2 ARM: mm: perform explicit branch predictor maintenance when required
The ARM ARM requires branch predictor maintenance if, for a given ASID, the instructions at a specific virtual address appear to change.
Quoted 1 line
From the kernel's point of view, that means:
- Changing the kernel's view of memory (e.g. switching to the identity map)
- ASID rollover (since ASIDs will be re-allocated to new tasks)
This patch adds explicit branch predictor maintenance when either of the two conditions above are met.
--- arch/arm/kernel/smp.c | 1 + arch/arm/kernel/suspend.c | 1 + arch/arm/mm/context.c | 4 +++- arch/arm/mm/idmap.c | 1 + 4 files changed, 6 insertions(+), 1 deletion(-)
arch/arm/kernel/smp.c +1 -0
@@ -285,6 +285,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void)* switch away from it before attempting any exclusive accesses. */ cpu_switch_mm (mm->pgd, mm); +local_flush_bp_all ();enter_lazy_tlb (mm, current);local_flush_tlb_all ();
arch/arm/kernel/suspend.c +1 -0
@@ -68,6 +68,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) ret =__cpu_suspend (arg, fn);if (ret ==0 ) {cpu_switch_mm (mm->pgd, mm); +local_flush_bp_all ();local_flush_tlb_all (); }
arch/arm/mm/context.c +3 -1
@@ -212,8 +212,10 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)atomic64_set (&mm->context.id, asid); } -if (cpumask_test_and_clear_cpu (cpu, &tlb_flush_pending)) +if (cpumask_test_and_clear_cpu (cpu, &tlb_flush_pending)) { +local_flush_bp_all ();local_flush_tlb_all (); + }atomic64_set (&per_cpu (active_asids, cpu), asid);cpumask_set_cpu (cpu,mm_cpumask (mm));
arch/arm/mm/idmap.c +1 -0
@@ -141,6 +141,7 @@ void setup_mm_for_reboot(void) {/* Switch to the identity mapping. */ cpu_switch_mm (idmap_pgd, &init_mm); +local_flush_bp_all ();#ifdef CONFIG_CPU_HAS_ASID /*
Signature · 1 line
1.8.0
PATCH 1/2 ARM: tlb: add branch predictor maintenance operations
In reply to Will DeaconQuoted 11 lines · Will Deacon
The ARM architecture requires explicit branch predictor maintenance when updating an instruction stream for a given virtual address. In reality, this isn't so much of a burden because the branch predictor is flushed during the cache maintenance required to make the new instructions visible to the I-side of the processor.
However, there are still some cases where explicit flushing is required, so add a
local_bp_flush_alloperation to deal with this.Cc <[email protected]>Signed-off-by Will Deacon <[email protected]>
PATCH 2/2 ARM: mm: perform explicit branch predictor maintenance when required
In reply to Will DeaconQuoted 9 lines · Will Deacon
The ARM ARM requires branch predictor maintenance if, for a given ASID, the instructions at a specific virtual address appear to change.
From the kernel's point of view, that means:
- Changing the kernel's view of memory (e.g. switching to the identity map)
- ASID rollover (since ASIDs will be re-allocated to new tasks)
Quoted 5 lines
This patch adds explicit branch predictor maintenance when either of the two conditions above are met.
Cc <[email protected]>Signed-off-by Will Deacon <[email protected]>