arm64: barriers: make use of barrier options with explicit barriers
Subsystems (5)
PATCH v2 1/5 arm64: barriers: make use of barrier options with explicit barriers
When calling our low-level barrier macros directly, we can often suffice with more relaxed behaviour than the default "all accesses, full system" option.
This patch updates the users of dsb() to specify the option which they actually require.
--- arch/arm64/include/asm/barrier.h | 2 +- arch/arm64/include/asm/cacheflush.h | 2 +- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/arm64/include/asm/tlbflush.h | 14 +++++++------- arch/arm64/kernel/process.c | 2 +- arch/arm64/kvm/sys_regs.c | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-)
arch/arm64/include/asm/barrier.h +1 -1
@@ -27,7 +27,7 @@#define isb () asm volatile("isb" : : : "memory")#define dsb (opt) asm volatile("dsb sy" : : : "memory") -#define mb () dsb() +#define mb () dsb(sy)#define rmb () asm volatile("dsb ld" : : : "memory")#define wmb () asm volatile("dsb st" : : : "memory")
arch/arm64/include/asm/cacheflush.h +1 -1
@@ -143,7 +143,7 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)* set_pte_at() called from vmap_pte_range() does not * have a DSB after cleaning the cache line. */ -dsb (); +dsb (ishst); }static inline void flush_cache_vunmap (unsigned long start,unsigned long end)
arch/arm64/include/asm/pgtable.h +2 -2
@@ -310,7 +310,7 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,static inline void set_pmd (pmd_t *pmdp,pmd_t pmd) { *pmdp = pmd; -dsb (); +dsb (ishst); }static inline void pmd_clear (pmd_t *pmdp)
@@ -340,7 +340,7 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)static inline void set_pud (pud_t *pudp,pud_t pud) { *pudp = pud; -dsb (); +dsb (ishst); }static inline void pud_clear (pud_t *pudp)
arch/arm64/include/asm/tlbflush.h +7 -7
@@ -72,9 +72,9 @@ extern struct cpu_tlb_fns cpu_tlb;*/ static inline void flush_tlb_all (void ) { -dsb (); +dsb (ishst); asm("tlbi vmalle1is" ); -dsb (); +dsb (ish);isb (); }
@@ -82,9 +82,9 @@ static inline void flush_tlb_mm(struct mm_struct *mm) {unsigned long asid = (unsigned long )ASID (mm) <<48 ; -dsb (); +dsb (ishst); asm("tlbi aside1is, %0" : :"r" (asid)); -dsb (); +dsb (ish); } static inline void flush_tlb_page(struct vm_area_struct *vma,
@@ -93,9 +93,9 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,unsigned long addr = uaddr >>12 | ((unsigned long )ASID (vma->vm_mm) <<48 ); -dsb (); +dsb (ishst); asm("tlbi vae1is, %0" : :"r" (addr)); -dsb (); +dsb (ish); }/*
@@ -114,7 +114,7 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,* set_pte() does not have a DSB, so make sure that the page table * write is visible. */ -dsb (); +dsb (ishst); }#define update_mmu_cache_pmd (vma, address, pmd) do { } while (0)
arch/arm64/kernel/process.c +1 -1
@@ -294,7 +294,7 @@ struct task_struct *__switch_to(struct task_struct *prev,* Complete any pending TLB or cache maintenance on this CPU in case * the thread migrates to a different CPU. */ -dsb (); +dsb (ish);/* the actual thread switch */ last =cpu_switch_to (prev, next);
arch/arm64/kvm/sys_regs.c +2 -2
@@ -70,13 +70,13 @@ static u32 get_ccsidr(u32 csselr)static void do_dc_cisw (u32 val) { asmvolatile ("dc cisw, %x0" : :"r" (val)); -dsb (); +dsb (ishst); }static void do_dc_csw (u32 val) { asmvolatile ("dc csw, %x0" : :"r" (val)); -dsb (); +dsb (ishst); }/* See note@ARM ARM B1.14.4 */
Signature · 1 line
1.8.2.2
PATCH v2 2/5 irqchip: gic: use writel instead of dsb + writel_relaxed
When sending an SGI to another CPU, we require a DSB to ensure that any pending stores to normal memory are made visible to the recipient before the interrupt arrives.
Rather than use a vanilla dsb() (which will soon cause an assembly error on arm64) before the writel_relaxed, we can instead use dsb(ishst), since we just need to ensure that any pending normal writes are visible within the inner-shareable domain before we poke the GIC.
---
v1 => v2: Use dsb ishst instead of writel (which requires an L2 sync)
since the sync should already have been executed by the caller if required. We *might* be able to relax this further to a dmb but Catalin and I haven't got to the bottom of that yet.
Marc: I dropped your Ack, so could you take another look please?
drivers/irqchip/irq-gic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/irqchip/irq-gic.c +1 -1
@@ -663,7 +663,7 @@ void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)* Ensure that stores to Normal memory are visible to the * other CPUs before issuing the IPI. */ -dsb (); +dsb (ishst);/* this always happens on GIC0 */ writel_relaxed (map <<16 | irq,gic_data_dist_base (&gic_data[0 ]) +GIC_DIST_SOFTINT );
Signature · 1 line
1.8.2.2
Catalin Marinas
maintainerPATCH v2 2/5 irqchip: gic: use writel instead of dsb + writel_relaxed
Quoted 12 lines · Will Deacon
When sending an SGI to another CPU, we require a DSB to ensure that any pending stores to normal memory are made visible to the recipient before the interrupt arrives.
Rather than use a vanilla
dsb()(which will soon cause an assembly error on arm64) before thewritel_relaxed, we can instead use dsb(ishst), since we just need to ensure that any pending normal writes are visible within the inner-shareable domain before we poke the GIC.Cc Thomas Gleixner <[email protected]>Cc Marc Zyngier <[email protected]>Signed-off-by Will Deacon <[email protected]>
Quoted 6 lines
---
v1 => v2: Use dsb ishst instead of writel (which requires an L2 sync)
since the sync should already have been executed by the caller if required. We *might* be able to relax this further to a dmb but Catalin and I haven't got to the bottom of that yet.
In the meantime,
PATCH v2 2/5 irqchip: gic: use writel instead of dsb + writel_relaxed
In reply to Will DeaconQuoted 12 lines · Will Deacon
When sending an SGI to another CPU, we require a DSB to ensure that any pending stores to normal memory are made visible to the recipient before the interrupt arrives.
Rather than use a vanilla
dsb()(which will soon cause an assembly error on arm64) before thewritel_relaxed, we can instead use dsb(ishst), since we just need to ensure that any pending normal writes are visible within the inner-shareable domain before we poke the GIC.Cc Thomas Gleixner <[email protected]>Cc Marc Zyngier <[email protected]>Signed-off-by Will Deacon <[email protected]>
Quoted 8 lines
---
v1 => v2: Use dsb ishst instead of writel (which requires an L2 sync)
since the sync should already have been executed by the caller if required. We *might* be able to relax this further to a dmb but Catalin and I haven't got to the bottom of that yet.Marc: I dropped your Ack, so could you take another look please?
M.
Signature · 1 line
Jazz is not dead. It just smells funny...
PATCH v2 3/5 iommu/arm-smmu: provide option to dsb macro when publishing tables
In reply to Will DeaconOn coherent systems, publishing new page tables to the SMMU walker is achieved with a dsb instruction. In fact, this can be a dsb(ishst) which also provides the mandatory barrier option for arm64.
--- drivers/iommu/arm-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/iommu/arm-smmu.c +1 -1
@@ -1417,7 +1417,7 @@ out_unlock:/* Ensure new page tables are visible to the hardware walker */ if (smmu->features &ARM_SMMU_FEAT_COHERENT_WALK ) -dsb (); +dsb (ishst);return ret; }
Signature · 1 line
1.8.2.2
PATCH v2 4/5 arm64: barriers: wire up new barrier options
In reply to Will DeaconNow that all callers of the barrier macros are updated to pass the mandatory options, update the macros so the option is actually used.
--- arch/arm64/include/asm/barrier.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
arch/arm64/include/asm/barrier.h +7 -6
@@ -25,11 +25,12 @@#define wfi () asm volatile("wfi" : : : "memory")#define isb () asm volatile("isb" : : : "memory") -#define dsb (opt) asm volatile("dsb sy" : : : "memory") +#define dmb (opt) asm volatile("dmb " #opt : : : "memory") +#define dsb (opt) asm volatile("dsb " #opt : : : "memory")#define mb () dsb(sy) -#define rmb () asm volatile("dsb ld" : : : "memory") -#define wmb () asm volatile("dsb st" : : : "memory") +#define rmb () dsb(ld) +#define wmb () dsb(st)#ifndef CONFIG_SMP #define smp_mb () barrier()
@@ -53,9 +54,9 @@ do { \#else -#define smp_mb () asm volatile("dmb ish" : : : "memory") -#define smp_rmb () asm volatile("dmb ishld" : : : "memory") -#define smp_wmb () asm volatile("dmb ishst" : : : "memory") +#define smp_mb () dmb(ish) +#define smp_rmb () dmb(ishld) +#define smp_wmb () dmb(ishst)#define smp_store_release (p, v) \ do { \
Signature · 1 line
1.8.2.2
PATCH v2 5/5 arm64: barriers: use barrier() instead of smp_mb() when !SMP
In reply to Will DeaconThe recently introduced acquire/release accessors refer to smp_mb() in the !CONFIG_SMP case. This is confusing when reading the code, so use barrier() directly when we know we're UP.
--- arch/arm64/include/asm/barrier.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/arm64/include/asm/barrier.h +2 -2
@@ -40,7 +40,7 @@#define smp_store_release (p, v) \ do { \ compiletime_assert_atomic_type(*p); \ - smp_mb(); \ + barrier(); \ ACCESS_ONCE(*p) = (v); \ } while (0)
@@ -48,7 +48,7 @@ do { \ ({ \ typeof(*p) ___p1 = ACCESS_ONCE(*p); \compiletime_assert_atomic_type (*p); \ -smp_mb (); \ +barrier (); \ ___p1; \ })
Signature · 1 line
1.8.2.2
Catalin Marinas
maintainerPATCH v2 1/5 arm64: barriers: make use of barrier options with explicit barriers
In reply to Will DeaconQuoted 8 lines · Will Deacon
When calling our low-level barrier macros directly, we can often suffice with more relaxed behaviour than the default "all accesses, full system" option.
This patch updates the users of
dsb()to specify the option which they actually require.Signed-off-by Will Deacon <[email protected]>