Skip to contentSkip to conversation

ARM mandatory barriers

PATCH New Catalin Marinas 2 participants 9 messages Mar 19, 2010 – Mar 23, 2010 Arm (32-bit)

lore
Share & tools
lore
#0 lore raw

v5 cover ARM mandatory barriers

Hi,

Pretty much as v4, just with minor changes as a result of patch reordering (in my devel branch -
http://www.linux-arm.org/git?p=linux-2.6.git;a=shortlog;h=refs/heads/devel)

Catalin Marinas (4):
ARM: Move the outer_cache definitions into a separate file
ARM: Add outer_cache_fns.sync function pointer
ARM: Add L2x0 outer_sync() support
ARM: Change the mandatory barriers implementation

 arch/arm/include/asm/cacheflush.h |   38 -------------------
 arch/arm/include/asm/outercache.h |   75 +++++++++++++++++++++++++++++++++++++
 arch/arm/include/asm/system.h     |   16 +++++---
 arch/arm/mm/Kconfig               |   13 ++++++
 arch/arm/mm/cache-l2x0.c          |   10 +++++
 5 files changed, 109 insertions(+), 43 deletions(-)
 create mode 100644 arch/arm/include/asm/outercache.h
Signature · 1 line
Catalin
#1 lore raw

PATCH v5 1/4 ARM: Move the outer_cache definitions into a separate file

To avoid #include collisions with subsequent patches in the series, this patch moves the outer_cache definitions to a separate asm/outercache.h file.

---
 arch/arm/include/asm/cacheflush.h |   38 +----------------------
 arch/arm/include/asm/outercache.h |   61 +++++++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 37 deletions(-)
 create mode 100644 arch/arm/include/asm/outercache.h
arch/arm/include/asm/cacheflush.h +1 -37
index 72da7e0..0d08d41 100644
@@ -15,6 +15,7 @@
 #include <asm/glue.h>
 #include <asm/shmparam.h>
 #include <asm/cachetype.h>
+#include <asm/outercache.h>
 
 #define CACHE_COLOUR(vaddr)	((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT)
 
@@ -219,12 +220,6 @@ struct cpu_cache_fns {
 	void (*dma_flush_range)(const void *, const void *);
 };
 
-struct outer_cache_fns {
-	void (*inv_range)(unsigned long, unsigned long);
-	void (*clean_range)(unsigned long, unsigned long);
-	void (*flush_range)(unsigned long, unsigned long);
-};
-
 /*
  * Select the calling method
  */
@@ -281,37 +276,6 @@ extern void dmac_flush_range(const void *, const void *);
 
 #endif
 
-#ifdef CONFIG_OUTER_CACHE
-
-extern struct outer_cache_fns outer_cache;
-
-static inline void outer_inv_range(unsigned long start, unsigned long end)
-{
-	if (outer_cache.inv_range)
-		outer_cache.inv_range(start, end);
-}
-static inline void outer_clean_range(unsigned long start, unsigned long end)
-{
-	if (outer_cache.clean_range)
-		outer_cache.clean_range(start, end);
-}
-static inline void outer_flush_range(unsigned long start, unsigned long end)
-{
-	if (outer_cache.flush_range)
-		outer_cache.flush_range(start, end);
-}
-
-#else
-
-static inline void outer_inv_range(unsigned long start, unsigned long end)
-{ }
-static inline void outer_clean_range(unsigned long start, unsigned long end)
-{ }
-static inline void outer_flush_range(unsigned long start, unsigned long end)
-{ }
-
-#endif
-
 /*
  * Copy user data from/to a page which is mapped into a different
  * processes address space.  Really, we want to allow our "user
arch/arm/include/asm/outercache.h new file +61 -0

61 lines not on this page · raw message

#2 lore raw

PATCH v5 1/4 ARM: Move the outer_cache definitions into a separate file

Quoted 3 lines · Catalin Marinas

To avoid #include collisions with subsequent patches in the series, this patch moves the outer_cache definitions to a separate asm/outercache.h file.

Ok.

#3 lore raw

PATCH v5 2/4 ARM: Add outer_cache_fns.sync function pointer

In reply to Catalin Marinas

This patch introduces the outer_cache_fns.sync function pointer together with the OUTER_CACHE_SYNC config option that can be used to drain the write buffer of the outer cache.

---
 arch/arm/include/asm/outercache.h |   14 ++++++++++++++
 arch/arm/mm/Kconfig               |    6 ++++++
 2 files changed, 20 insertions(+), 0 deletions(-)
arch/arm/include/asm/outercache.h +14 -0
index c8571cb..25f76ba 100644
@@ -25,6 +25,9 @@ struct outer_cache_fns {
 	void (*inv_range)(unsigned long, unsigned long);
 	void (*clean_range)(unsigned long, unsigned long);
 	void (*flush_range)(unsigned long, unsigned long);
+#ifdef CONFIG_OUTER_CACHE_SYNC
+	void (*sync)(void);
+#endif
 };
 
 #ifdef CONFIG_OUTER_CACHE
@@ -58,4 +61,15 @@ static inline void outer_flush_range(unsigned long start, unsigned long end)
 
 #endif
 
+#ifdef CONFIG_OUTER_CACHE_SYNC
+static inline void outer_sync(void)
+{
+	if (outer_cache.sync)
+		outer_cache.sync();
+}
+#else
+static inline void outer_sync(void)
+{ }
+#endif
+
 #endif	/* __ASM_OUTERCACHE_H */
arch/arm/mm/Kconfig +6 -0
index c4ed9f9..88a24de 100644
@@ -736,6 +736,12 @@ config NEEDS_SYSCALL_FOR_CMPXCHG
 config OUTER_CACHE
 	bool
 
+config OUTER_CACHE_SYNC
+	bool
+	help
+	  The outer cache has a outer_cache_fns.sync function pointer
+	  that can be used to drain the write buffer of the outer cache.
+
 config CACHE_FEROCEON_L2
 	bool "Enable the Feroceon L2 cache controller"
 	depends on ARCH_KIRKWOOD || ARCH_MV78XX0
#4 lore raw

PATCH v5 2/4 ARM: Add outer_cache_fns.sync function pointer

Quoted 3 lines · Catalin Marinas

This patch introduces the outer_cache_fns.sync function pointer together with the OUTER_CACHE_SYNC config option that can be used to drain the write buffer of the outer cache.

Ok.

#5 lore raw

PATCH v5 3/4 ARM: Add L2x0 outer_sync() support

In reply to Catalin Marinas

The L2x0 cache controllers need to explicitly drain their write buffer even for Normal Noncacheable memory accesses.

---
 arch/arm/mm/Kconfig      |    1 +
 arch/arm/mm/cache-l2x0.c |   10 ++++++++++
 2 files changed, 11 insertions(+), 0 deletions(-)
arch/arm/mm/Kconfig +1 -0
index 88a24de..55a2a00 100644
@@ -763,6 +763,7 @@ config CACHE_L2X0
 		   REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK || ARCH_OMAP4
 	default y
 	select OUTER_CACHE
+	select OUTER_CACHE_SYNC
 	help
 	  This option enables the L2x0 PrimeCell.
 
arch/arm/mm/cache-l2x0.c +10 -0
index 0733463..21ad68b 100644
@@ -93,6 +93,15 @@ static inline void l2x0_flush_line(unsigned long addr)
 }
 #endif
 
+static void l2x0_cache_sync(void)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&l2x0_lock, flags);
+	cache_sync();
+	spin_unlock_irqrestore(&l2x0_lock, flags);
+}
+
 static inline void l2x0_inv_all(void)
 {
 	unsigned long flags;
@@ -225,6 +234,7 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
 	outer_cache.inv_range = l2x0_inv_range;
 	outer_cache.clean_range = l2x0_clean_range;
 	outer_cache.flush_range = l2x0_flush_range;
+	outer_cache.sync = l2x0_cache_sync;
 
 	printk(KERN_INFO "L2X0 cache controller enabled\n");
 }
#6 lore raw

PATCH v5 3/4 ARM: Add L2x0 outer_sync() support

Quoted 2 lines · Catalin Marinas

The L2x0 cache controllers need to explicitly drain their write buffer even for Normal Noncacheable memory accesses.

Ok.

#7 lore raw

PATCH v5 4/4 ARM: Change the mandatory barriers implementation

In reply to Catalin Marinas

The mandatory barriers (mb, rmb, wmb) are used even on uniprocessor systems for things like ordering Normal Non-cacheable memory accesses with DMA transfer (via Device memory writes). The current implementation uses dmb() for mb() and friends but this is not sufficient. The DMB only ensures the relative ordering of the observability of accesses by other processors or devices acting as masters. In case of DMA transfers started by writes to device memory, the relative ordering is not ensured because accesses to slave ports of a device are not considered observable by the DMB definition.

A DSB is required for the data to reach the main memory (even if mapped as Normal Non-cacheable) before the device receives the notification to begin the transfer. Furthermore, some L2 cache controllers (like L2x0 or PL310) buffer stores to Normal Non-cacheable memory and this would need to be drained with the outer_sync() function call.

The patch also allows platforms to define their own mandatory barriers implementation by selecting CONFIG_ARCH_HAS_BARRIERS and providing a mach/barriers.h file.

Note that the SMP barriers are unchanged (being DMBs as before) since they are only guaranteed to work with Normal Cacheable memory.

---
 arch/arm/include/asm/system.h |   16 ++++++++++------
 arch/arm/mm/Kconfig           |    6 ++++++
 2 files changed, 16 insertions(+), 6 deletions(-)
arch/arm/include/asm/system.h +10 -6
index ca88e6a..4ace45e 100644
@@ -60,6 +60,8 @@
 #include <linux/linkage.h>
 #include <linux/irqflags.h>
 
+#include <asm/outercache.h>
+
 #define __exception	__attribute__((section(".exception.text")))
 
 struct thread_info;
@@ -137,10 +139,12 @@ extern unsigned int user_debug;
 #define dmb() __asm__ __volatile__ ("" : : : "memory")
 #endif
 
-#if __LINUX_ARM_ARCH__ >= 7 || defined(CONFIG_SMP)
-#define mb()		dmb()
+#ifdef CONFIG_ARCH_HAS_BARRIERS
+#include <mach/barriers.h>
+#elif __LINUX_ARM_ARCH__ >= 7 || defined(CONFIG_SMP)
+#define mb()		do { dsb(); outer_sync(); } while (0)
 #define rmb()		dmb()
-#define wmb()		dmb()
+#define wmb()		mb()
 #else
 #define mb()	do { if (arch_is_coherent()) dmb(); else barrier(); } while (0)
 #define rmb()	do { if (arch_is_coherent()) dmb(); else barrier(); } while (0)
@@ -152,9 +156,9 @@ extern unsigned int user_debug;
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
 #else
-#define smp_mb()	mb()
-#define smp_rmb()	rmb()
-#define smp_wmb()	wmb()
+#define smp_mb()	dmb()
+#define smp_rmb()	dmb()
+#define smp_wmb()	dmb()
 #endif
 
 #define read_barrier_depends()		do { } while(0)
arch/arm/mm/Kconfig +6 -0
index 55a2a00..5bd7c89 100644
@@ -788,3 +788,9 @@ config ARM_L1_CACHE_SHIFT
 	int
 	default 6 if ARM_L1_CACHE_SHIFT_6
 	default 5
+
+config ARCH_HAS_BARRIERS
+	bool
+	help
+	  This option allows the use of custom mandatory barriers
+	  included via the mach/barriers.h file.
#8 lore raw

PATCH v5 4/4 ARM: Change the mandatory barriers implementation

Quoted 16 lines · Catalin Marinas

The mandatory barriers (mb, rmb, wmb) are used even on uniprocessor systems for things like ordering Normal Non-cacheable memory accesses with DMA transfer (via Device memory writes). The current implementation uses dmb() for mb() and friends but this is not sufficient. The DMB only ensures the relative ordering of the observability of accesses by other processors or devices acting as masters. In case of DMA transfers started by writes to device memory, the relative ordering is not ensured because accesses to slave ports of a device are not considered observable by the DMB definition.

A DSB is required for the data to reach the main memory (even if mapped as Normal Non-cacheable) before the device receives the notification to begin the transfer. Furthermore, some L2 cache controllers (like L2x0 or PL310) buffer stores to Normal Non-cacheable memory and this would need to be drained with the outer_sync() function call.

Quoted 6 lines

The patch also allows platforms to define their own mandatory barriers implementation by selecting CONFIG_ARCH_HAS_BARRIERS and providing a mach/barriers.h file.

Note that the SMP barriers are unchanged (being DMBs as before) since they are only guaranteed to work with Normal Cacheable memory.

Ok.