ARM: Broadcom Brahma-B15 readahead cache support

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

The readahead cache only intercepts reads, not writes, as such, some data can remain stale in any of its buffers, such that we need to flush it, which is an operation that needs to happen in a particular order:

- disable the readahead cache
- flush it
- call the appropriate cache-v7.S function
- re-enable

This patch tries to minimize the impact to the cache-v7.S file by only providing a stub in case CONFIG_CACHE_B15_RAC is enabled (default for ARCH_BRCMSTB since it is the current user).

Signed-off-by Alamy Liu <[email protected]>

Signed-off-by Florian Fainelli <[email protected]>

---
 arch/arm/include/asm/cacheflush.h             |   2 +-
 arch/arm/include/asm/glue-cache.h             |   4 +
 arch/arm/include/asm/hardware/cache-b15-rac.h |  12 ++
 arch/arm/mm/Kconfig                           |   8 ++
 arch/arm/mm/Makefile                          |   1 +
 arch/arm/mm/cache-b15-rac.c                   | 181 ++++++++++++++++++++++++++
 6 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm/include/asm/hardware/cache-b15-rac.h
 create mode 100644 arch/arm/mm/cache-b15-rac.c

arch/arm/include/asm/cacheflush.h +1 -1

index 2d46862e7bef..4d847e185cf6 100644
@@ -199,7 +199,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
  */
 #if (defined(CONFIG_CPU_V7) && \
      (defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K))) || \
-	defined(CONFIG_SMP_ON_UP)
+	defined(CONFIG_SMP_ON_UP) || defined(CONFIG_CACHE_B15_RAC)
 #define __flush_icache_preferred	__cpuc_flush_icache_all
 #elif __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
 #define __flush_icache_preferred	__flush_icache_all_v7_smp

arch/arm/include/asm/glue-cache.h +4 -0

index a3c24cd5b7c8..11f33b5f9284 100644
@@ -117,6 +117,10 @@
 # endif
 #endif
 
+#if defined(CONFIG_CACHE_B15_RAC)
+# define MULTI_CACHE 1
+#endif
+
 #if defined(CONFIG_CPU_V7M)
 # ifdef _CACHE
 #  define MULTI_CACHE 1

arch/arm/include/asm/hardware/cache-b15-rac.h new file +12 -0

new file mode 100644
index 000000000000..76b888f53f90
@@ -0,0 +1,12 @@
+#ifndef __ASM_ARM_HARDWARE_CACHE_B15_RAC_H
+#define __ASM_ARM_HARDWARE_CACHE_B15_RAC_H
+
+#ifndef __ASSEMBLY__
+
+void b15_flush_kern_cache_all(void);
+void b15_flush_kern_cache_louis(void);
+void b15_flush_icache_all(void);
+
+#endif
+
+#endif

arch/arm/mm/Kconfig +8 -0

index 9b4f29e595a4..4d5652a39304 100644
@@ -853,6 +853,14 @@ config OUTER_CACHE_SYNC
 	  The outer cache has a outer_cache_fns.sync function pointer
 	  that can be used to drain the write buffer of the outer cache.
 
+config CACHE_B15_RAC
+	bool "Enable the Broadcom Brahma-B15 read-ahead cache controller"
+	depends on ARCH_BRCMSTB
+	default y
+	help
+	  This option enables the Broadcom Brahma-B15 read-ahead cache
+	  controller. If disabled, the read-ahead cache remains off.
+
 config CACHE_FEROCEON_L2
 	bool "Enable the Feroceon L2 cache controller"
 	depends on ARCH_MV78XX0 || ARCH_MVEBU

arch/arm/mm/Makefile +1 -0

index d3afdf9eb65a..a6797fdb6721 100644
@@ -96,6 +96,7 @@ AFLAGS_proc-v6.o	:=-Wa,-march=armv6
 AFLAGS_proc-v7.o	:=-Wa,-march=armv7-a
 
 obj-$(CONFIG_OUTER_CACHE)	+= l2c-common.o
+obj-$(CONFIG_CACHE_B15_RAC)	+= cache-b15-rac.o
 obj-$(CONFIG_CACHE_FEROCEON_L2)	+= cache-feroceon-l2.o
 obj-$(CONFIG_CACHE_L2X0)	+= cache-l2x0.o l2c-l2x0-resume.o
 obj-$(CONFIG_CACHE_XSC3L2)	+= cache-xsc3l2.o

arch/arm/mm/cache-b15-rac.c new file +25 -0

new file mode 100644
index 000000000000..1c5bca6e906b
@@ -0,0 +1,181 @@
+/*
+ * Broadcom Brahma-B15 CPU read-ahead cache management functions
+ *
+ * Copyright (C) 2015, Broadcom Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/spinlock.h>
+#include <linux/io.h>
+#include <linux/bitops.h>
+#include <linux/of_address.h>
+
+#include <asm/cacheflush.h>
+#include <asm/hardware/cache-b15-rac.h>
+
+extern void v7_flush_kern_cache_all(void);
+extern void v7_flush_kern_cache_louis(void);
+extern void v7_flush_icache_all(void);
+
+/* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
+#define RAC_CONFIG0_REG			(0x78)

arch/arm/mm/cache-b15-rac.c new file +46 -0

+
+static void __iomem *b15_rac_base;
+static DEFINE_SPINLOCK(rac_lock);
+
+/* Initialization flag to avoid checking for b15_rac_base, and to prevent
+ * multi-platform kernels from crashing here as well.
+ */
+static unsigned long b15_rac_flags;
+
+static inline u32 __b15_rac_disable(void)
+{
+	u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
+	__raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
+	dmb();
+	return val;
+}
+
+static inline void __b15_rac_flush(void)
+{
+	u32 reg;
+
+	__raw_writel(FLUSH_RAC, b15_rac_base + RAC_FLUSH_REG);
+	do {
+		/* This dmb() is required to force the Bus Interface Unit
+		 * to clean oustanding writes, and forces an idle cycle
+		 * to be inserted.
+		 */
+		dmb();
+		reg = __raw_readl(b15_rac_base + RAC_FLUSH_REG);
+	} while (reg & RAC_FLUSH_REG);
+}
+
+static inline u32 b15_rac_disable_and_flush(void)
+{
+	u32 reg;
+
+	reg = __b15_rac_disable();
+	__b15_rac_flush();
+	return reg;
+}
+
+static inline void __b15_rac_enable(u32 val)
+{
+	__raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
+	/* dsb() is required here to be consistent with __flush_icache_all() */
+	dsb();

Quoted 25 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

The readahead cache only intercepts reads, not writes, as such, some data can remain stale in any of its buffers, such that we need to flush it, which is an operation that needs to happen in a particular order:

- disable the readahead cache
- flush it
- call the appropriate cache-v7.S function
- re-enable

This patch tries to minimize the impact to the cache-v7.S file by only providing a stub in case CONFIG_CACHE_B15_RAC is enabled (default for ARCH_BRCMSTB since it is the current user).

Signed-off-by Alamy Liu <[email protected]>

Signed-off-by Florian Fainelli <[email protected]>

---

arch/arm/mm/cache-b15-rac.c line 43

Quoted 7 lines

arch/arm/mm/cache-b15-rac.c new file +7 -0

+/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
+#define RAC_DATA_INST_EN_MASK		(1 << RACPREFINST_SHIFT | \
+					 RACENPREF_MASK << RACENINST_SHIFT | \
+					 1 << RACPREFDATA_SHIFT | \
+					 RACENPREF_MASK << RACENDATA_SHIFT)
+
+#define RAC_ENABLED			(1 << 0)

					BIT(0) ?

However, you don't use RAC_ENABLED as a bitmask, but a bit index, so shouldn't this be zero?

arch/arm/mm/cache-b15-rac.c line 99

Quoted 8 lines

arch/arm/mm/cache-b15-rac.c new file +8 -0

+#define BUILD_RAC_CACHE_OP(name, bar)				\
+void b15_flush_##name(void)					\
+{								\
+	unsigned int do_flush;					\
+	u32 val = 0;						\
+								\
+	spin_lock(&rac_lock);					\
+	do_flush = test_bit(RAC_ENABLED, &b15_rac_flags);	\

Do you need to use test_bit() here? You set and test this location under a spinlock, so it's safe to use non-atomic ops here.

arch/arm/mm/cache-b15-rac.c line 139

Quoted 8 lines

arch/arm/mm/cache-b15-rac.c new file +7 -0

+static void b15_rac_enable(void)
+{
+	unsigned int cpu;
+	u32 enable = 0;
+
+	for_each_possible_cpu(cpu)
+		enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));

		enable |= RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT);

You don't need the additional parens - the right hand side of |= is already expected to be an expression by the compiler.

arch/arm/mm/cache-b15-rac.c line 167

Quoted 4 lines

arch/arm/mm/cache-b15-rac.c new file +4 -0

+	spin_lock(&rac_lock);
+	reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
+	for_each_possible_cpu(cpu)
+		en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));

		en_mask |= 1 << (RACPREFDATA_SHIFT + cpu * RAC_CPU_SHIFT);

looks nicer, rather than having two shifts.

What happens when the system goes down (eg, for kexec?) Does the RAC need to be disabled for that?

Quoted 26 lines · Russell King - ARM Linux

Quoted 25 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

The readahead cache only intercepts reads, not writes, as such, some data can remain stale in any of its buffers, such that we need to flush it, which is an operation that needs to happen in a particular order:

- disable the readahead cache
- flush it
- call the appropriate cache-v7.S function
- re-enable

This patch tries to minimize the impact to the cache-v7.S file by only providing a stub in case CONFIG_CACHE_B15_RAC is enabled (default for ARCH_BRCMSTB since it is the current user).

Signed-off-by Alamy Liu <[email protected]>

Signed-off-by Florian Fainelli <[email protected]>

---

[snip]

Quoted 10 lines

arch/arm/mm/cache-b15-rac.c line 43

Quoted 7 lines

arch/arm/mm/cache-b15-rac.c new file +7 -0

+/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
+#define RAC_DATA_INST_EN_MASK		(1 << RACPREFINST_SHIFT | \
+					 RACENPREF_MASK << RACENINST_SHIFT | \
+					 1 << RACPREFDATA_SHIFT | \
+					 RACENPREF_MASK << RACENDATA_SHIFT)
+
+#define RAC_ENABLED			(1 << 0)

					BIT(0) ?

Quoted 2 lines

However, you don't use RAC_ENABLED as a bitmask, but a bit index, so shouldn't this be zero?

In subsequent patches we have a need for distinguishing RAC_ENABLED from RAC_SUSPENDED, so that's the primary reason for using it as a bitmask (could make that clear somewhere).

Quoted 2 lines

What happens when the system goes down (eg, for kexec?) Does the RAC need to be disabled for that?

Per boot convention, I would say so, yes, since this is another level of instruction and data cache, we should turn it off. Can we register some sort of notifier specifically for kexec?

Quoted 40 lines · Florian Fainelli

Quoted 26 lines · Russell King - ARM Linux

Quoted 25 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

The readahead cache only intercepts reads, not writes, as such, some data can remain stale in any of its buffers, such that we need to flush it, which is an operation that needs to happen in a particular order:

- disable the readahead cache
- flush it
- call the appropriate cache-v7.S function
- re-enable

This patch tries to minimize the impact to the cache-v7.S file by only providing a stub in case CONFIG_CACHE_B15_RAC is enabled (default for ARCH_BRCMSTB since it is the current user).

Signed-off-by Alamy Liu <[email protected]>

Signed-off-by Florian Fainelli <[email protected]>

---

[snip]

Quoted 10 lines

arch/arm/mm/cache-b15-rac.c line 43

Quoted 7 lines

arch/arm/mm/cache-b15-rac.c new file +7 -0

+/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
+#define RAC_DATA_INST_EN_MASK		(1 << RACPREFINST_SHIFT | \
+					 RACENPREF_MASK << RACENINST_SHIFT | \
+					 1 << RACPREFDATA_SHIFT | \
+					 RACENPREF_MASK << RACENDATA_SHIFT)
+
+#define RAC_ENABLED			(1 << 0)

					BIT(0) ?

Quoted 6 lines

Quoted 2 lines

However, you don't use RAC_ENABLED as a bitmask, but a bit index, so shouldn't this be zero?

In subsequent patches we have a need for distinguishing RAC_ENABLED from RAC_SUSPENDED, so that's the primary reason for using it as a bitmask (could make that clear somewhere).

However, test_bit() etc take a bit _number_ not a bit _mask_. So:

Passing in 1 << 0 will test bit 1 rather than bit 0. Passing in 1 << 1 will test bit 2 rather than bit 1. Passing in 1 << 2 will test bit 4 rather than bit 2. Passing in 1 << 3 will test bit 8 rather than bit 3. etc.

This is not what you wanted. Either use a mask directly, or use test_bit() with a bit number etc. Don't try and do both together. :)

Quoted 7 lines

Quoted 3 lines

What happens when the system goes down (eg, for kexec?) Does the RAC need to be disabled for that?

Per boot convention, I would say so, yes, since this is another level of instruction and data cache, we should turn it off. Can we register some sort of notifier specifically for kexec?

The code at present doesn't expect there to be platform specific caches, so that probably isn't catered for yet. I mentioned the point to raise the issue that there's an oversight here.

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

Quoted 5 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

Quoted 3 lines

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

I'm struggling to understand why you care about flush_kern_cache_louis and flush_icache_all for a cache that sits the other side of the L2.

Can you explain why we need to do anything in these cases, please?

Quoted 6 lines · Will Deacon

Quoted 5 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

Quoted 8 lines

Quoted 3 lines

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

I'm struggling to understand why you care about flush_kern_cache_louis and flush_icache_all for a cache that sits the other side of the L2.

Can you explain why we need to do anything in these cases, please?

Let's try, as you may have read in the comment, all MVA-based cache maintenance operations are snooped by the RAC, so they are effectively "transparent" to software, all others are not.

flush_kern_cache_louis() and flush_icache_all() both use ICALLIUS in the SMP case and ICIALLU in the UP case which were flagged as not being transparently handled.

The concern is that, if you perform a L1 cache (data or instruction) flush (essentially an invalidate), this will also flush (invalidate) corresponding L2 cache lines, but the RAC has no way to be signaled that is should also invalidate its own RAC cache lines pertaining to that data, and RAC holds per-CPU "super" cache lines.

In arch/arm/kernel/smp.c, all uses of flush_cache_louis() are for writing-back data, so the RAC is not an issue. In arch/arm/kernel/suspend.c, flush_cache_louis() is known not to guarantee a "clean" all the way to main memory, so __cpu_flush_dcache_area is used in conjunction. In arch/arm/mm/idmap.c and mmu.c, the use of flush_cache_louis() seems to be meant to see fresh data, not write-back, so not transparent to the RAC, is that right?

It may very well be that we are super cautious here and that the only case to take care of is essentially flush_cache_all(), and nothing more.

Would you suggestions on how to instrument/exercise whether we really need to deal with flush_cache_louis() and flush_icache_all()?

Quoted 41 lines · Florian Fainelli

Quoted 6 lines · Will Deacon

Quoted 5 lines · Florian Fainelli

This patch adds support for the Broadcom Brahma-B15 CPU readahead cache controller. This cache controller sits between the L2 and the memory bus and its purpose is to provide a friendler burst size towards the DDR interface than the native cache line size.

Quoted 8 lines

Quoted 3 lines

The readahead cache is mostly transparent, except for flush_kern_cache_all, flush_kern_cache_louis and flush_icache_all, which is precisely what we are overriding here.

I'm struggling to understand why you care about flush_kern_cache_louis and flush_icache_all for a cache that sits the other side of the L2.

Can you explain why we need to do anything in these cases, please?

Let's try, as you may have read in the comment, all MVA-based cache maintenance operations are snooped by the RAC, so they are effectively "transparent" to software, all others are not.

flush_kern_cache_louis() and flush_icache_all() both use ICALLIUS in the SMP case and ICIALLU in the UP case which were flagged as not being transparently handled.

The concern is that, if you perform a L1 cache (data or instruction) flush (essentially an invalidate), this will also flush (invalidate) corresponding L2 cache lines, but the RAC has no way to be signaled that is should also invalidate its own RAC cache lines pertaining to that data, and RAC holds per-CPU "super" cache lines.

In arch/arm/kernel/smp.c, all uses of flush_cache_louis() are for writing-back data, so the RAC is not an issue. In arch/arm/kernel/suspend.c, flush_cache_louis() is known not to guarantee a "clean" all the way to main memory, so __cpu_flush_dcache_area is used in conjunction. In arch/arm/mm/idmap.c and mmu.c, the use of flush_cache_louis() seems to be meant to see fresh data, not write-back, so not transparent to the RAC, is that right?

It may very well be that we are super cautious here and that the only case to take care of is essentially flush_cache_all(), and nothing more.

Quoted 2 lines

Would you suggestions on how to instrument/exercise whether we really need to deal with flush_cache_louis() and flush_icache_all()?

I think that both flush_cache_louis and flush_icache_all only care about the inner-shareable domain, so you don't need to do anything with the RAC. It's a bit like the PL310 outer-cache, which is also not affected by these operations.

I don't think there's a good way to determine statically if we have missing cacheflush calls. Maybe a better bet would be to implement a RAC driver using the outer_cache framework and only implement the flush_all callback.

Hi all,

This patch series adds support for the Broadcom Brahma-B15 CPU read-ahead cache which is to be found in several Broadcom SoCs, and mainline brcmstb.

Let me know your thoughts on this, we have tried to minimize both the compile-time and runtime impact for non-Broadcom CPU parts.

Florian Fainelli (5):
ARM: v7: allow setting different cache functions
ARM: Add Broadcom Brahma-B15 readahead cache support
ARM: Hook B15 readahead cache functions based on processor
ARM: B15: Add CPU hotplug awareness
ARM: B15: Add suspend/resume hooks

 arch/arm/include/asm/cacheflush.h             |   2 +-
 arch/arm/include/asm/glue-cache.h             |   4 +
 arch/arm/include/asm/hardware/cache-b15-rac.h |  12 +
 arch/arm/mm/Kconfig                           |   8 +
 arch/arm/mm/Makefile                          |   1 +
 arch/arm/mm/cache-b15-rac.c                   | 329 ++++++++++++++++++++++++++
 arch/arm/mm/cache-v7.S                        |  21 ++
 arch/arm/mm/proc-v7.S                         |   6 +-
 8 files changed, 379 insertions(+), 4 deletions(-)
 create mode 100644 arch/arm/include/asm/hardware/cache-b15-rac.h
 create mode 100644 arch/arm/mm/cache-b15-rac.c