Skip to contentSkip to conversation

ARM: cache-l2x0: make better use of background cache handling

RFC PATCH New Russell King - ARM Linux 2 participants 4 messages Dec 12, 2009 – Dec 14, 2009 Arm (32-bit)

lore
Share & tools
lore
#0 lore raw

RFC PATCH 2/2 ARM: cache-l2x0: make better use of background cache handling

There's no point having the hardware support background operations if we issue a cache operation, and then wait for it to complete before calculating the address of the next operation. We gain no advantage in the cache controller stalling the bus until completion.

What we should be doing is using the 'wait' time productively by calculating the address of the next operation, and only then waiting for the previous operation to complete. This means that cache operations can occur in parallel with the CPU calculating the next address.

---
 arch/arm/mm/cache-l2x0.c |   34 +++++++++++++++++++++++-----------
 1 files changed, 23 insertions(+), 11 deletions(-)
arch/arm/mm/cache-l2x0.c +23 -11
index c1b7bff..ec85dda 100644
@@ -28,18 +28,18 @@
 static void __iomem *l2x0_base;
 static DEFINE_SPINLOCK(l2x0_lock);
 
-static inline void sync_writel(unsigned long val, unsigned long reg,
-			       unsigned long complete_mask)
+static inline void cache_wait(void __iomem *reg, unsigned long mask)
 {
-	writel(val, l2x0_base + reg);
 	/* wait for the operation to complete */
-	while (readl(l2x0_base + reg) & complete_mask)
+	while (readl(reg) & mask)
 		;
 }
 
 static inline void cache_sync(void)
 {
-	sync_writel(0, L2X0_CACHE_SYNC, 1);
+	void __iomem *base = l2x0_base;
+	writel(0, base + L2X0_CACHE_SYNC);
+	cache_wait(base + L2X0_CACHE_SYNC, 1);
 }
 
 static inline void l2x0_inv_all(void)
@@ -48,32 +48,37 @@ static inline void l2x0_inv_all(void)
 
 	/* invalidate all ways */
 	spin_lock_irqsave(&l2x0_lock, flags);
-	sync_writel(0xff, L2X0_INV_WAY, 0xff);
+	writel(0xff, l2x0_base + L2X0_INV_WAY);
+	cache_wait(l2x0_base + L2X0_INV_WAY, 0xff);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_inv_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
 	if (start & (CACHE_LINE_SIZE - 1)) {
 		start &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 		start += CACHE_LINE_SIZE;
 	}
 
 	if (end & (CACHE_LINE_SIZE - 1)) {
 		end &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(end, base + L2X0_CLEAN_INV_LINE_PA);
 	}
 
 	while (start < end) {
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_INV_LINE_PA, 1);
+			cache_wait(base + L2X0_INV_LINE_PA, 1);
+			writel(start, base + L2X0_INV_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -82,12 +87,14 @@ static void l2x0_inv_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_INV_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_clean_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
@@ -96,7 +103,8 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_CLEAN_LINE_PA, 1);
+			cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+			writel(start, base + L2X0_CLEAN_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -105,12 +113,14 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
 static void l2x0_flush_range(unsigned long start, unsigned long end)
 {
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
@@ -119,7 +129,8 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+			cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+			writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 			start += CACHE_LINE_SIZE;
 		}
 
@@ -128,6 +139,7 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
 			spin_lock_irqsave(&l2x0_lock, flags);
 		}
 	}
+	cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
 	cache_sync();
 	spin_unlock_irqrestore(&l2x0_lock, flags);
 }
Signature · 1 line
1.6.2.5
#1 lore raw

RFC PATCH 2/2 ARM: cache-l2x0: make better use of background cache handling

Quoted 10 lines · Russell King - ARM Linux

There's no point having the hardware support background operations if we issue a cache operation, and then wait for it to complete before calculating the address of the next operation. We gain no advantage in the cache controller stalling the bus until completion.

What we should be doing is using the 'wait' time productively by calculating the address of the next operation, and only then waiting for the previous operation to complete. This means that cache operations can occur in parallel with the CPU calculating the next address.

[...]

Quoted 10 lines

static void l2x0_inv_range(unsigned long start, unsigned long end) {

arch/arm/mm/cache-l2x0.c +2 -1
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
 	if (start & (CACHE_LINE_SIZE - 1)) {
 		start &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);

arch/arm/mm/cache-l2x0.c static inline void l2x0_inv_all(void) · line 74

Quoted 10 lines
arch/arm/mm/cache-l2x0.c +3 -1
+		writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 		start += CACHE_LINE_SIZE;
 	}
 
 	if (end & (CACHE_LINE_SIZE - 1)) {
 		end &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(end, base + L2X0_CLEAN_INV_LINE_PA);
 	}

Do we need a cache_wait() on L2X0_CLEAN_INV_LINE_PA here? Actually, do we need the cache_wait() before writel()? Can we have it after?

arch/arm/mm/cache-l2x0.c static inline void l2x0_inv_all(void) · line 82

Quoted 8 lines
arch/arm/mm/cache-l2x0.c +2 -1
 	while (start < end) {
 		unsigned long blk_end = start + min(end - start, 4096UL);
 
 		while (start < blk_end) {
-			sync_writel(start, L2X0_INV_LINE_PA, 1);
+			cache_wait(base + L2X0_INV_LINE_PA, 1);
+			writel(start, base + L2X0_INV_LINE_PA);
 			start += CACHE_LINE_SIZE;

This one waits on a different register than the one written above.

Signature · 1 line
Catalin
#2 lore raw

RFC PATCH 2/2 ARM: cache-l2x0: make better use of background cache handling

Quoted 26 lines · Catalin Marinas
Quoted 10 lines · Russell King - ARM Linux

There's no point having the hardware support background operations if we issue a cache operation, and then wait for it to complete before calculating the address of the next operation. We gain no advantage in the cache controller stalling the bus until completion.

What we should be doing is using the 'wait' time productively by calculating the address of the next operation, and only then waiting for the previous operation to complete. This means that cache operations can occur in parallel with the CPU calculating the next address.

[...]

Quoted 14 lines

static void l2x0_inv_range(unsigned long start, unsigned long end) {

arch/arm/mm/cache-l2x0.c +3 -1
+	void __iomem *base = l2x0_base;
 	unsigned long flags;
 
 	spin_lock_irqsave(&l2x0_lock, flags);
 	if (start & (CACHE_LINE_SIZE - 1)) {
 		start &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(start, base + L2X0_CLEAN_INV_LINE_PA);
 		start += CACHE_LINE_SIZE;
 	}
Quoted 9 lines

arch/arm/mm/cache-l2x0.c static inline void l2x0_inv_all(void) · line 74

Quoted 6 lines
arch/arm/mm/cache-l2x0.c +2 -1
 	if (end & (CACHE_LINE_SIZE - 1)) {
 		end &= ~(CACHE_LINE_SIZE - 1);
-		sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+		cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+		writel(end, base + L2X0_CLEAN_INV_LINE_PA);
 	}

Do we need a cache_wait() on L2X0_CLEAN_INV_LINE_PA here? Actually, do we need the cache_wait() before writel()? Can we have it after?

I'm afraid that you've completely missed the whole point of this patch as you've made that statement.

The point here is that rather than spinning on the L2 controller, waiting for it to say that it's completed, and _then_ calculating the data for the next operation, we do the two operations in parallel - while the L2 controller is busy, we can be preparing the next operation.

Now, as for "do we need a cache_wait() on L2X0_CLEAN_INV_LINE_PA", I've no idea.

The specs are not entirely clear, but from my reading of them, _any_ "register 7" read will do. What I need is for someone to talk to the hardware people to confirm that - for PL210, PL220 and PL310. You will find further details on this in my email to you on the 19th November with the subject of "cache-l2x0.c".

#3 lore raw

RFC PATCH 2/2 ARM: cache-l2x0: make better use of backgroundcache handling

Quoted 37 lines · Russell King - ARM Linux
Quoted 33 lines · Catalin Marinas
Quoted 10 lines · Russell King - ARM Linux

There's no point having the hardware support background operations if we issue a cache operation, and then wait for it to complete before calculating the address of the next operation. We gain no advantage in the cache controller stalling the bus until completion.

What we should be doing is using the 'wait' time productively by calculating the address of the next operation, and only then waiting for the previous operation to complete. This means that cache operations can occur in parallel with the CPU calculating the next address.

[...]

Quoted 10 lines

static void l2x0_inv_range(unsigned long start, unsigned long end) {

arch/arm/mm/cache-l2x0.c +2 -1
+   void __iomem *base = l2x0_base;
    unsigned long flags;
 
    spin_lock_irqsave(&l2x0_lock, flags);
    if (start & (CACHE_LINE_SIZE - 1)) {
            start &= ~(CACHE_LINE_SIZE - 1);
-           sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1);
+           cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);

arch/arm/mm/cache-l2x0.c static inline void l2x0_inv_all(void) · line 74

Quoted 10 lines
arch/arm/mm/cache-l2x0.c +3 -1
+           writel(start, base + L2X0_CLEAN_INV_LINE_PA);
            start += CACHE_LINE_SIZE;
    }
 
    if (end & (CACHE_LINE_SIZE - 1)) {
            end &= ~(CACHE_LINE_SIZE - 1);
-           sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1);
+           cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+           writel(end, base + L2X0_CLEAN_INV_LINE_PA);
    }
Quoted 2 lines

Do we need a cache_wait() on L2X0_CLEAN_INV_LINE_PA here? Actually, do we need the cache_wait() before writel()? Can we have it after?

Quoted 7 lines

I'm afraid that you've completely missed the whole point of this patch as you've made that statement.

The point here is that rather than spinning on the L2 controller, waiting for it to say that it's completed, and _then_ calculating the data for the next operation, we do the two operations in parallel - while the L2 controller is busy, we can be preparing the next operation.

I got this point and it makes sense in the loop but for these two cases there was no loop (and all the loops have an additional cache_wait at the end).

Quoted 8 lines

Now, as for "do we need a cache_wait() on L2X0_CLEAN_INV_LINE_PA", I've no idea.

The specs are not entirely clear, but from my reading of them, _any_ "register 7" read will do. What I need is for someone to talk to the hardware people to confirm that - for PL210, PL220 and PL310. You will find further details on this in my email to you on the 19th November with the subject of "cache-l2x0.c".

OK, I think the spec is pretty clear that there is only one C bit for background operation in progress and it can be read from several registers. I just confirmed with the hardware people and the C bit is indeed common to all registers. In this case:

Signature · 1 line
Catalin