Skip to contentSkip to conversation

introduce L1_CACHE_SHIFT_MAX

PATCH In review Andrew Morton 2 participants 3 messages Aug 11, 2002

lore
Share & tools
lore

Andrew Morton

author
#0 lore raw

PATCH 19/21 introduce L1_CACHE_SHIFT_MAX

zone->lock and zone->lru_lock are two of the hottest locks in the kernel. Their usage patterns are quite independent. And they have just been put into the same structure. It is essential that they not fall into the same cacheline.

That could be fixed by padding with L1_CACHE_BYTES. But the problem with this is that a kernel which was configured for (say) a PIII will perform poorly on SMP PIV. This will cause problems for kernel vendors. For example, RH currently ship PII and Athlon binaries. To get best SMP performance they will end up needing to ship a lot of differently configured kernels.

To solve this we need to know, at compile time, the maximum L1 size which this kernel will ever run on.

This patch adds L1_CACHE_SHIFT_MAX to every architecture's cache.h.

Of course it'll break when newer chips come out with increased
cacheline sizes. Better suggestions are welcome.

 asm-alpha/cache.h   |    1 +
 asm-arm/cache.h     |    2 ++
 asm-cris/cache.h    |    2 ++
 asm-i386/cache.h    |    2 ++
 asm-ia64/cache.h    |    2 ++
 asm-m68k/cache.h    |    2 ++
 asm-mips/cache.h    |    1 +
 asm-mips64/cache.h  |    1 +
 asm-parisc/cache.h  |    1 +
 asm-ppc/cache.h     |    1 +
 asm-ppc64/cache.h   |    1 +
 asm-s390/cache.h    |    1 +
 asm-s390x/cache.h   |    1 +
 asm-sh/cache.h      |    2 ++
 asm-sparc/cache.h   |    1 +
 asm-sparc64/cache.h |    1 +
 asm-x86_64/cache.h  |    1 +
 linux/cache.h       |    9 +++++++++
 18 files changed, 32 insertions(+)
include/linux/cache.h renamed from include/linux/cache.h~l1-max-size +9 -0
@@ -44,4 +44,13 @@
 #endif /* CONFIG_SMP */
 #endif
 
+#if !defined(____cacheline_maxaligned_in_smp)
+#if defined(CONFIG_SMP)
+#define ____cacheline_maxaligned_in_smp \
+	__attribute__((__aligned__(1 << (L1_CACHE_SHIFT_MAX))))
+#else
+#define ____cacheline_maxaligned_in_smp
+#endif
+#endif
+
 #endif /* __LINUX_CACHE_H */
include/asm-alpha/cache.h renamed from include/asm-alpha/cache.h~l1-max-size +1 -0
@@ -20,5 +20,6 @@
 
 #define L1_CACHE_ALIGN(x)  (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))
 #define SMP_CACHE_BYTES    L1_CACHE_BYTES
+#define L1_CACHE_SHIFT_MAX 6	/* largest L1 which this arch supports */
 
 #endif
include/asm-arm/cache.h renamed from include/asm-arm/cache.h~l1-max-size +2 -0
@@ -16,4 +16,6 @@
 		 __section__(".data.cacheline_aligned")))
 #endif
 
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
+
 #endif
include/asm-cris/cache.h renamed from include/asm-cris/cache.h~l1-max-size +2 -0
@@ -7,4 +7,6 @@
 
 #define L1_CACHE_BYTES 32
 
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
+
 #endif /* _ASM_CACHE_H */
include/asm-i386/cache.h renamed from include/asm-i386/cache.h~l1-max-size +2 -0
@@ -10,4 +10,6 @@
 #define L1_CACHE_SHIFT	(CONFIG_X86_L1_CACHE_SHIFT)
 #define L1_CACHE_BYTES	(1 << L1_CACHE_SHIFT)
 
+#define L1_CACHE_SHIFT_MAX 7	/* largest L1 which this arch supports */
+
 #endif
include/asm-ia64/cache.h renamed from include/asm-ia64/cache.h~l1-max-size +2 -0
@@ -12,6 +12,8 @@
 #define L1_CACHE_SHIFT		CONFIG_IA64_L1_CACHE_SHIFT
 #define L1_CACHE_BYTES		(1 << L1_CACHE_SHIFT)
 
+#define L1_CACHE_SHIFT_MAX 7	/* largest L1 which this arch supports */
+
 #ifdef CONFIG_SMP
 # define SMP_CACHE_SHIFT	L1_CACHE_SHIFT
 # define SMP_CACHE_BYTES	L1_CACHE_BYTES
include/asm-m68k/cache.h renamed from include/asm-m68k/cache.h~l1-max-size +2 -0
@@ -8,4 +8,6 @@
 #define        L1_CACHE_SHIFT  4
 #define        L1_CACHE_BYTES  (1<< L1_CACHE_SHIFT)
 
+#define L1_CACHE_SHIFT_MAX 4	/* largest L1 which this arch supports */
+
 #endif
include/asm-mips/cache.h renamed from include/asm-mips/cache.h~l1-max-size +1 -0
@@ -35,5 +35,6 @@ struct cache_desc {
 #endif
 
 #define SMP_CACHE_BYTES		L1_CACHE_BYTES
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
 
 #endif /* _ASM_CACHE_H */
include/asm-mips64/cache.h renamed from include/asm-mips64/cache.h~l1-max-size +1 -0
@@ -11,5 +11,6 @@
 
 /* bytes per L1 cache line */
 #define L1_CACHE_BYTES		(1 << CONFIG_L1_CACHE_SHIFT)
+#define L1_CACHE_SHIFT_MAX 7	/* largest L1 which this arch supports */
 
 #endif /* _ASM_CACHE_H */
include/asm-parisc/cache.h renamed from include/asm-parisc/cache.h~l1-max-size +1 -0
@@ -34,6 +34,7 @@
 #define L1_CACHE_ALIGN(x)       (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))
 
 #define SMP_CACHE_BYTES L1_CACHE_BYTES
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
 
 #define __cacheline_aligned __attribute__((__aligned__(L1_CACHE_BYTES)))
include/asm-ppc/cache.h renamed from include/asm-ppc/cache.h~l1-max-size +1 -0
@@ -28,6 +28,7 @@
 
 #define	L1_CACHE_BYTES L1_CACHE_LINE_SIZE
 #define	SMP_CACHE_BYTES L1_CACHE_BYTES
+#define L1_CACHE_SHIFT_MAX 7	/* largest L1 which this arch supports */
 
 #define	L1_CACHE_ALIGN(x)       (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))
 #define	L1_CACHE_PAGES		8
include/asm-ppc64/cache.h renamed from include/asm-ppc64/cache.h~l1-max-size +1 -0
@@ -12,5 +12,6 @@
 #define L1_CACHE_BYTES	(1 << L1_CACHE_SHIFT)
 
 #define SMP_CACHE_BYTES L1_CACHE_BYTES
+#define L1_CACHE_SHIFT_MAX 7	/* largest L1 which this arch supports */
 
 #endif
include/asm-s390/cache.h renamed from include/asm-s390/cache.h~l1-max-size +1 -0
@@ -13,5 +13,6 @@
 
 #define L1_CACHE_BYTES     256
 #define L1_CACHE_SHIFT     8
+#define L1_CACHE_SHIFT_MAX 8	/* largest L1 which this arch supports */
 
 #endif
include/asm-s390x/cache.h renamed from include/asm-s390x/cache.h~l1-max-size +1 -0
@@ -13,5 +13,6 @@
 
 #define L1_CACHE_BYTES     256
 #define L1_CACHE_SHIFT     8
+#define L1_CACHE_SHIFT_MAX 8	/* largest L1 which this arch supports */
 
 #endif
include/asm-sh/cache.h renamed from include/asm-sh/cache.h~l1-max-size +2 -0
@@ -14,4 +14,6 @@
 #define        L1_CACHE_BYTES  32
 #endif
 
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
+
 #endif /* __ASM_SH_CACHE_H */
include/asm-sparc/cache.h renamed from include/asm-sparc/cache.h~l1-max-size +1 -0
@@ -13,6 +13,7 @@
 #define L1_CACHE_SHIFT 5
 #define L1_CACHE_BYTES 32
 #define L1_CACHE_ALIGN(x) ((((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1)))
+#define L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
 
 #define SMP_CACHE_BYTES 32
include/asm-sparc64/cache.h renamed from include/asm-sparc64/cache.h~l1-max-size +1 -0
@@ -9,6 +9,7 @@
 #define        L1_CACHE_BYTES	32 /* Two 16-byte sub-blocks per line. */
 
 #define        L1_CACHE_ALIGN(x)       (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))
+#define		L1_CACHE_SHIFT_MAX 5	/* largest L1 which this arch supports */
 
 #define        SMP_CACHE_BYTES_SHIFT	6
 #define        SMP_CACHE_BYTES		(1 << SMP_CACHE_BYTES_SHIFT) /* L2 cache line size. */
include/asm-x86_64/cache.h renamed from include/asm-x86_64/cache.h~l1-max-size +1 -0
@@ -9,5 +9,6 @@
 /* L1 cache line size */
 #define L1_CACHE_SHIFT	(CONFIG_X86_L1_CACHE_SHIFT)
 #define L1_CACHE_BYTES	(1 << L1_CACHE_SHIFT)
+#define L1_CACHE_SHIFT_MAX 6	/* largest L1 which this arch supports */
 
 #endif

.

#1 lore raw
Quoted 12 lines · Andrew Morton

zone->lock and zone->lru_lock are two of the hottest locks in the kernel. Their usage patterns are quite independent. And they have just been put into the same structure. It is essential that they not fall into the same cacheline.

That could be fixed by padding with L1_CACHE_BYTES. But the problem with this is that a kernel which was configured for (say) a PIII will perform poorly on SMP PIV. This will cause problems for kernel vendors. For example, RH currently ship PII and Athlon binaries. To get best SMP performance they will end up needing to ship a lot of differently configured kernels.

Quoted 7 lines

To solve this we need to know, at compile time, the maximum L1 size which this kernel will ever run on.

This patch adds L1_CACHE_SHIFT_MAX to every architecture's cache.h.

Of course it'll break when newer chips come out with increased
cacheline sizes. Better suggestions are welcome.

I think you're being too paranoid. You pushed the performance degradation from the PIV to the PIII (because it will tend to hit more cachelines than it should) and you won't be able to build a kernel that is optimal for the PIII any more. I'd say that is PIII kernel is *supposed* to suck to some degree when run on a PIV, otherwise why bother having the PIV option?

I expect the performance difference you're talking about is marginal anyway. Maybe you've measured it?

Signature · 1 line
Daniel
author
#2 lore raw
Quoted 17 lines · Daniel Phillips
Quoted 15 lines · Andrew Morton

zone->lock and zone->lru_lock are two of the hottest locks in the kernel. Their usage patterns are quite independent. And they have just been put into the same structure. It is essential that they not fall into the same cacheline.

That could be fixed by padding with L1_CACHE_BYTES. But the problem with this is that a kernel which was configured for (say) a PIII will perform poorly on SMP PIV. This will cause problems for kernel vendors. For example, RH currently ship PII and Athlon binaries. To get best SMP performance they will end up needing to ship a lot of differently configured kernels.

To solve this we need to know, at compile time, the maximum L1 size which this kernel will ever run on.

Quoted 6 lines
Quoted 4 lines

This patch adds L1_CACHE_SHIFT_MAX to every architecture's cache.h.

Of course it'll break when newer chips come out with increased
cacheline sizes. Better suggestions are welcome.

I think you're being too paranoid.

Staring at too many horrific profile outputs does that to one.

These are *the* two big locks.

Quoted 3 lines

You pushed the performance degradation
from the PIV to the PIII (because it will tend to hit more cachelines than it should)

The buddy info is all in one cacheline and the LRU info is in another. So there's no loss to PIII here. But those two things are soooo hot that paranoia is warranted.

Quoted 6 lines

and you won't be able to build a kernel that is optimal for the PIII any more. I'd say that is PIII kernel is *supposed* to suck to some degree when run on a PIV, otherwise why bother having the PIV option?

I expect the performance difference you're talking about is marginal anyway. Maybe you've measured it?

No, I haven't. NUMA boxes don't need it if the node-local allocation is working right. But if they go cross-node much, it'll help. On high-performance UMA SMP, allowing those two particular locks to fall into the same cacheline is a big goofup.