Change maxaligned_in_smp alignemnt macros to internodealigned_in_smp macros
Subsystems (6)
PATCH 1/2 Change maxaligned_in_smp alignemnt macros to internodealigned_in_smp macros
Andrew,
The following 2 patches changes maxaligned macros to internodealigned macros, and kills L1_CACHE_SHIFT_MAX. Please consider for to -mm.
Thanks,
Kiran
---
____cacheline_maxaligned_in_smp is currently used to align critical structures and avoid false sharing. It uses per-arch L1_CACHE_SHIFT_MAX and people find L1_CACHE_SHIFT_MAX useless.
However, we have been using ____cacheline_maxaligned_in_smp to align structures on the internode cacheline size. As per Andi's suggestion, following patch kills ____cacheline_maxaligned_in_smp and introduces INTERNODE_CACHE_SHIFT, which defaults to L1_CACHE_SHIFT for all arches. Arches needing L3/Internode cacheline alignment can define INTERNODE_CACHE_SHIFT in the arch asm/cache.h. Patch replaces ____cacheline_maxaligned_in_smp with ____cacheline_internodealigned_in_smp
With this patch, L1_CACHE_SHIFT_MAX can be killed
Signed-off-by Ravikiran Thirumalai <[email protected]>
Signed-off-by Shai Fultheim <[email protected]>
Signed-off-by Andi Kleen <[email protected]>
arch/i386/kernel/init_task.c +1 -1
@@ -42,5 +42,5 @@* per-CPU TSS segments. Threads are completely 'soft' on Linux, * no more per-task TSS's. */ -DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_maxaligned_in_smp = INIT_TSS; +DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_internodealigned_in_smp = INIT_TSS;
arch/i386/kernel/irq.c +1 -1
@@ -19,7 +19,7 @@#include <linux/cpu.h> #include <linux/delay.h> -DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_maxaligned_in_smp; +DEFINE_PER_CPU (irq_cpustat_t, irq_stat)____cacheline_internodealigned_in_smp ;EXPORT_PER_CPU_SYMBOL (irq_stat);#ifndef CONFIG_X86_LOCAL_APIC
arch/x86_64/kernel/init_task.c +1 -1
@@ -44,6 +44,6 @@* section. Since TSS's are completely CPU-local, we want them * on exact cacheline boundaries, to eliminate cacheline ping-pong. */ -DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_maxaligned_in_smp = INIT_TSS; +DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_internodealigned_in_smp = INIT_TSS;#define ALIGN_TO_4K __attribute__((section(".data.init_task")))
include/linux/cache.h +13 -4
@@ -45,12 +45,21 @@#endif /* CONFIG_SMP */ #endif -#if !defined(____cacheline_maxaligned_in_smp) +/* +* The maximum alignment needed for some critical structures +* These could be inter-node cacheline sizes/L3 cacheline +* size etc. Define this in asm/cache.h for your arch +*/ +#ifndef INTERNODE_CACHE_SHIFT +#define INTERNODE_CACHE_SHIFT L1_CACHE_SHIFT +#endif + +#if !defined(____cacheline_internodealigned_in_smp)#if defined(CONFIG_SMP ) -#define ____cacheline_maxaligned_in_smp \ - __attribute__((__aligned__(1 << (L1_CACHE_SHIFT_MAX)))) +#define ____cacheline_internodealigned_in_smp \ + __attribute__((__aligned__(1 << (INTERNODE_CACHE_SHIFT))))#else -#define ____cacheline_maxaligned_in_smp +#define ____cacheline_internodealigned_in_smp#endif #endif
include/linux/ide.h +1 -1
@@ -922,7 +922,7 @@unsigned dma;void (*led_act)(void *data,int rw); -}____cacheline_maxaligned_in_smp ide_hwif_t; +}____cacheline_internodealigned_in_smp ide_hwif_t;/* * internal ide interrupt handler type
include/linux/mmzone.h +2 -2
@@ -38,7 +38,7 @@#if defined(CONFIG_SMP )struct zone_padding {char x[0 ]; -} ____cacheline_maxaligned_in_smp; +}____cacheline_internodealigned_in_smp ;#define ZONE_PADDING (name) struct zone_padding name;#else #define ZONE_PADDING (name)
@@ -233,7 +233,7 @@* rarely used fields: */ char *name; -} ____cacheline_maxaligned_in_smp; +}____cacheline_internodealigned_in_smp ;/*
include/linux/rcupdate.h +1 -1
@@ -65,7 +65,7 @@long cur;/* Current batch number. */ long completed;/* Number of the last completed batch */ int next_pending;/* Is the next batch already waiting? */ -} ____cacheline_maxaligned_in_smp; +}____cacheline_internodealigned_in_smp ;/* Is batch a before batch b ? */ static inline int rcu_batch_before (long a,long b)
kernel/rcupdate.c +2 -2
@@ -61,9 +61,9 @@/* for current batch to proceed. */ }; -static struct rcu_state rcu_state ____cacheline_maxaligned_in_smp = +static struct rcu_state rcu_state____cacheline_internodealigned_in_smp = {.lock =SPIN_LOCK_UNLOCKED , .cpumask =CPU_MASK_NONE }; -static struct rcu_state rcu_bh_state ____cacheline_maxaligned_in_smp = +static struct rcu_state rcu_bh_state____cacheline_internodealigned_in_smp = {.lock =SPIN_LOCK_UNLOCKED , .cpumask =CPU_MASK_NONE }; DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
mm/sparse.c +2 -2
@@ -18,10 +18,10 @@*/ #ifdef CONFIG_SPARSEMEM_EXTREME struct mem_section *mem_section[NR_SECTION_ROOTS ] - ____cacheline_maxaligned_in_smp; +____cacheline_internodealigned_in_smp ;#else struct mem_section mem_section[NR_SECTION_ROOTS ][SECTIONS_PER_ROOT ] - ____cacheline_maxaligned_in_smp; +____cacheline_internodealigned_in_smp ;#endif EXPORT_SYMBOL (mem_section);
PATCH 2/2 Kill L1_CACHE_SHIFT_MAX
Patch to kill L1_CACHE_SHIFT from all arches.
Since L1_CACHE_SHIFT_MAX is not used anymore with the introduction of INTERNODE_CACHE, kill L1_CACHE_SHIFT_MAX.
Signed-off-by Ravikiran Thirumalai <[email protected]>
Signed-off-by Shai Fultheim <[email protected]>
Signed-off-by Andi Kleen <[email protected]>
include/asm-alpha/cache.h +0 -1
@@ -20,6 +20,5 @@#define L1_CACHE_ALIGN (x) (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))#define SMP_CACHE_BYTES L1_CACHE_BYTES -#define L1_CACHE_SHIFT_MAX L1_CACHE_SHIFT#endif
include/asm-arm/cache.h +0 -5
@@ -7,9 +7,4 @@#define L1_CACHE_SHIFT 5#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -/* -* largest L1 which this arch supports -*/ -#define L1_CACHE_SHIFT_MAX 5 -#endif
include/asm-cris/arch-v10/cache.h +0 -1
@@ -4,6 +4,5 @@/* Etrax 100LX have 32-byte cache-lines. */ #define L1_CACHE_BYTES 32#define L1_CACHE_SHIFT 5 -#define L1_CACHE_SHIFT_MAX 5#endif /* _ASM_ARCH_CACHE_H */
include/asm-cris/arch-v32/cache.h +0 -1
@@ -4,6 +4,5 @@/* A cache-line is 32 bytes. */ #define L1_CACHE_BYTES 32#define L1_CACHE_SHIFT 5 -#define L1_CACHE_SHIFT_MAX 5#endif /* _ASM_CRIS_ARCH_CACHE_H */
include/asm-cris/dma-mapping.h +1 -1
@@ -153,7 +153,7 @@static inline int dma_get_cache_alignment (void ) { -return (1 <<L1_CACHE_SHIFT_MAX ); +return (1 <<INTERNODE_CACHE_SHIFT ); }#define dma_is_consistent (d) (1)
include/asm-generic/dma-mapping.h +1 -1
@@ -274,7 +274,7 @@ {/* no easy way to get cache size on all processors, so return * the maximum possible, to be safe */ -return (1 <<L1_CACHE_SHIFT_MAX ); +return (1 <<INTERNODE_CACHE_SHIFT ); } static inline void
include/asm-i386/cache.h +0 -2
@@ -10,6 +10,4 @@#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 7/* largest L1 which this arch supports */ -#endif
include/asm-ia64/cache.h +0 -2
@@ -12,8 +12,6 @@#define L1_CACHE_SHIFT CONFIG_IA64_L1_CACHE_SHIFT#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 7/* largest L1 which this arch supports */ -#ifdef CONFIG_SMP # define SMP_CACHE_SHIFT L1_CACHE_SHIFT# define SMP_CACHE_BYTES L1_CACHE_BYTES
include/asm-m32r/cache.h +0 -2
@@ -7,6 +7,4 @@#define L1_CACHE_SHIFT 4#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 4 -#endif /* _ASM_M32R_CACHE_H */
include/asm-m68k/cache.h +0 -2
@@ -8,6 +8,4 @@#define L1_CACHE_SHIFT 4#define L1_CACHE_BYTES (1<< L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 4/* largest L1 which this arch supports */ -#endif
include/asm-mips/cache.h +0 -1
@@ -15,7 +15,6 @@#define L1_CACHE_SHIFT CONFIG_MIPS_L1_CACHE_SHIFT#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 6#define SMP_CACHE_SHIFT L1_CACHE_SHIFT#define SMP_CACHE_BYTES L1_CACHE_BYTES
include/asm-parisc/cache.h +0 -1
@@ -28,7 +28,6 @@#define L1_CACHE_ALIGN (x) (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))#define SMP_CACHE_BYTES L1_CACHE_BYTES -#define L1_CACHE_SHIFT_MAX 5/* largest L1 which this arch supports */ extern void flush_data_cache_local (void );/* flushes local data-cache only */ extern void flush_instruction_cache_local (void );/* flushes local code-cache only */
include/asm-powerpc/cache.h +0 -1
@@ -19,7 +19,6 @@#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)#define SMP_CACHE_BYTES L1_CACHE_BYTES -#define L1_CACHE_SHIFT_MAX 7/* largest L1 which this arch supports */ #if defined(__powerpc64__) && !defined(__ASSEMBLY__) struct ppc64_caches {
include/asm-s390/cache.h +0 -1
@@ -13,7 +13,6 @@#define L1_CACHE_BYTES 256#define L1_CACHE_SHIFT 8 -#define L1_CACHE_SHIFT_MAX 8/* largest L1 which this arch supports */ #define ARCH_KMALLOC_MINALIGN 8
include/asm-sh/cache.h +0 -2
@@ -22,8 +22,6 @@#define L1_CACHE_ALIGN (x) (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1)) -#define L1_CACHE_SHIFT_MAX 5/* largest L1 which this arch supports */ -struct cache_info {unsigned int ways;unsigned int sets;
include/asm-sh64/cache.h +0 -2
@@ -20,8 +20,6 @@#define L1_CACHE_ALIGN_MASK (~(L1_CACHE_BYTES - 1))#define L1_CACHE_ALIGN (x) (((x)+(L1_CACHE_BYTES - 1)) & L1_CACHE_ALIGN_MASK)#define L1_CACHE_SIZE_BYTES (L1_CACHE_BYTES << 10) -/* Largest L1 which this arch supports */ -#define L1_CACHE_SHIFT_MAX 5#ifdef MODULE #define __cacheline_aligned __attribute__((__aligned__(L1_CACHE_BYTES)))
include/asm-sparc/cache.h +0 -1
@@ -13,7 +13,6 @@#define L1_CACHE_SHIFT 5#define L1_CACHE_BYTES 32#define L1_CACHE_ALIGN (x) ((((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1))) -#define L1_CACHE_SHIFT_MAX 5/* largest L1 which this arch supports */ #define SMP_CACHE_BYTES 32
include/asm-sparc64/cache.h +0 -1
@@ -9,7 +9,6 @@#define L1_CACHE_BYTES 32/* Two 16-byte sub-blocks per line. */ #define L1_CACHE_ALIGN (x) (((x)+(L1_CACHE_BYTES-1))&~(L1_CACHE_BYTES-1)) -#define L1_CACHE_SHIFT_MAX 5/* largest L1 which this arch supports */ #define SMP_CACHE_BYTES_SHIFT 6#define SMP_CACHE_BYTES (1 << SMP_CACHE_BYTES_SHIFT)/* L2 cache line size. */
include/asm-um/cache.h +0 -3
@@ -13,9 +13,6 @@# define L1_CACHE_SHIFT 5#endif -/* XXX: this is valid for x86 and x86_64. */ -#define L1_CACHE_SHIFT_MAX 7/* largest L1 which this arch supports */ -#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)#endif
include/asm-v850/cache.h +0 -2
@@ -23,6 +23,4 @@#define L1_CACHE_SHIFT 4#endif -#define L1_CACHE_SHIFT_MAX L1_CACHE_SHIFT -#endif /* __V850_CACHE_H__ */
include/asm-x86_64/cache.h +0 -1
@@ -9,6 +9,5 @@/* L1 cache line size */ #define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define L1_CACHE_SHIFT_MAX 7/* largest L1 which this arch supports */ #endif