percpu: align percpu readmostly subsection to cacheline
Subsystems (12)
PATCH RESEND 1/1 percpu: align percpu readmostly subsection to cacheline
Currently percpu readmostly subsection may share cachelines with other percpu subsections which may result in unnecessary cacheline bounce and performance degradation.
This patch adds @cacheline parameter to PERCPU() and PERCPU_VADDR() linker macros, makes each arch linker scripts specify its cacheline size and use it to align percpu subsections.
This is based on Shaohua's x86 only patch.
---
Shaohua, can you please verify this achieves the same result? If no one objects, I'll route it through the percpu tree.
Thanks.
(arch ML address was wrong, resending with the correct one)
arch/alpha/kernel/vmlinux.lds.S | 2 +- arch/arm/kernel/vmlinux.lds.S | 2 +- arch/blackfin/kernel/vmlinux.lds.S | 2 +- arch/cris/kernel/vmlinux.lds.S | 2 +- arch/frv/kernel/vmlinux.lds.S | 2 +- arch/ia64/kernel/vmlinux.lds.S | 2 +- arch/m32r/kernel/vmlinux.lds.S | 2 +- arch/mips/kernel/vmlinux.lds.S | 2 +- arch/mn10300/kernel/vmlinux.lds.S | 2 +- arch/parisc/kernel/vmlinux.lds.S | 2 +- arch/powerpc/kernel/vmlinux.lds.S | 2 +- arch/s390/kernel/vmlinux.lds.S | 2 +- arch/sh/kernel/vmlinux.lds.S | 2 +- arch/sparc/kernel/vmlinux.lds.S | 2 +- arch/tile/kernel/vmlinux.lds.S | 2 +- arch/um/include/asm/common.lds.S | 2 +- arch/x86/kernel/vmlinux.lds.S | 4 ++-- arch/xtensa/kernel/vmlinux.lds.S | 2 +- include/asm-generic/vmlinux.lds.h | 35 ++++++++++++++++++++++------------- 19 files changed, 41 insertions(+), 32 deletions(-)
arch/alpha/kernel/vmlinux.lds.S +1 -1
@@ -38,7 +38,7 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) - PERCPU(PAGE_SIZE) + PERCPU(64, PAGE_SIZE) /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page needed for the THREAD_SIZE aligned init_task gets freed after init */ . = ALIGN(THREAD_SIZE);
arch/arm/kernel/vmlinux.lds.S +1 -1
@@ -70,7 +70,7 @@ SECTIONS #endif } - PERCPU(PAGE_SIZE) + PERCPU(32, PAGE_SIZE) #ifndef CONFIG_XIP_KERNEL . = ALIGN(PAGE_SIZE);
arch/blackfin/kernel/vmlinux.lds.S +1 -1
@@ -136,7 +136,7 @@ SECTIONS . = ALIGN(16); INIT_DATA_SECTION(16) - PERCPU(4) + PERCPU(32, 4) .exit.data : {
arch/cris/kernel/vmlinux.lds.S +1 -1
@@ -107,7 +107,7 @@ SECTIONS #endif __vmlinux_end = .; /* Last address of the physical file. */ #ifdef CONFIG_ETRAX_ARCH_V32 - PERCPU(PAGE_SIZE) + PERCPU(32, PAGE_SIZE) .init.ramfs : { INIT_RAM_FS
arch/frv/kernel/vmlinux.lds.S +1 -1
@@ -37,7 +37,7 @@ SECTIONS _einittext = .; INIT_DATA_SECTION(8) - PERCPU(4096) + PERCPU(L1_CACHE_BYTES, 4096) . = ALIGN(PAGE_SIZE); __init_end = .;
arch/ia64/kernel/vmlinux.lds.S +1 -1
@@ -198,7 +198,7 @@ SECTIONS { /* Per-cpu data: */ . = ALIGN(PERCPU_PAGE_SIZE); - PERCPU_VADDR(PERCPU_ADDR, :percpu) + PERCPU_VADDR(SMP_CACHE_BYTES, PERCPU_ADDR, :percpu) __phys_per_cpu_start = __per_cpu_load; /* * ensure percpu data fits
arch/m32r/kernel/vmlinux.lds.S +1 -1
@@ -53,7 +53,7 @@ SECTIONS __init_begin = .; INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) - PERCPU(PAGE_SIZE) + PERCPU(32, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */
arch/mips/kernel/vmlinux.lds.S +1 -1
@@ -108,7 +108,7 @@ SECTIONS EXIT_DATA } - PERCPU(PAGE_SIZE) + PERCPU(1 << CONFIG_MIPS_L1_CACHE_SHIFT, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */
arch/mn10300/kernel/vmlinux.lds.S +1 -1
@@ -70,7 +70,7 @@ SECTIONS .exit.text : { EXIT_TEXT; } .exit.data : { EXIT_DATA; } - PERCPU(PAGE_SIZE) + PERCPU(32, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */
arch/parisc/kernel/vmlinux.lds.S +1 -1
@@ -145,7 +145,7 @@ SECTIONS EXIT_DATA } - PERCPU(PAGE_SIZE) + PERCPU(L1_CACHE_BYTES, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */
arch/powerpc/kernel/vmlinux.lds.S +1 -1
@@ -160,7 +160,7 @@ SECTIONS INIT_RAM_FS } - PERCPU(PAGE_SIZE) + PERCPU(L1_CACHE_BYTES, PAGE_SIZE) . = ALIGN(8); .machine.desc : AT(ADDR(.machine.desc) - LOAD_OFFSET) {
arch/s390/kernel/vmlinux.lds.S +1 -1
@@ -77,7 +77,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); INIT_DATA_SECTION(0x100) - PERCPU(PAGE_SIZE) + PERCPU(0x100, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */
arch/sh/kernel/vmlinux.lds.S +1 -1
@@ -66,7 +66,7 @@ SECTIONS __machvec_end = .; } - PERCPU(PAGE_SIZE) + PERCPU(L1_CACHE_BYTES, PAGE_SIZE) /* * .exit.text is discarded at runtime, not link time, to deal with
arch/sparc/kernel/vmlinux.lds.S +1 -1
@@ -108,7 +108,7 @@ SECTIONS __sun4v_2insn_patch_end = .; } - PERCPU(PAGE_SIZE) + PERCPU(SMP_CACHE_BYTES, PAGE_SIZE) . = ALIGN(PAGE_SIZE); __init_end = .;
arch/tile/kernel/vmlinux.lds.S +1 -1
@@ -63,7 +63,7 @@ SECTIONS *(.init.page) } :data =0 INIT_DATA_SECTION(16) - PERCPU(PAGE_SIZE) + PERCPU(L2_CACHE_BYTES, PAGE_SIZE) . = ALIGN(PAGE_SIZE); VMLINUX_SYMBOL(_einitdata) = .;
arch/um/include/asm/common.lds.S +1 -1
@@ -42,7 +42,7 @@ INIT_SETUP(0) } - PERCPU(32) + PERCPU(32, 32) .initcall.init : { INIT_CALLS
arch/x86/kernel/vmlinux.lds.S +2 -2
@@ -226,7 +226,7 @@ SECTIONS * output PHDR, so the next output section - .init.text - should * start another segment - init. */ - PERCPU_VADDR(0, :percpu) + PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu) #endif INIT_TEXT_SECTION(PAGE_SIZE)
@@ -301,7 +301,7 @@ SECTIONS } #if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP) - PERCPU(THREAD_SIZE) + PERCPU(INTERNODE_CACHE_BYTES, THREAD_SIZE) #endif . = ALIGN(PAGE_SIZE);
arch/xtensa/kernel/vmlinux.lds.S +1 -1
@@ -155,7 +155,7 @@ SECTIONS INIT_RAM_FS } - PERCPU(PAGE_SIZE) + PERCPU(XCHAL_ICACHE_LINESIZE, PAGE_SIZE) /* We need this dummy segment here */
include/asm-generic/vmlinux.lds.h +22 -13
@@ -15,7 +15,7 @@* HEAD_TEXT_SECTION * INIT_TEXT_SECTION(PAGE_SIZE) * INIT_DATA_SECTION(...) -* PERCPU(PAGE_SIZE) +* PERCPU(CACHELINE_SIZE, PAGE_SIZE) * __init_end = .; * * _stext = .;
@@ -666,13 +666,18 @@/** * PERCPU_VADDR - define output section for percpu area +* @cacheline: cacheline size * @vaddr: explicit base address (optional) * @phdr: destination PHDR (optional) * -* Macro which expands to output section for percpu area. If @vaddr -* is not blank, it specifies explicit base address and all percpu -* symbols will be offset from the given address. If blank, @vaddr -* always equals @laddr + LOAD_OFFSET. +* Macro which expands to output section for percpu area. +* +* @cacheline is used to align subsections to avoid false cacheline +* sharing between subsections for different purposes. +* +* If @vaddr is not blank, it specifies explicit base address and all +* percpu symbols will be offset from the given address. If blank, +* @vaddr always equals @laddr + LOAD_OFFSET. * * @phdr defines the output PHDR to use if not blank. Be warned that * output PHDR is sticky. If @phdr is specified, the next output
@@ -683,7 +688,7 @@* If there is no need to put the percpu section at a predetermined * address, use PERCPU(). */ -#define PERCPU_VADDR (vaddr, phdr) \ +#define PERCPU_VADDR (cacheline, vaddr, phdr) \ VMLINUX_SYMBOL(__per_cpu_load) = .; \ .data..percpu vaddr : AT(VMLINUX_SYMBOL(__per_cpu_load) \ - LOAD_OFFSET) { \
@@ -691,7 +696,9 @@ *(.data..percpu..first) \ . = ALIGN(PAGE_SIZE); \ *(.data..percpu..page_aligned) \ + . = ALIGN(cacheline); \ *(.data..percpu..readmostly) \ + . = ALIGN(cacheline); \ *(.data..percpu) \ *(.data..percpu..shared_aligned) \ VMLINUX_SYMBOL(__per_cpu_end) = .; \
@@ -700,18 +707,18 @@/** * PERCPU - define output section for percpu area, simple version +* @cacheline: cacheline size * @align: required alignment * -* Align to @align and outputs output section for percpu area. This -* macro doesn't maniuplate @vaddr or @phdr and __per_cpu_load and +* Align to @align and outputs output section for percpu area. This macro +* doesn't manipulate @vaddr or @phdr and __per_cpu_load and * __per_cpu_start will be identical. * -* This macro is equivalent to ALIGN(align); PERCPU_VADDR( , ) except -* that __per_cpu_load is defined as a relative symbol against -* .data..percpu which is required for relocatable x86_32 -* configuration. +* This macro is equivalent to ALIGN(@align); PERCPU_VADDR(@cacheline,,) +* except that __per_cpu_load is defined as a relative symbol against +* .data..percpu which is required for relocatable x86_32 configuration. */ -#define PERCPU (align) \ +#define PERCPU (cacheline, align) \ . = ALIGN(align); \ .data..percpu : AT(ADDR(.data..percpu) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__per_cpu_load) = .; \
@@ -719,7 +726,9 @@ *(.data..percpu..first) \ . = ALIGN(PAGE_SIZE); \ *(.data..percpu..page_aligned) \ + . = ALIGN(cacheline); \ *(.data..percpu..readmostly) \ + . = ALIGN(cacheline); \ *(.data..percpu) \ *(.data..percpu..shared_aligned) \ VMLINUX_SYMBOL(__per_cpu_end) = .; \
Quoted 8 lines · Tejun Heo
Currently percpu readmostly subsection may share cachelines with other percpu subsections which may result in unnecessary cacheline bounce and performance degradation.
This patch adds @cacheline parameter to
PERCPU()andPERCPU_VADDR()linker macros, makes each arch linker scripts specify its cacheline size and use it to align percpu subsections.
Quoted 8 lines
This is based on Shaohua's x86 only patch.
Signed-off-by Tejun Heo <[email protected]>Cc Shaohua Li <[email protected]>---
Shaohua, can you please verify this achieves the same result? If no one objects, I'll route it through the percpu tree.
For x86:
Quoted 46 lines · Tejun Heo
Currently percpu readmostly subsection may share cachelines with other percpu subsections which may result in unnecessary cacheline bounce and performance degradation.
This patch adds @cacheline parameter to
PERCPU()andPERCPU_VADDR()linker macros, makes each arch linker scripts specify its cacheline size and use it to align percpu subsections.This is based on Shaohua's x86 only patch.
Signed-off-by Tejun Heo <[email protected]>Cc Shaohua Li <[email protected]>---
Shaohua, can you please verify this achieves the same result? If no one objects, I'll route it through the percpu tree.Thanks.
(arch ML address was wrong, resending with the correct one)
arch/alpha/kernel/vmlinux.lds.S | 2 +- arch/arm/kernel/vmlinux.lds.S | 2 +- arch/blackfin/kernel/vmlinux.lds.S | 2 +- arch/cris/kernel/vmlinux.lds.S | 2 +- arch/frv/kernel/vmlinux.lds.S | 2 +- arch/ia64/kernel/vmlinux.lds.S | 2 +- arch/m32r/kernel/vmlinux.lds.S | 2 +- arch/mips/kernel/vmlinux.lds.S | 2 +- arch/mn10300/kernel/vmlinux.lds.S | 2 +- arch/parisc/kernel/vmlinux.lds.S | 2 +- arch/powerpc/kernel/vmlinux.lds.S | 2 +- arch/s390/kernel/vmlinux.lds.S | 2 +- arch/sh/kernel/vmlinux.lds.S | 2 +- arch/sparc/kernel/vmlinux.lds.S | 2 +- arch/tile/kernel/vmlinux.lds.S | 2 +- arch/um/include/asm/common.lds.S | 2 +- arch/x86/kernel/vmlinux.lds.S | 4 ++-- arch/xtensa/kernel/vmlinux.lds.S | 2 +- include/asm-generic/vmlinux.lds.h | 35 ++++++++++++++++++++++------------- 19 files changed, 41 insertions(+), 32 deletions(-)arch/alpha/kernel/vmlinux.lds.S +0
-0
arch/alpha/kernel/vmlinux.lds.S SECTIONS · line 44
Quoted 8 lines
arch/alpha/kernel/vmlinux.lds.S +1
-1@@ -38,7 +38,7 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) - PERCPU(PAGE_SIZE) + PERCPU(64, PAGE_SIZE) /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page needed for the THREAD_SIZE aligned init_task gets freed after init */ . = ALIGN(THREAD_SIZE);
It would have been better to include cache.h and then use L1_CACHE_BYTES, as the value differs for EV4.
It will work with 64 as this is the bigger of the two.
It looks like we could do this for almost all archs. But then I am not sure if "L1_CACHE_BYTES" is the same as a cacheline on the different archs.
Sam
H. Peter Anvin
reviewerQuoted 9 lines · Sam Ravnborg
It would have been better to include
cache.hand then useL1_CACHE_BYTES, as the value differs for EV4.
It will work with 64 as this is the bigger of the two.It looks like we could do this for almost all archs. But then I am not sure if "
L1_CACHE_BYTES" is the same as a cacheline on the different archs.
For x86, L1 is definitely not the right cache line to use, in terms of what matters for SMP sharing. And yes, there are x86's with smaller L1 than L2/3 cache line size.
-hpa
Hello,
Quoted 9 lines · Sam Ravnborg
Quoted 9 lines
arch/alpha/kernel/vmlinux.lds.S +0
-1@@ -38,7 +38,7 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) - PERCPU(PAGE_SIZE)
Quoted 8 lines
arch/alpha/kernel/vmlinux.lds.S SECTIONS · line 44
Quoted 4 lines
arch/alpha/kernel/vmlinux.lds.S +1
-0+ PERCPU(64, PAGE_SIZE) /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page needed for the THREAD_SIZE aligned init_task gets freed after init */ . = ALIGN(THREAD_SIZE);It would have been better to include
cache.hand then useL1_CACHE_BYTES, as the value differs for EV4.
It will work with 64 as this is the bigger of the two.
Hmmm... I took the 64 from RW_DATA_SECTION() macro. If L1_CACHE_BYTES is better fit for PERCPU, I'm pretty sure we would be better off with that for RW_DATA_SECTION() too. I'll follow up with a patch to change both to L1_CACHE_BYTES.
Quoted 3 lines
It looks like we could do this for almost all archs. But then I am not sure if "
L1_CACHE_BYTES" is the same as a cacheline on the different archs.
hpa already replied but it seems that different archs have subtle differences regarding which cacheline size determines what. If we want to unify this, it probably would be best to define INTERNODE_CACHE_BYTES on all archs and use it for all places where false sharing should be avoided.
Thanks.
Signature · 1 line
tejun
PATCH 1/1 alpha: use L1_CACHE_BYTES for cacheline size in the linker script
Currently the linker script uses 64 for cacheline size which isn't optimal for all cases. Include asm/cache.h and use L1_CACHE_BYTES instead as suggested by Sam Ravnborg.
---
Does this look okay?
arch/alpha/kernel/vmlinux.lds.S | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-)
arch/alpha/kernel/vmlinux.lds.S +3 -2
@@ -1,5 +1,6 @@ #include <asm-generic/vmlinux.lds.h> #include <asm/thread_info.h> +#include <asm/cache.h> #include <asm/page.h> OUTPUT_FORMAT("elf64-alpha")
@@ -38,7 +39,7 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) - PERCPU(64, PAGE_SIZE) + PERCPU(L1_CACHE_BYTES, PAGE_SIZE) /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page needed for the THREAD_SIZE aligned init_task gets freed after init */ . = ALIGN(THREAD_SIZE);
@@ -46,7 +47,7 @@ SECTIONS /* Freed after init ends here */ _data = .; - RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE) + RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) .got : { *(.got)
Signature · 1 line
1.7.1
Tejun Heo
AppliedRe: [PATCH] alpha: use L1_CACHE_BYTES for cacheline size in the linker script
Quoted 6 lines · Tejun Heo
Currently the linker script uses 64 for cacheline size which isn't optimal for all cases. Include
asm/cache.hand useL1_CACHE_BYTESinstead as suggested by Sam Ravnborg.Signed-off-by Tejun Heo <[email protected]>Cc Sam Ravnborg <[email protected]>
applied to percpu#for-2.6.39.
Thanks.
Signature · 1 line
tejun
Quoted 8 lines · Tejun Heo
Currently percpu readmostly subsection may share cachelines with other percpu subsections which may result in unnecessary cacheline bounce and performance degradation.
This patch adds @cacheline parameter to
PERCPU()andPERCPU_VADDR()linker macros, makes each arch linker scripts specify its cacheline size and use it to align percpu subsections.
Quoted 7 lines
This is based on Shaohua's x86 only patch.
Signed-off-by Tejun Heo <[email protected]>Cc Shaohua Li <[email protected]>---
Shaohua, can you please verify this achieves the same result? If no one objects, I'll route it through the percpu tree.
yes, the x86 part works.
Thanks,
Shaohua
Tejun Heo
AppliedQuoted 8 lines · Tejun Heo
Currently percpu readmostly subsection may share cachelines with other percpu subsections which may result in unnecessary cacheline bounce and performance degradation.
This patch adds @cacheline parameter to
PERCPU()andPERCPU_VADDR()linker macros, makes each arch linker scripts specify its cacheline size and use it to align percpu subsections.
Quoted 7 lines
This is based on Shaohua's x86 only patch.
Signed-off-by Tejun Heo <[email protected]>Cc Shaohua Li <[email protected]>---
Shaohua, can you please verify this achieves the same result? If no one objects, I'll route it through the percpu tree.
Applied to percpu#for-2.6.39.
Thanks.
Signature · 1 line
tejun