RISC-V: Fix a register to store the percpu offset
Subsystems (5)
RFC PATCH 1/1 RISC-V: Fix a register to store the percpu offset
The following data was collected from tests conducted on the Spacemit(R) X60 using the fixed register method:
No modifications:
6.77, 6.791, 6.792, 6.826, 6.784, 6.839, 6.776, 6.733, 6.795, 6.763
Average: 6.786839305
ffix-x27:
7.106, 7.035, 7.362, 7.141, 7.096, 7.182, 7.109, 7.126, 7.186, 7.12
Average: 7.145826539
ffix-x27 + x27(s11) used for offset optimization:
7.059, 6.951, 6.961, 6.985, 6.93, 6.964, 6.977, 6.907, 6.983, 6.944
Average: 6.965993173
Analysis:
The fixed register method reduced performance by 5.29%. The per-CPU offset optimization improved performance by 2.52%.
Issues with the fixed register method (beyond code size):
Performance degradation due to the loss of one general-purpose register. Each handle_exception() call requires loading the per-CPU offset into the fixed register.
--- Makefile | 4 ++-- arch/riscv/include/asm/percpu.h | 22 ++++++++++++++++++++++ arch/riscv/kernel/asm-offsets.c | 1 + arch/riscv/kernel/entry.S | 11 +++++++++-- arch/riscv/kernel/smpboot.c | 3 +++ 5 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 arch/riscv/include/asm/percpu.h
Makefile +2 -2
@@ -1026,8 +1026,8 @@ include $(addprefix $(srctree)/, $(include-y)) # Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTFLAGS as the last assignments KBUILD_CPPFLAGS += $(KCPPFLAGS) -KBUILD_AFLAGS += $(KAFLAGS) -KBUILD_CFLAGS += $(KCFLAGS) +KBUILD_AFLAGS += $(KAFLAGS) -ffixed-x27 +KBUILD_CFLAGS += $(KCFLAGS) -ffixed-x27 KBUILD_RUSTFLAGS += $(KRUSTFLAGS) KBUILD_LDFLAGS_MODULE += --build-id=sha1
arch/riscv/include/asm/percpu.h new file +22 -0
@@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +static inline void set_my_cpu_offset (unsigned long off) +{ + asmvolatile ("addi s11, %0, 0" ::"r" (off)); +} + +static inline unsigned long __kern_my_cpu_offset (void ) +{ +unsigned long off; + asm ("mv %0, s11" :"=r" (off) :); +return off; +} + +#define __my_cpu_offset __kern_my_cpu_offset() + +#include <asm-generic/percpu.h> + +#endif
arch/riscv/kernel/asm-offsets.c +1 -0
@@ -35,6 +35,7 @@ void asm_offsets(void)OFFSET (TASK_THREAD_S9 , task_struct, thread.s[9 ]);OFFSET (TASK_THREAD_S10 , task_struct, thread.s[10 ]);OFFSET (TASK_THREAD_S11 , task_struct, thread.s[11 ]); +OFFSET (TASK_TI_CPU , task_struct, thread_info.cpu);OFFSET (TASK_TI_FLAGS , task_struct, thread_info.flags);OFFSET (TASK_TI_PREEMPT_COUNT , task_struct, thread_info.preempt_count);OFFSET (TASK_TI_KERNEL_SP , task_struct, thread_info.kernel_sp);
arch/riscv/kernel/entry.S +9 -2
@@ -77,6 +77,13 @@ SYM_CODE_START(handle_exception) */ csrw CSR_SCRATCH, x0 + /* load __per_cpu_offset[cpu] to s11*/ + REG_L t6, TASK_TI_CPU(tp) + slli t6, t6, 3 + la s11, __per_cpu_offset + add s11, s11, t6 + REG_L s11, 0(s11) + /* Load the global pointer */ load_global_pointer
@@ -298,7 +305,7 @@ SYM_FUNC_START(__switch_to) REG_S s8, TASK_THREAD_S8_RA(a3) REG_S s9, TASK_THREAD_S9_RA(a3) REG_S s10, TASK_THREAD_S10_RA(a3) - REG_S s11, TASK_THREAD_S11_RA(a3) + /* REG_S s11, TASK_THREAD_S11_RA(a3) */ /* Save the kernel shadow call stack pointer */ scs_save_current /* Restore context from next->thread */
@@ -315,7 +322,7 @@ SYM_FUNC_START(__switch_to) REG_L s8, TASK_THREAD_S8_RA(a4) REG_L s9, TASK_THREAD_S9_RA(a4) REG_L s10, TASK_THREAD_S10_RA(a4) - REG_L s11, TASK_THREAD_S11_RA(a4) + /* REG_L s11, TASK_THREAD_S11_RA(a4) */ /* The offset of thread_info in task_struct is zero. */ move tp, a1 /* Switch to the next shadow call stack */
arch/riscv/kernel/smpboot.c +3 -0
@@ -43,6 +43,7 @@ static DECLARE_COMPLETION(cpu_running);void __init smp_prepare_boot_cpu (void ) { +set_my_cpu_offset (per_cpu_offset (smp_processor_id ())); }void __init smp_prepare_cpus (unsigned int max_cpus)
@@ -240,6 +241,8 @@ asmlinkage __visible void smp_callin(void)mmgrab (mm); current->active_mm = mm; +set_my_cpu_offset (per_cpu_offset (curr_cpuid)); +store_cpu_topology (curr_cpuid);notify_cpu_starting (curr_cpuid);
Signature · 1 line
2.43.0
Quoted 7 lines · Yunhui Cui
The following data was collected from tests conducted on the Spacemit(R) X60 using the fixed register method:
No modifications:
6.77, 6.791, 6.792, 6.826, 6.784, 6.839, 6.776, 6.733, 6.795, 6.763
Average: 6.786839305
Quoted 7 lines
ffix-x27:
7.106, 7.035, 7.362, 7.141, 7.096, 7.182, 7.109, 7.126, 7.186, 7.12
Average: 7.145826539ffix-x27 + x27(s11) used for offset optimization:
7.059, 6.951, 6.961, 6.985, 6.93, 6.964, 6.977, 6.907, 6.983, 6.944
Average: 6.965993173
Hi Yunhui,
Out of curiosity, did you tried using a register different than x27 ?
Thanks,
Clément
Quoted 132 lines
Analysis:
The fixed register method reduced performance by 5.29%. The per-CPU offset optimization improved performance by 2.52%.Issues with the fixed register method (beyond code size):
Performance degradation due to the loss of one general-purpose register. Eachhandle_exception()call requires loading the per-CPU offset into the fixed register.Signed-off-by Yunhui Cui <[email protected]>--- Makefile | 4 ++-- arch/riscv/include/asm/percpu.h | 22 ++++++++++++++++++++++ arch/riscv/kernel/asm-offsets.c | 1 + arch/riscv/kernel/entry.S | 11 +++++++++-- arch/riscv/kernel/smpboot.c | 3 +++ 5 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 arch/riscv/include/asm/percpu.hMakefile +2
-2@@ -1026,8 +1026,8 @@ include $(addprefix $(srctree)/, $(include-y)) # Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTFLAGS as the last assignments KBUILD_CPPFLAGS += $(KCPPFLAGS) -KBUILD_AFLAGS += $(KAFLAGS) -KBUILD_CFLAGS += $(KCFLAGS) +KBUILD_AFLAGS += $(KAFLAGS) -ffixed-x27 +KBUILD_CFLAGS += $(KCFLAGS) -ffixed-x27 KBUILD_RUSTFLAGS += $(KRUSTFLAGS) KBUILD_LDFLAGS_MODULE += --build-id=sha1arch/riscv/include/asm/percpu.h new file +22
-0@@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +static inline void set_my_cpu_offset (unsigned long off) +{ + asmvolatile ("addi s11, %0, 0" ::"r" (off)); +} + +static inline unsigned long __kern_my_cpu_offset (void ) +{ +unsigned long off; + asm ("mv %0, s11" :"=r" (off) :); +return off; +} + +#define __my_cpu_offset __kern_my_cpu_offset() + +#include <asm-generic/percpu.h> + +#endif arch/riscv/kernel/asm-offsets.c +1
-0@@ -35,6 +35,7 @@ void asm_offsets(void)OFFSET (TASK_THREAD_S9 , task_struct, thread.s[9 ]);OFFSET (TASK_THREAD_S10 , task_struct, thread.s[10 ]);OFFSET (TASK_THREAD_S11 , task_struct, thread.s[11 ]); +OFFSET (TASK_TI_CPU , task_struct, thread_info.cpu);OFFSET (TASK_TI_FLAGS , task_struct, thread_info.flags);OFFSET (TASK_TI_PREEMPT_COUNT , task_struct, thread_info.preempt_count);OFFSET (TASK_TI_KERNEL_SP , task_struct, thread_info.kernel_sp);arch/riscv/kernel/entry.S +9
-2@@ -77,6 +77,13 @@ SYM_CODE_START(handle_exception) */ csrw CSR_SCRATCH, x0 + /* load __per_cpu_offset[cpu] to s11*/ + REG_L t6, TASK_TI_CPU(tp) + slli t6, t6, 3 + la s11, __per_cpu_offset + add s11, s11, t6 + REG_L s11, 0(s11) + /* Load the global pointer */ load_global_pointer@@ -298,7 +305,7 @@ SYM_FUNC_START(__switch_to) REG_S s8, TASK_THREAD_S8_RA(a3) REG_S s9, TASK_THREAD_S9_RA(a3) REG_S s10, TASK_THREAD_S10_RA(a3) - REG_S s11, TASK_THREAD_S11_RA(a3) + /* REG_S s11, TASK_THREAD_S11_RA(a3) */ /* Save the kernel shadow call stack pointer */ scs_save_current /* Restore context from next->thread */@@ -315,7 +322,7 @@ SYM_FUNC_START(__switch_to) REG_L s8, TASK_THREAD_S8_RA(a4) REG_L s9, TASK_THREAD_S9_RA(a4) REG_L s10, TASK_THREAD_S10_RA(a4) - REG_L s11, TASK_THREAD_S11_RA(a4) + /* REG_L s11, TASK_THREAD_S11_RA(a4) */ /* The offset of thread_info in task_struct is zero. */ move tp, a1 /* Switch to the next shadow call stack */arch/riscv/kernel/smpboot.c +3
-0@@ -43,6 +43,7 @@ static DECLARE_COMPLETION(cpu_running);void __init smp_prepare_boot_cpu (void ) { +set_my_cpu_offset (per_cpu_offset (smp_processor_id ())); }void __init smp_prepare_cpus (unsigned int max_cpus)@@ -240,6 +241,8 @@ asmlinkage __visible void smp_callin(void)mmgrab (mm); current->active_mm = mm; +set_my_cpu_offset (per_cpu_offset (curr_cpuid)); +store_cpu_topology (curr_cpuid);notify_cpu_starting (curr_cpuid);
PATCH 1/1 RISC-V: store percpu offset in CSR_SCRATCH
In reply to Yunhui Cui2025-07-04T16:45:00+08:00, Yunhui Cui <[email protected]>:
Quoted 5 lines
The following data was collected from tests conducted on the Spacemit(R) X60 using the fixed register method:
[...]
The fixed register method reduced performance by 5.29%. The per-CPU offset optimization improved performance by 2.52%.
What is the performance if you use the scratch register?
The patch below is completely unoptimized as I didn't want to shuffle code around too much, but it could give a rough idea.
Thanks.
---8<---
The scratch register currently denotes the mode before exception, but we can just use two different exception entry points to provide the same information, which frees the scratch register for the percpu offset.
The user/kernel entry paths need more through rewrite, because they are terribly wasteful right now.
---
Applies on top of d7b8f8e20813f0179d8ef519541a3527e7661d3a (v6.16-rc5)
arch/riscv/include/asm/percpu.h | 13 ++++++++++ arch/riscv/kernel/entry.S | 46 ++++++++++++++++++++------------- arch/riscv/kernel/head.S | 7 +---- arch/riscv/kernel/smpboot.c | 7 +++++ arch/riscv/kernel/stacktrace.c | 4 +-- 5 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 arch/riscv/include/asm/percpu.h
arch/riscv/include/asm/percpu.h new file +13 -0
@@ -0,0 +1,13 @@ +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +static inline void set_my_cpu_offset (unsigned long off) +{ +csr_write (CSR_SCRATCH , off); +} + +#define __my_cpu_offset csr_read(CSR_SCRATCH) + +#include <asm-generic/percpu.h> + +#endif
arch/riscv/kernel/entry.S +28 -18
71 lines not on this page · raw message
arch/riscv/kernel/head.S +1 -6
@@ -188,14 +188,9 @@ secondary_start_sbi: .align 2 .Lsetup_trap_vector: /* Set trap vector to exception handler */ - la a0, handle_exception + la a0, handle_kernel_exception csrw CSR_TVEC, a0 - /* - * Set sup0 scratch register to 0, indicating to exception vector that - * we are presently executing in kernel. - */ - csrw CSR_SCRATCH, zero ret SYM_CODE_END(_start)
arch/riscv/kernel/smpboot.c +7 -0
@@ -41,6 +41,11 @@static DECLARE_COMPLETION (cpu_running); +void __init smp_prepare_boot_cpu (void ) +{ +set_my_cpu_offset (per_cpu_offset (smp_processor_id ())); +} +void __init smp_prepare_cpus (unsigned int max_cpus) {int cpuid;
@@ -225,6 +230,8 @@ asmlinkage __visible void smp_callin(void)mmgrab (mm); current->active_mm = mm; +set_my_cpu_offset (per_cpu_offset (curr_cpuid)); +store_cpu_topology (curr_cpuid);notify_cpu_starting (curr_cpuid);
arch/riscv/kernel/stacktrace.c +2 -2
@@ -16,7 +16,7 @@#ifdef CONFIG_FRAME_POINTER -extern asmlinkage void handle_exception(void); +extern asmlinkage void handle_kernel_exception(void);extern unsigned long ret_from_exception_end;static inline int fp_is_valid (unsigned long fp,unsigned long sp)
@@ -72,7 +72,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, fp = frame->fp; pc =ftrace_graph_ret_addr (current, &graph_idx, frame->ra, &frame->ra); -if (pc >= (unsigned long )handle_exception && +if (pc >= (unsigned long )handle_kernel_exception && pc < (unsigned long )&ret_from_exception_end) {if (unlikely (!fn (arg, pc)))break ;
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
Hi Radim,
Quoted 107 lines · Radim Krčmář
2025-07-04T16:45:00+08:00, Yunhui Cui <[email protected]>:Quoted 5 lines
The following data was collected from tests conducted on the Spacemit(R) X60 using the fixed register method:
[...]
The fixed register method reduced performance by 5.29%. The per-CPU offset optimization improved performance by 2.52%.What is the performance if you use the scratch register?
The patch below is completely unoptimized as I didn't want to shuffle code around too much, but it could give a rough idea.
Thanks.
---8<---
The scratch register currently denotes the mode before exception, but we can just use two different exception entry points to provide the same information, which frees the scratch register for the percpu offset.The user/kernel entry paths need more through rewrite, because they are terribly wasteful right now.
---
Applies on top ofd7b8f8e20813f0179d8ef519541a3527e7661d3a(v6.16-rc5)arch/riscv/include/asm/percpu.h | 13 ++++++++++ arch/riscv/kernel/entry.S | 46 ++++++++++++++++++++------------- arch/riscv/kernel/head.S | 7 +---- arch/riscv/kernel/smpboot.c | 7 +++++ arch/riscv/kernel/stacktrace.c | 4 +-- 5 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 arch/riscv/include/asm/percpu.harch/riscv/include/asm/percpu.h new file +13
-0@@ -0,0 +1,13 @@ +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +static inline void set_my_cpu_offset (unsigned long off) +{ +csr_write (CSR_SCRATCH , off); +} + +#define __my_cpu_offset csr_read(CSR_SCRATCH) + +#include <asm-generic/percpu.h> + +#endif arch/riscv/kernel/entry.S +17
-15@@ -91,18 +91,8 @@ REG_L a0, TASK_TI_A0(tp) .endm - -SYM_CODE_START(handle_exception) - /* - * If coming from userspace, preserve the user thread pointer and load - * the kernel thread pointer. If we came from the kernel, the scratch - * register will contain 0, and we should continue on the current TP. - */ - csrrw tp, CSR_SCRATCH, tp - bnez tp, .Lsave_context - -.Lrestore_kernel_tpsp: - csrr tp, CSR_SCRATCH +SYM_CODE_START(handle_kernel_exception) + csrw CSR_SCRATCH, tp #ifdef CONFIG_64BIT /*@@ -126,8 +116,22 @@ SYM_CODE_START(handle_exception) bnez sp, handle_kernel_stack_overflow REG_L sp, TASK_TI_KERNEL_SP(tp) #endif + j handle_exception +ASM_NOKPROBE(handle_kernel_exception) +SYM_CODE_END(handle_kernel_exception) -.Lsave_context: +SYM_CODE_START(handle_user_exception) + /* + * If coming from userspace, preserve the user thread pointer and load + * the kernel thread pointer. + */ + csrrw tp, CSR_SCRATCH, tp + j handle_exception + +SYM_CODE_END(handle_user_exception) +ASM_NOKPROBE(handle_user_exception) + +SYM_CODE_START_NOALIGN(handle_exception) REG_S sp, TASK_TI_USER_SP(tp) REG_L sp, TASK_TI_KERNEL_SP(tp) addi sp, sp, -(PT_SIZE_ON_STACK)@@ -158,11 +162,15 @@ SYM_CODE_START(handle_exception) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) - /* - * Set the scratch register to 0, so that if a recursive exception
arch/riscv/kernel/entry.S SYM_CODE_START(handle_exception) · line 171
Quoted 10 lines
arch/riscv/kernel/entry.S +7
-3- * occurs, the exception vector knows it came from the kernel - */ - csrw CSR_SCRATCH, x0 + REG_L s0, TASK_TI_CPU(tp) + slli s0, s0, 3 + la s1, __per_cpu_offset + add s1, s1, s0 + REG_L s1, 0(s1) + + csrw CSR_SCRATCH, s1
This patch cleverly differentiates whether an exception originates from user mode or kernel mode. However, there's still an issue with using CSR_SCRATCH: each time handle_exception() is called, the following instructions must be executed:
REG_L s0, TASK_TI_CPU(tp)
slli s0, s0, 3
la s1, __per_cpu_offset
add s1, s1, s0REG_L s1, 0(s1)
csrw CSR_SCRATCH, s1
Should we consider adding a dedicated CSR (e.g., CSR_SCRATCH2) to store the percpu offset instead?
See: https://lists.riscv.org/g/tech-privileged/topic/113437553#msg2506
Quoted 73 lines
arch/riscv/kernel/entry.S +4
-0+ la s1, handle_kernel_exception + csrw CSR_TVEC, s1 /* Load the global pointer */ load_global_pointer@@ -236,6 +244,8 @@ SYM_CODE_START_NOALIGN(ret_from_exception) * structures again. */ csrw CSR_SCRATCH, tp + la a0, handle_user_exception + csrw CSR_TVEC, a0 1: #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sparch/riscv/kernel/head.S +1
-6@@ -188,14 +188,9 @@ secondary_start_sbi: .align 2 .Lsetup_trap_vector: /* Set trap vector to exception handler */ - la a0, handle_exception + la a0, handle_kernel_exception csrw CSR_TVEC, a0 - /* - * Set sup0 scratch register to 0, indicating to exception vector that - * we are presently executing in kernel. - */ - csrw CSR_SCRATCH, zero ret SYM_CODE_END(_start)arch/riscv/kernel/smpboot.c +7
-0@@ -41,6 +41,11 @@static DECLARE_COMPLETION (cpu_running); +void __init smp_prepare_boot_cpu (void ) +{ +set_my_cpu_offset (per_cpu_offset (smp_processor_id ())); +} +void __init smp_prepare_cpus (unsigned int max_cpus) {int cpuid;@@ -225,6 +230,8 @@ asmlinkage __visible void smp_callin(void)mmgrab (mm); current->active_mm = mm; +set_my_cpu_offset (per_cpu_offset (curr_cpuid)); +store_cpu_topology (curr_cpuid);notify_cpu_starting (curr_cpuid);arch/riscv/kernel/stacktrace.c +1
-1@@ -16,7 +16,7 @@#ifdef CONFIG_FRAME_POINTER -extern asmlinkage void handle_exception(void); +extern asmlinkage void handle_kernel_exception(void);extern unsigned long ret_from_exception_end;static inline int fp_is_valid (unsigned long fp,unsigned long sp)
arch/riscv/kernel/stacktrace.c void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, · line 78
Quoted 8 lines
arch/riscv/kernel/stacktrace.c +1
-1@@ -72,7 +72,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, fp = frame->fp; pc =ftrace_graph_ret_addr (current, &graph_idx, frame->ra, &frame->ra); -if (pc >= (unsigned long )handle_exception && +if (pc >= (unsigned long )handle_kernel_exception && pc < (unsigned long )&ret_from_exception_end) {if (unlikely (!fn (arg, pc)))break ;
Thanks,
Yunhui
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
2025-07-08T18:07:27+08:00, yunhui cui <[email protected]>:
Quoted 11 lines
This patch cleverly differentiates whether an exception originates from user mode or kernel mode. However, there's still an issue with using
CSR_SCRATCH: each timehandle_exception()is called, the following instructions must be executed:
REG_Ls0,TASK_TI_CPU(tp)
slli s0, s0, 3
la s1,__per_cpu_offset
add s1, s1, s0REG_Ls1, 0(s1)
csrwCSR_SCRATCH, s1
We can minimize the cost at exception entry by storing the precomputed offset in thread_info, which bloats the struct, and also incurs update cost on cpu migration, but should still be a net performance gain.
The minimal code at exception entry would be:
REG_L s0, TASK_TI_PERCPU_OFFSET(tp)
csrw CSR_SCRATCH, s0
Quoted 3 lines
Should we consider adding a dedicated CSR (e.g.,
CSR_SCRATCH2) to store the percpu offset instead?
See: https://lists.riscv.org/g/tech-privileged/topic/113437553#msg2506
It would be nice to gather more data on the CSR_SCRATCH approach. Basically, the overhead of "REG_L s0, TASK_TI_PERCPU_OFFSET(tp)". (Or the longer sequence if we think it is worth it.)
Can you benchmark the patch after reverting percpu.h, so we include the overhead of switching CSR_SCRATCH, but without any benefits provided by the per-cpu offset?
The baseline would be the patch with reverted percpu.h, and reverted the sequence that sets the CSR_SCRATCH in handle_exception, so we roughly estimate the benefit of adding CSR_SCRATCH2.
The CSR_SCRATCH2 does add overhead to hardware, and to domain context switches, and we also have to do something else for a few years anyway, because it's not even ratified... It's possible we might not benefit enough from CSR_SCRATCH2 to make a good case for it.
Thanks.
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
Hi Radim,
Quoted 38 lines · Radim Krčmář
2025-07-08T18:07:27+08:00, yunhui cui <[email protected]>:Quoted 11 lines
This patch cleverly differentiates whether an exception originates from user mode or kernel mode. However, there's still an issue with using
CSR_SCRATCH: each timehandle_exception()is called, the following instructions must be executed:
REG_Ls0,TASK_TI_CPU(tp)
slli s0, s0, 3
la s1,__per_cpu_offset
add s1, s1, s0REG_Ls1, 0(s1)
csrwCSR_SCRATCH, s1We can minimize the cost at exception entry by storing the precomputed offset in
thread_info, which bloats the struct, and also incurs update cost on cpu migration, but should still be a net performance gain.The minimal code at exception entry would be:
REG_Ls0,TASK_TI_PERCPU_OFFSET(tp)
csrwCSR_SCRATCH, s0Quoted 3 lines
Should we consider adding a dedicated CSR (e.g.,
CSR_SCRATCH2) to store the percpu offset instead?
See: https://lists.riscv.org/g/tech-privileged/topic/113437553#msg2506It would be nice to gather more data on the
CSR_SCRATCHapproach. Basically, the overhead of "REG_Ls0,TASK_TI_PERCPU_OFFSET(tp)". (Or the longer sequence if we think it is worth it.)Can you benchmark the patch after reverting
percpu.h, so we include the overhead of switchingCSR_SCRATCH, but without any benefits provided by the per-cpu offset?
The baseline would be the patch with revertedpercpu.h, and reverted the sequence that sets theCSR_SCRATCHinhandle_exception, so we roughly estimate the benefit of addingCSR_SCRATCH2.
Quoted 6 lines
The
CSR_SCRATCH2does add overhead to hardware, and to domain context switches, and we also have to do something else for a few years anyway, because it's not even ratified... It's possible we might not benefit enough fromCSR_SCRATCH2to make a good case for it.Thanks.
Bench platform: Spacemit(R) X60
No changes:
6.77, 6.791, 6.792, 6.826, 6.784, 6.839, 6.776, 6.733, 6.795, 6.763 Geometric mean: 6.786839305
Reusing the current scratch:
7.085, 7.09, 7.021, 7.089, 7.068, 7.034, 7.06, 7.062, 7.065, 7.051 Geometric mean: 7.062466876
A degradation of approximately 4.06% is observed. The possible cause of the degradation is that the CSR_TVEC register is set every time a kernel/user exception occurs.
The following is the patch without percpu optimization, which only tests the overhead of separating exceptions into kernel and user modes.
--- arch/riscv/kernel/entry.S | 39 ++++++++++++++++++++++----------------- arch/riscv/kernel/head.S | 7 +------ 2 files changed, 23 insertions(+), 23 deletions(-)
arch/riscv/kernel/entry.S +22 -17
@@ -19,17 +19,8 @@ .section .irqentry.text, "ax" -SYM_CODE_START(handle_exception) - /* - * If coming from userspace, preserve the user thread pointer and load - * the kernel thread pointer. If we came from the kernel, the scratch - * register will contain 0, and we should continue on the current TP. - */ - csrrw tp, CSR_SCRATCH, tp - bnez tp, .Lsave_context - -.Lrestore_kernel_tpsp: - csrr tp, CSR_SCRATCH +SYM_CODE_START(handle_kernel_exception) + csrw CSR_SCRATCH, tp REG_S sp, TASK_TI_KERNEL_SP(tp) #ifdef CONFIG_VMAP_STACK
@@ -40,7 +31,20 @@ SYM_CODE_START(handle_exception) REG_L sp, TASK_TI_KERNEL_SP(tp) #endif -.Lsave_context: + j handle_exception +SYM_CODE_END(handle_kernel_exception) + +SYM_CODE_START(handle_user_exception) + /* + * If coming from userspace, preserve the user thread pointer and load + * the kernel thread pointer. + */ + csrrw tp, CSR_SCRATCH, tp + j handle_exception + +SYM_CODE_END(handle_user_exception) + +SYM_CODE_START_NOALIGN(handle_exception) REG_S sp, TASK_TI_USER_SP(tp) REG_L sp, TASK_TI_KERNEL_SP(tp) addi sp, sp, -(PT_SIZE_ON_STACK)
@@ -71,11 +75,8 @@ SYM_CODE_START(handle_exception) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) - /* - * Set the scratch register to 0, so that if a recursive exception - * occurs, the exception vector knows it came from the kernel - */ - csrw CSR_SCRATCH, x0 + la s1, handle_kernel_exception + csrw CSR_TVEC, s1 /* Load the global pointer */ load_global_pointer
@@ -141,6 +142,10 @@ SYM_CODE_START_NOALIGN(ret_from_exception) * structures again. */ csrw CSR_SCRATCH, tp + + la a0, handle_user_exception + csrw CSR_TVEC, a0 + 1: #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sp
arch/riscv/kernel/head.S +1 -6
@@ -172,14 +172,9 @@ secondary_start_sbi: .align 2 .Lsetup_trap_vector: /* Set trap vector to exception handler */ - la a0, handle_exception + la a0, handle_kernel_exception csrw CSR_TVEC, a0 - /* - * Set sup0 scratch register to 0, indicating to exception vector that - * we are presently executing in kernel. - */ - csrw CSR_SCRATCH, zero ret .align 2
--
2.43.0
Thanks,
Yunhui
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
2025-07-09T19:42:26+08:00, yunhui cui <[email protected]>:
Quoted 7 lines
Bench platform: Spacemit(R) X60
No changes:
6.77, 6.791, 6.792, 6.826, 6.784, 6.839, 6.776, 6.733, 6.795, 6.763 Geometric mean: 6.786839305
Reusing the current scratch:
7.085, 7.09, 7.021, 7.089, 7.068, 7.034, 7.06, 7.062, 7.065, 7.051 Geometric mean: 7.062466876
Great results.
Quoted 3 lines
A degradation of approximately 4.06% is observed. The possible cause of the degradation is that the
CSR_TVECregister is set every time a kernel/user exception occurs.
I assume the same.
Quoted 3 lines
The following is the patch without percpu optimization, which only tests the overhead of separating exceptions into kernel and user modes.
Is the overhead above with this patch? And when we then use the CSR_SCRATCH for percpu, does it degrade even further?
Thanks.
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
Hi Radim,
Quoted 18 lines · Radim Krčmář
2025-07-09T19:42:26+08:00, yunhui cui <[email protected]>:Quoted 7 lines
Bench platform: Spacemit(R) X60
No changes:
6.77, 6.791, 6.792, 6.826, 6.784, 6.839, 6.776, 6.733, 6.795, 6.763 Geometric mean: 6.786839305
Reusing the current scratch:
7.085, 7.09, 7.021, 7.089, 7.068, 7.034, 7.06, 7.062, 7.065, 7.051 Geometric mean: 7.062466876Great results.
Quoted 3 lines
A degradation of approximately 4.06% is observed. The possible cause of the degradation is that the
CSR_TVECregister is set every time a kernel/user exception occurs.I assume the same.
Quoted 8 lines
Quoted 3 lines
The following is the patch without percpu optimization, which only tests the overhead of separating exceptions into kernel and user modes.
Is the overhead above with this patch? And when we then use the
CSR_SCRATCHfor percpu, does it degrade even further?Thanks.
We can see that the percpu optimization is around 2.5% through the method of fixing registers, and we can consider that the percpu optimization can bring a 2.5% gain. Is there no need to add the percpu optimization logic on the basis of the scratch patch for testing?
Reference: https://lists.riscv.org/g/tech-privileged/message/2485
Thanks,
Yunhui
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
2025-07-10T11:45:06+08:00, yunhui cui <[email protected]>:
Quoted 10 lines · Radim Krčmář
Quoted 2 lines
Is the overhead above with this patch? And when we then use the
CSR_SCRATCHfor percpu, does it degrade even further?We can see that the percpu optimization is around 2.5% through the method of fixing registers, and we can consider that the percpu optimization can bring a 2.5% gain. Is there no need to add the percpu optimization logic on the basis of the scratch patch for testing?
Reference: https://lists.riscv.org/g/tech-privileged/message/2485
That is when the value is in a GPR, though, and we don't know the performance of a CSR_SCRATCH access.
We can hope that it's not much worse than a GPR, but an implementation might choose to be very slow with CSR_SCRATCH.
I have in mind another method where we can use the current CSR_SCRATCH without changing CSR_TVAL, but I don't really want to spend time on it if reading the CSR doesn't give any benefit.
It would be to store the percpu offset in CSR_SCRATCH permanently, do the early exception register shuffling with a percpu area storage, and load the thread pointer from there as well.
That method would also eliminate writing CSR_SCRATCH on every exception entry+exit, so maybe it makes sense to try it even if CSRs are slow...
Thanks.
Re: [External] [PATCH] RISC-V: store percpu offset in CSR_SCRATCH
Hi Radim,
Quoted 22 lines · Radim Krčmář
2025-07-10T11:45:06+08:00, yunhui cui <[email protected]>:Quoted 10 lines · Radim Krčmář
Quoted 2 lines
Is the overhead above with this patch? And when we then use the
CSR_SCRATCHfor percpu, does it degrade even further?We can see that the percpu optimization is around 2.5% through the method of fixing registers, and we can consider that the percpu optimization can bring a 2.5% gain. Is there no need to add the percpu optimization logic on the basis of the scratch patch for testing?
Reference: https://lists.riscv.org/g/tech-privileged/message/2485
That is when the value is in a GPR, though, and we don't know the performance of a
CSR_SCRATCHaccess.
We can hope that it's not much worse than a GPR, but an implementation might choose to be very slow withCSR_SCRATCH.I have in mind another method where we can use the current
CSR_SCRATCHwithout changingCSR_TVAL, but I don't really want to spend time on it if reading the CSR doesn't give any benefit.
Quoted 7 lines
It would be to store the percpu offset in
CSR_SCRATCHpermanently, do the early exception register shuffling with a percpu area storage, and load the thread pointer from there as well.
That method would also eliminate writingCSR_SCRATCHon every exception entry+exit, so maybe it makes sense to try it even if CSRs are slow...Thanks.
Based on the patch, optimizations for percpu offset have been added, with the following data:
6.989 7.046 6.976 6.986 7.001 7.017 7.007 7.064 7.008 7.039 Geometric mean: 7.013248303
Compared to reusing the scratch register, the performance has improved by approximately 0.7%.
If more optimizations can be made to the scratch register, there should be further performance improvements.
Patch:
--- arch/riscv/include/asm/percpu.h | 14 ++++++++++++++ arch/riscv/kernel/asm-offsets.c | 1 + arch/riscv/kernel/entry.S | 7 +++++++ arch/riscv/kernel/smpboot.c | 3 +++ 4 files changed, 25 insertions(+) create mode 100644 arch/riscv/include/asm/percpu.h
arch/riscv/include/asm/percpu.h new file +14 -0
@@ -0,0 +1,14 @@ +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +static inline void set_my_cpu_offset (unsigned long off) +{ +csr_write (CSR_SCRATCH , off); +} + +#define __my_cpu_offset csr_read(CSR_SCRATCH) + +#include <asm-generic/percpu.h> + +#endif +
arch/riscv/kernel/asm-offsets.c +1 -0
@@ -35,6 +35,7 @@ void asm_offsets(void)OFFSET (TASK_THREAD_S9 , task_struct, thread.s[9 ]);OFFSET (TASK_THREAD_S10 , task_struct, thread.s[10 ]);OFFSET (TASK_THREAD_S11 , task_struct, thread.s[11 ]); +OFFSET (TASK_TI_CPU , task_struct, thread_info.cpu);OFFSET (TASK_TI_FLAGS , task_struct, thread_info.flags);OFFSET (TASK_TI_PREEMPT_COUNT , task_struct, thread_info.preempt_count);OFFSET (TASK_TI_KERNEL_SP , task_struct, thread_info.kernel_sp);
arch/riscv/kernel/entry.S +7 -0
@@ -75,6 +75,13 @@ SYM_CODE_START_NOALIGN(handle_exception) REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) + REG_L s0, TASK_TI_CPU(tp) + slli s0, s0, 3 + la s1, __per_cpu_offset + add s1, s1, s0 + REG_L s1, 0(s1) + csrw CSR_SCRATCH, s1 + la s1, handle_kernel_exception csrw CSR_TVEC, s1
arch/riscv/kernel/smpboot.c +3 -0
@@ -43,6 +43,7 @@ static DECLARE_COMPLETION(cpu_running);void __init smp_prepare_boot_cpu (void ) { +set_my_cpu_offset (per_cpu_offset (smp_processor_id ())); }void __init smp_prepare_cpus (unsigned int max_cpus)
@@ -240,6 +241,8 @@ asmlinkage __visible void smp_callin(void)mmgrab (mm); current->active_mm = mm; +set_my_cpu_offset (per_cpu_offset (curr_cpuid)); +store_cpu_topology (curr_cpuid);notify_cpu_starting (curr_cpuid);
--
2.43.0
Thanks,
Yunhui
PATCH 1/1 RISC-V: store precomputed percpu_offset in the task struct
2025-07-10T19:47:27+08:00, yunhui cui <[email protected]>:
Quoted 7 lines · Radim Krčmář
Quoted 5 lines
It would be to store the percpu offset in
CSR_SCRATCHpermanently, do the early exception register shuffling with a percpu area storage, and load the thread pointer from there as well.
That method would also eliminate writingCSR_SCRATCHon every exception entry+exit, so maybe it makes sense to try it even if CSRs are slow...
Quoted 6 lines
Based on the patch, optimizations for percpu offset have been added, with the following data:
6.989 7.046 6.976 6.986 7.001 7.017 7.007 7.064 7.008 7.039 Geometric mean: 7.013248303
Compared to reusing the scratch register, the performance has improved by approximately 0.7%.
Nice, thanks. The CSR_SCRATCH accesses seem much slower than GPRs, and possibly even slower than L1 hit -- we might gain more by storing the precomputed offset in the task struct.
Can you check this patch as well?
(It should be compared against a variant of CSR_SCRATCH that uses the
TASK_TI_PERCPU_OFFSET optimizations, but we can try to interpolate. :])
---8<---
RISC-V: store precomputed percpu_offset in the task struct
Exploring the memoization trade-off... hoping that __set_task_cpu covers everything. :)
I didn't put any though into where the percpu_offset should live, and the naive approach is to put it next to cpu.
This needs more work to not break build on other arches, because I directly added RISC-V specific code to __set_task_cpu, to save time figuring out where else it could be.
--- arch/riscv/include/asm/asm.h | 6 +----- arch/riscv/include/asm/percpu.h | 8 ++++++++ arch/riscv/include/asm/thread_info.h | 3 ++- arch/riscv/kernel/asm-offsets.c | 1 + arch/riscv/kernel/smpboot.c | 6 ++++++ kernel/sched/sched.h | 1 + 6 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 arch/riscv/include/asm/percpu.h
arch/riscv/include/asm/asm.h +1 -5
@@ -91,11 +91,7 @@#endif .macro asm_per_cpu dst sym tmp - REG_L \tmp, TASK_TI_CPU_NUM(tp) - slli \tmp, \tmp, PER_CPU_OFFSET_SHIFT - la \dst, __per_cpu_offset - add \dst, \dst, \tmp - REG_L \tmp, 0(\dst) + REG_L \tmp, TASK_TI_PERCPU_OFFSET(tp) la \dst, \sym add \dst, \dst, \tmp .endm
arch/riscv/include/asm/percpu.h new file +8 -0
@@ -0,0 +1,8 @@ +#ifndef __ASM_PERCPU_H +#define __ASM_PERCPU_H + +#define __my_cpu_offset (current_thread_info()->percpu_offset) + +#include <asm-generic/percpu.h> + +#endif
arch/riscv/include/asm/thread_info.h +2 -1
@@ -60,8 +60,9 @@ struct thread_info {*/ long kernel_sp;/* Kernel stack pointer */ long user_sp;/* User stack pointer */ -int cpu; +int cpu;// TODO: could be packed better unsigned long syscall_work;/* SYSCALL_WORK_ flags */ +unsigned long percpu_offset;// XXX: randomly placed here #ifdef CONFIG_SHADOW_CALL_STACK void *scs_base;void *scs_sp;
arch/riscv/kernel/asm-offsets.c +1 -0
@@ -50,6 +50,7 @@ void asm_offsets(void)#endif OFFSET (TASK_TI_CPU_NUM , task_struct, thread_info.cpu); +OFFSET (TASK_TI_PERCPU_OFFSET , task_struct, thread_info.percpu_offset);OFFSET (TASK_THREAD_F0 , task_struct, thread.fstate.f[0 ]);OFFSET (TASK_THREAD_F1 , task_struct, thread.fstate.f[1 ]);OFFSET (TASK_THREAD_F2 , task_struct, thread.fstate.f[2 ]);
arch/riscv/kernel/smpboot.c +6 -0
@@ -41,6 +41,11 @@static DECLARE_COMPLETION (cpu_running); +void __init smp_prepare_boot_cpu (void ) +{ +current_thread_info ()->percpu_offset =per_cpu_offset (smp_processor_id ()); +} +void __init smp_prepare_cpus (unsigned int max_cpus) {int cpuid;
@@ -183,6 +188,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) {int ret =0 ; tidle->thread_info.cpu = cpu; + tidle->thread_info.percpu_offset =per_cpu_offset (cpu); ret =start_secondary_cpu (cpu, tidle);if (!ret) {
kernel/sched/sched.h +1 -0
@@ -2199,6 +2199,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)*/ smp_wmb ();WRITE_ONCE (task_thread_info (p)->cpu, cpu); +WRITE_ONCE (task_thread_info (p)->percpu_offset,per_cpu_offset (cpu)); p->wake_cpu = cpu;#endif }