use runtime constant to optimize handle_arch_irq access
Subsystems (5)
cover use runtime constant to optimize handle_arch_irq access
Currently, on GENERIC_IRQ_MULTI_HANDLER or arm64 platforms, the handle_arch_irq is a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Tested on Sipeed Lichee Pi 4A (riscv64) board, the perf sched benchmark is improved by ~5.8%
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
As for the GENERIC_IRQ_MULTI_HANDLER platforms but without runtime const support, such as arm, csky, loongarch, microblaze and openrisc, there should be no changes.
Jisheng Zhang (3):
vmlinux.lds.h: add _handle_arch_irq RUNTIME_CONST section
genirq: use runtime constant to optimize handle_arch_irq access
arm64: use runtime constant to optimize handle_arch_irq access
arch/arm64/kernel/entry-common.c | 4 +++- arch/arm64/kernel/irq.c | 9 ++++++--- include/asm-generic/vmlinux.lds.h | 3 ++- include/linux/irq.h | 4 +++- kernel/irq/handle.c | 8 +++++--- 5 files changed, 19 insertions(+), 9 deletions(-)
Signature · 1 line
2.51.0
PATCH 1/3 vmlinux.lds.h: add _handle_arch_irq RUNTIME_CONST section
We will use the runtime constant to optimize the handle_arch_irq accessing. This is preparation patch.
--- include/asm-generic/vmlinux.lds.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
include/asm-generic/vmlinux.lds.h +2 -1
@@ -973,7 +973,8 @@ RUNTIME_CONST(shift, d_hash_shift) \ RUNTIME_CONST(ptr, dentry_hashtable) \ RUNTIME_CONST(ptr, __dentry_cache) \ -RUNTIME_CONST (ptr, __names_cache) + RUNTIME_CONST(ptr, __names_cache) \ +RUNTIME_CONST (ptr, _handle_arch_irq) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */#define KUNIT_TABLE () \
Signature · 1 line
2.51.0
Guo Ren
maintainerRe: [PATCH 1/3] vmlinux.lds.h: add _handle_arch_irq RUNTIME_CONST section
Quoted 24 lines · Jisheng Zhang
We will use the runtime constant to optimize the
handle_arch_irqaccessing. This is preparation patch.Signed-off-by Jisheng Zhang <[email protected]>--- include/asm-generic/vmlinux.lds.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)include/asm-generic/vmlinux.lds.h +2
-2@@ -973,7 +973,8 @@ RUNTIME_CONST(shift, d_hash_shift) \ RUNTIME_CONST(ptr, dentry_hashtable) \ RUNTIME_CONST(ptr, __dentry_cache) \ -RUNTIME_CONST (ptr, __names_cache) + RUNTIME_CONST(ptr, __names_cache) \ +RUNTIME_CONST (ptr, _handle_arch_irq) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */#define KUNIT_TABLE () \ --
Quoted 2 lines
2.51.0
Reviewed by: Guo Ren (Alibaba Damo Academy) <[email protected]>
Signature · 2 lines
Best Regards Guo Ren
PATCH 2/3 genirq: use runtime constant to optimize handle_arch_irq access
In reply to Jisheng ZhangCurrently, on GENERIC_IRQ_MULTI_HANDLER platforms, the handle_arch_irq is a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Tested on Sipeed Lichee Pi 4A (riscv64) board, the perf sched benchmark is improved by ~5.8%
--- include/linux/irq.h | 4 +++- kernel/irq/handle.c | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-)
include/linux/irq.h +3 -1
@@ -1274,6 +1274,7 @@ void ipi_mux_process(void);int ipi_mux_create (unsigned int nr_ipi,void (*mux_send)(unsigned int cpu));#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER +#include <asm/runtime-const.h> /* * Registers a generic IRQ handling function as the top-level IRQ handler in * the system, which is generally the first C code called from an assembly
@@ -1288,7 +1289,8 @@ int __init set_handle_irq(void (*handle_irq)(struct pt_regs *));* Allows interrupt handlers to find the irqchip that's been registered as the * top-level IRQ handler. */ -extern void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +extern void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq) asmlinkage void generic_handle_arch_irq(struct pt_regs *regs);#else #ifndef set_handle_irq
kernel/irq/handle.c +5 -3
@@ -15,13 +15,14 @@#include <linux/kernel_stat.h> #include <asm/irq_regs.h> +#include <asm/runtime-const.h> #include <trace/events/irq.h> #include "internals.h" #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ;#endif /**
@@ -270,10 +271,11 @@ irqreturn_t handle_irq_event(struct irq_desc *desc)#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq) +if (_handle_arch_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);return 0 ; }
Signature · 1 line
2.51.0
Thomas Gleixner
maintainerRe: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 5 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
The proper solution is to use a static call and update it in set_handle_irq(). That removes the complete indirect call issue from the hot path.
Thanks,
tglx
Re: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 10 lines · Thomas Gleixner
Quoted 5 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.The proper solution is to use a static call and update it in
set_handle_irq(). That removes the complete indirect call issue from the hot path.
+ Ard, Mark,
Good idea. The remaining problem is no static call support for current GENERIC_IRQ_MULTI_HANDLER (or similar, arm64 e.g) platforms.
For arm64, Ard tried to add the static call support[1] in 2021, but Mark concerned "compiler could easily violate our expectations in future"[2], and asked for static calls "critical rather than a nice-to-have" usage.
Hi Ard, Mark,
Could this irq performance improvement be used as a "critical" usage for arm64 static call? Per my test, about 6.5% improvement was seen on quad CA55.
Another alternative:
disable static call if CFI is enabled, and give the platform/SoC users chance to enable static call to benefit from it.
Any comment is appreciated.
Thanks
[1] https://www.spinics.net/lists/arm-kernel/msg931861.html
[2] https://www.spinics.net/lists/arm-kernel/msg932481.html
Quoted 4 lines
Thanks,
tglx
Mark Rutland
reviewerRe: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 8 lines · Jisheng Zhang
Quoted 7 lines · Thomas Gleixner
Quoted 5 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Quoted 8 lines
Quoted 3 lines
The proper solution is to use a static call and update it in
set_handle_irq(). That removes the complete indirect call issue from the hot path.+ Ard, Mark,
Good idea. The remaining problem is no static call support for current
GENERIC_IRQ_MULTI_HANDLER(or similar, arm64 e.g) platforms.
There are various reasons for not supporting static calls, and in general we end up having to have a fall-back path that's *more* expensive than just loading the pointer.
Quoted 3 lines
For arm64, Ard tried to add the static call support[1] in 2021, but Mark concerned "compiler could easily violate our expectations in future"[2],
To be clear, that's ONE specific concern, not the ONLY reason.
Quoted 7 lines
and asked for static calls "critical rather than a nice-to-have" usage.
Hi Ard, Mark,
Could this irq performance improvement be used as a "critical" usage for arm64 static call? Per my test, about 6.5% improvement was seen on quad CA55.
As per my other mail, does this meaningfully affect a real workload?
Quoted 3 lines
Another alternative: disable static call if CFI is enabled, and give the platform/SoC users chance to enable static call to benefit from it.
Who is this actually going to matter to?
Mark.
Quoted 15 lines
Any comment is appreciated.
Thanks
[1] https://www.spinics.net/lists/arm-kernel/msg931861.html
[2] https://www.spinics.net/lists/arm-kernel/msg932481.html
Quoted 4 lines
Thanks,
tglx
Re: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 13 lines · Mark Rutland
Quoted 12 lines · Jisheng Zhang
Quoted 10 lines · Thomas Gleixner
Quoted 5 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.The proper solution is to use a static call and update it in
set_handle_irq(). That removes the complete indirect call issue from the hot path.
Quoted 8 lines
Quoted 4 lines
+ Ard, Mark,
Good idea. The remaining problem is no static call support for current
GENERIC_IRQ_MULTI_HANDLER(or similar, arm64 e.g) platforms.There are various reasons for not supporting static calls, and in general we end up having to have a fall-back path that's *more* expensive than just loading the pointer.
indeed, if arch doesn't support static call, the fall-back addes one more loading overhead.
Quoted 10 lines
Quoted 4 lines
For arm64, Ard tried to add the static call support[1] in 2021, but Mark concerned "compiler could easily violate our expectations in future"[2],
To be clear, that's ONE specific concern, not the ONLY reason.
Quoted 3 lines
and asked for static calls "critical rather than a nice-to-have" usage.
Quoted 6 lines
Quoted 4 lines
Hi Ard, Mark,
Could this irq performance improvement be used as a "critical" usage for arm64 static call? Per my test, about 6.5% improvement was seen on quad CA55.
As per my other mail, does this meaningfully affect a real workload?
This improves generic irq processcing, I think all real workload is affected.
Quoted 7 lines
Quoted 4 lines
Another alternative: disable static call if CFI is enabled, and give the platform/SoC users chance to enable static call to benefit from it.
Who is this actually going to matter to?
Mark Rutland
reviewerRe: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 7 lines · Jisheng Zhang
Quoted 3 lines · Mark Rutland
There are various reasons for not supporting static calls, and in general we end up having to have a fall-back path that's *more* expensive than just loading the pointer.
indeed, if arch doesn't support static call, the fall-back addes one more loading overhead.
I think you've misunderstood my point.
I'm saying that *even if* arm64 supported static calls, we'd have to have dynamic fallback paths that are more expensive. For example, where branch range limitations force indirection via an out-of-line stub, adding an extra BL+RET pair. Note that was the case in the patches you linked from Ard.
[...]
Quoted 11 lines
Quoted 9 lines
Quoted 7 lines
and asked for static calls "critical rather than a nice-to-have" usage.
Hi Ard, Mark,
Could this irq performance improvement be used as a "critical" usage for arm64 static call? Per my test, about 6.5% improvement was seen on quad CA55.
As per my other mail, does this meaningfully affect a real workload?
This improves generic irq processcing, I think all real workload is affected.
I asked about meaningful impact. For a real workload, does this show up at all, or does this fall within the noise?
At present, I don't think your singular microbenchmark result changes our previous decisions regarding static calls. For various reasons, on arm64 static calls are nowhere near as significant an optimization (and can be worse).
I'd be happy to use a runtime constant (modulo my concerns about the initial value) given that the we already have the infrastructure and the maintenanace impact is minimal.
Mark.
Re: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
In reply to Jisheng ZhangQuoted 9 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.Tested on Sipeed Lichee Pi 4A (riscv64) board, the perf sched benchmark is improved by ~5.8%
Thx for the work. It's a visible improvement.
Compared to the original handler pointer approach, this solution introduces no observable drawbacks. Since the static_call alternative requires further analysis and consensus, adopting this implementation serves as a pragmatic interim step.
So, I give:
Quoted 66 lines
Signed-off-by Jisheng Zhang <[email protected]>--- include/linux/irq.h | 4 +++- kernel/irq/handle.c | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-)include/linux/irq.h +3
-1@@ -1274,6 +1274,7 @@ void ipi_mux_process(void);int ipi_mux_create (unsigned int nr_ipi,void (*mux_send)(unsigned int cpu));#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER +#include <asm/runtime-const.h> /* * Registers a generic IRQ handling function as the top-level IRQ handler in * the system, which is generally the first C code called from an assembly @@ -1288,7 +1289,8 @@ int __init set_handle_irq(void (*handle_irq)(struct pt_regs *));* Allows interrupt handlers to find the irqchip that's been registered as the * top-level IRQ handler. */ -extern void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +extern void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq) asmlinkage void generic_handle_arch_irq(struct pt_regs *regs);#else #ifndef set_handle_irqkernel/irq/handle.c +5
-4@@ -15,13 +15,14 @@#include <linux/kernel_stat.h> #include <asm/irq_regs.h> +#include <asm/runtime-const.h> #include <trace/events/irq.h> #include "internals.h" #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ;#endif /** @@ -270,10 +271,11 @@ irqreturn_t handle_irq_event(struct irq_desc *desc)#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq) +if (_handle_arch_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);return 0 ; } --2.51.0
Signature · 2 lines
Best Regards Guo Ren
Guo Ren
maintainerRe: [PATCH 2/3] genirq: use runtime constant to optimize handle_arch_irq access
Quoted 13 lines · Guo Ren
Quoted 9 lines · Jisheng Zhang
Currently, on
GENERIC_IRQ_MULTI_HANDLERplatforms, thehandle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.Tested on Sipeed Lichee Pi 4A (riscv64) board, the perf sched benchmark is improved by ~5.8%
Thx for the work. It's a visible improvement.
Quoted 7 lines
Compared to the original handler pointer approach, this solution introduces no observable drawbacks. Since the
static_callalternative requires further analysis and consensus, adopting this implementation serves as a pragmatic interim step.So, I give:
Oh, don't forget Mark's suggestion for the uninitialized handler, which is also suitable for genirq:
"That means that if set_handle_irq() isn't called, an IRQ will result in a call to that bogus address rather than default_handle_irq(), ..."
Maybe we need an empty _handle_arch_irq instance as a default for the early stage.
Quoted 72 lines
Quoted 65 lines
Signed-off-by Jisheng Zhang <[email protected]>--- include/linux/irq.h | 4 +++- kernel/irq/handle.c | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-)include/linux/irq.h +3
-1@@ -1274,6 +1274,7 @@ void ipi_mux_process(void);int ipi_mux_create (unsigned int nr_ipi,void (*mux_send)(unsigned int cpu));#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER +#include <asm/runtime-const.h> /* * Registers a generic IRQ handling function as the top-level IRQ handler in * the system, which is generally the first C code called from an assembly @@ -1288,7 +1289,8 @@ int __init set_handle_irq(void (*handle_irq)(struct pt_regs *));* Allows interrupt handlers to find the irqchip that's been registered as the * top-level IRQ handler. */ -extern void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +extern void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq) asmlinkage void generic_handle_arch_irq(struct pt_regs *regs);#else #ifndef set_handle_irqkernel/irq/handle.c +5
-4@@ -15,13 +15,14 @@#include <linux/kernel_stat.h> #include <asm/irq_regs.h> +#include <asm/runtime-const.h> #include <trace/events/irq.h> #include "internals.h" #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init ; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init ;#endif /** @@ -270,10 +271,11 @@ irqreturn_t handle_irq_event(struct irq_desc *desc)#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq) +if (_handle_arch_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);return 0 ; } --Quoted 2 lines
2.51.0
--
Best Regards
Guo Ren
Signature · 2 lines
Best Regards Guo Ren
PATCH 3/3 arm64: use runtime constant to optimize handle_arch_irq access
In reply to Jisheng ZhangCurrently, on arm64 platforms, the handle_arch_irq is a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
--- arch/arm64/kernel/entry-common.c | 4 +++- arch/arm64/kernel/irq.c | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-)
arch/arm64/kernel/entry-common.c +3 -1
@@ -25,6 +25,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/processor.h> +#include <asm/runtime-const.h> #include <asm/sdei.h> #include <asm/stacktrace.h> #include <asm/sysreg.h>
@@ -139,7 +140,8 @@ static void do_interrupt_handler(struct pt_regs *regs,set_irq_regs (old_regs); } -extern void (*handle_arch_irq)(struct pt_regs *); +extern void (*_handle_arch_irq)(struct pt_regs *); +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)extern void (*handle_arch_fiq)(struct pt_regs *); static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
arch/arm64/kernel/irq.c +6 -3
@@ -23,6 +23,7 @@#include <asm/daifflags.h> #include <asm/exception.h> #include <asm/numa.h> +#include <asm/runtime-const.h> #include <asm/softirq_stack.h> #include <asm/stacktrace.h> #include <asm/vmap_stack.h>
@@ -84,15 +85,17 @@ static void default_handle_fiq(struct pt_regs *regs)panic ("FIQ taken without a root FIQ handler\n" ); } -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)void (*handle_arch_fiq)(struct pt_regs *)__ro_after_init = default_handle_fiq;int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq != default_handle_irq) +if (_handle_arch_irq != default_handle_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);pr_info ("Root IRQ handler: %ps\n" , handle_irq);return 0 ; }
Signature · 1 line
2.51.0
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Hi Jisheng,
Quoted 7 lines · Jisheng Zhang
Currently, on arm64 platforms, the
handle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
6.5% is a quite high margin, especially for only one pointer's change. Maybe it is good to share more info for which compiler you are using, how you tested and the detailed results.
I played a bit on my juno board on CA73 cores with the command:
perf bench sched all
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------+
+---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% | |seccomp-notify (sec) | 46.848 | 46.938 | 46.973 | 46.920 | +2.98% | +---------------------+--------+--------+--------+--------+--------+
With this patch, the messaging test shows a minor improvement (0.52%).
The pipe test performs worse (-4.15%) after applying the patch. However, one positive signal is that the minimum latency is 23.159, which is lower than without the change (24.017).
For seccomp, the results indicate a benefit (2.98%) from the change.
Hope this is helpful for maintainers to judge the change.
I'd leave maintainers to review the code.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Quoted 1 line · Leo Yan
Hi Jisheng,
Hi Leo,
Quoted 7 lines · Jisheng Zhang
Quoted 5 lines
Currently, on arm64 platforms, the
handle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Quoted 6 lines
Quoted 2 lines
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
6.5% is a quite high margin, especially for only one pointer's change. Maybe it is good to share more info for which compiler you are using, how you tested and the detailed results.
Sure.
aarch64-linux-gnu-gcc version 15.2.0
my kernel defconfig is a minimal arm64 version which disables most drivers, only keep timer, gic, pll/clk, uart, regulator and i2c controller The reason is to avoid OS noise as much as possible.
It's also put at the end of the email for reference.
testing cmd:
perf bench sched pipe
testing steps:
booting into a buildroot minimal initramfs
force cpufreq governor as performance
run above cmd
testing resuls:
before the patch:
9.471988 usecs/op
105574 ops/sec
after the patch:
8.896280 usecs/op
112406 ops/sec
(112406 - 105574) * 100 / 105574 = ~6.5 %
Let me know if you need more details
Quoted 2 lines
I played a bit on my juno board on CA73 cores with the command:
IIRC, Juno is powered by a big.little SoC, then we need to ensure the benchmark is always running on big or little cores when testing, to ensure apple to apple comparison.
I also have a CA73 platform, but can't access it now. I'll test the patch next week.
Quoted 14 lines
perf bench sched all
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------+
Quoted 5 lines
+---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |
If you check the result, this result variance is abnormal, it means your OS is noiser.
Quoted 8 lines
|seccomp-notify (sec) | 46.848 | 46.938 | 46.973 | 46.920 | +2.98% | +---------------------+--------+--------+--------+--------+--------+With this patch, the messaging test shows a minor improvement (0.52%).
The pipe test performs worse (-4.15%) after applying the patch. However, one positive signal is that the minimum latency is 23.159, which is lower than without the change (24.017).
The perf bench is sensitive to OS noise, so I disable most devices, only uart, timer, irq chip, i2c and regulator is kept, and stop all daemons only keep a usespace process bash shell.
defconfig for reference:
# CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=yCONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=yCONFIG_PREEMPT=y
# CONFIG_CPU_ISOLATION is not set CONFIG_BLK_DEV_INITRD=y
# CONFIG_RD_GZIP is not set
# CONFIG_RD_BZIP2 is not set
# CONFIG_RD_LZMA is not set
# CONFIG_RD_XZ is not set
# CONFIG_RD_LZO is not set
# CONFIG_RD_LZ4 is not set
# CONFIG_INITRAMFS_PRESERVE_MTIME is not set CONFIG_EXPERT=y
# CONFIG_UID16 is not set
# CONFIG_SYSFS_SYSCALL is not set CONFIG_PROFILING=yCONFIG_ARCH_BERLIN=y
# CONFIG_ARM64_ERRATUM_826319 is not set
# CONFIG_ARM64_ERRATUM_827319 is not set
# CONFIG_ARM64_ERRATUM_824069 is not set
# CONFIG_ARM64_ERRATUM_819472 is not set
# CONFIG_ARM64_ERRATUM_832075 is not set
# CONFIG_ARM64_ERRATUM_845719 is not set
# CONFIG_ARM64_ERRATUM_843419 is not set
# CONFIG_ARM64_ERRATUM_1418040 is not set # CONFIG_ARM64_ERRATUM_1165522 is not set # CONFIG_ARM64_ERRATUM_1463225 is not set # CONFIG_CAVIUM_ERRATUM_22375 is not set
# CONFIG_CAVIUM_ERRATUM_23154 is not set
# CONFIG_CAVIUM_ERRATUM_27456 is not set
# CONFIG_CAVIUM_ERRATUM_30115 is not set
# CONFIG_CAVIUM_TX2_ERRATUM_219 is not set # CONFIG_FUJITSU_ERRATUM_010001 is not set # CONFIG_HISILICON_ERRATUM_161600802 is not set # CONFIG_QCOM_FALKOR_ERRATUM_1003 is not set # CONFIG_QCOM_FALKOR_ERRATUM_1009 is not set # CONFIG_QCOM_QDF2400_ERRATUM_0065 is not set # CONFIG_QCOM_FALKOR_ERRATUM_E1041 is not set # CONFIG_SOCIONEXT_SYNQUACER_PREITS is not set CONFIG_NR_CPUS=4CONFIG_HOTPLUG_CPU=yCONFIG_HZ_100=y
# CONFIG_UNMAP_KERNEL_AT_EL0 is not set CONFIG_COMPAT=yCONFIG_ARMV8_DEPRECATED=y CONFIG_SWP_EMULATION=y CONFIG_CP15_BARRIER_EMULATION=y CONFIG_SETEND_EMULATION=y
# CONFIG_ARM64_HW_AFDBM is not set
# CONFIG_ARM64_PTR_AUTH is not set
# CONFIG_ARM64_AMU_EXTN is not set
# CONFIG_ARM64_SVE is not set
# CONFIG_EFI is not set
# CONFIG_SUSPEND is not set CONFIG_CPU_IDLE=y CONFIG_CPU_IDLE_GOV_TEO=y CONFIG_ARM_PSCI_CPUIDLE=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPUFREQ_DT=yCONFIG_JUMP_LABEL=y
# CONFIG_VMAP_STACK is not set
# CONFIG_BLOCK is not set
# CONFIG_COMPAT_BRK is not set
# CONFIG_COMPACTION is not set CONFIG_CMA=y
# CONFIG_ZONE_DMA32 is not set
# CONFIG_VM_EVENT_COUNTERS is not set CONFIG_SYNA_DOLPHIN_PLL=y CONFIG_SYNA_CLK_BASE=y CONFIG_SYNA_MYNA2_CLK=y CONFIG_PINCTRL_MYNA2=y CONFIG_USB_DWC3_SYNA=y
# CONFIG_SYNAPTICS_PCIE is not set CONFIG_REGULATOR_HL7593=y CONFIG_PHY_SYNA_USB=y
# CONFIG_SYNAPTICS_I2C is not set CONFIG_BERLIN_CHIPID=y CONFIG_BERLIN_AXI_METER=y
# CONFIG_SYNAPTICS_NET is not set
# CONFIG_SYNAPTICS_VIDEO is not set
# CONFIG_SYNAPTICS_SOUND is not set
# CONFIG_SYNAPTICS_DEVFREQ is not set
# CONFIG_SYNAPTICS_DMABUF is not set
# CONFIG_SYNAPTICS_IRQCHIP is not set
# CONFIG_SYNAPTICS_INPUT is not set
# CONFIG_SYNAPTICS_BLUETOOTH is not set
# CONFIG_SYNAPTICS_CLOCKSOURCE is not set # CONFIG_SYNAPTICS_DMA is not set
# CONFIG_SYNAPTICS_TTY is not set
# CONFIG_SYNAPTICS_LEDS is not set
# CONFIG_SYNAPTICS_MFD is not set
# CONFIG_SYNAPTICS_MISC is not set
# CONFIG_SYNAPTICS_MTD is not set
# CONFIG_SYNAPTICS_PWM is not set
# CONFIG_SYNAPTICS_RTC is not set
# CONFIG_SYNAPTICS_SPI is not set
# CONFIG_SYNAPTICS_STAGING is not set
# CONFIG_SYNAPTICS_WATCHDOG is not set
# CONFIG_SYNAPTICS_MAILBOX is not set
# CONFIG_SYNAPTICS_RPMSG is not set
# CONFIG_SYNAPTICS_CRYPTO is not set CONFIG_UEVENT_HELPER=yCONFIG_DEVTMPFS=yCONFIG_DEVTMPFS_MOUNT=y
# CONFIG_ALLOW_DEV_COREDUMP is not set CONFIG_INPUT_EVDEV=y
# CONFIG_INPUT_KEYBOARD is not set
# CONFIG_INPUT_MOUSE is not set
# CONFIG_SERIO is not set
# CONFIG_VT is not set
# CONFIG_LEGACY_PTYS is not set
# CONFIG_LEGACY_TIOCSTI is not set CONFIG_SERIAL_8250=y
# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_NR_UARTS=2 CONFIG_SERIAL_8250_RUNTIME_UARTS=2 CONFIG_SERIAL_8250_DW=y
# CONFIG_HW_RANDOM is not set
# CONFIG_DEVPORT is not setCONFIG_I2C=yCONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_CORE=y CONFIG_GPIO_SYSFS=yCONFIG_GPIO_DWAPB=yCONFIG_GPIO_FXL6408=y
# CONFIG_HWMON is not set CONFIG_MFD_SYSCON=yCONFIG_REGULATOR=y CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_REGULATOR_GPIO=y CONFIG_REGULATOR_TPS6286X=y
# CONFIG_HID_SUPPORT is not set
# CONFIG_USB_SUPPORT is not set CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_SYSFS_STATS=y CONFIG_DMABUF_HEAPS_CMA=y
# CONFIG_VIRTIO_MENU is not set
# CONFIG_VHOST_MENU is not set
# CONFIG_SURFACE_PLATFORMS is not set
# CONFIG_ARM64_PLATFORM_DEVICES is not set # CONFIG_ARM_ARCH_TIMER_EVTSTREAM is not set # CONFIG_FSL_ERRATUM_A008585 is not set
# CONFIG_HISILICON_ERRATUM_161010101 is not set # CONFIG_ARM64_ERRATUM_858921 is not set
# CONFIG_IOMMU_SUPPORT is not set CONFIG_RESET_CONTROLLER=y
# CONFIG_DNOTIFY is not set
# CONFIG_PROC_PAGE_MONITOR is not set CONFIG_TMPFS=yCONFIG_CONFIGFS_FS=y
# CONFIG_MISC_FILESYSTEMS is not set CONFIG_DMA_CMA=yCONFIG_PRINTK_TIME=y
# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_DEBUG_FS=y
# CONFIG_SLUB_DEBUG is not set CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_STACKTRACE=y
# CONFIG_RCU_TRACE is not set
# CONFIG_FTRACE is not set
# CONFIG_STRICT_DEVMEM is not set
# CONFIG_RUNTIME_TESTING_MENU is not set
Quoted 9 lines
For seccomp, the results indicate a benefit (2.98%) from the change.
Hope this is helpful for maintainers to judge the change.
I'd leave maintainers to review the code.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Quoted 72 lines · Jisheng Zhang
Quoted 1 line · Leo Yan
Hi Jisheng,
Hi Leo,
Quoted 7 lines · Jisheng Zhang
Quoted 5 lines
Currently, on arm64 platforms, the
handle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.Quoted 6 lines
Quoted 2 lines
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
6.5% is a quite high margin, especially for only one pointer's change. Maybe it is good to share more info for which compiler you are using, how you tested and the detailed results.
Sure.
aarch64-linux-gnu-gcc version 15.2.0my kernel defconfig is a minimal arm64 version which disables most drivers, only keep timer, gic, pll/clk, uart, regulator and i2c controller The reason is to avoid OS noise as much as possible.
It's also put at the end of the email for reference.testing cmd:
perf bench sched pipetesting steps:
booting into a buildroot minimal initramfs
force cpufreq governor as performance
run above cmdtesting resuls:
before the patch:
9.471988 usecs/op
105574 ops/secafter the patch:
8.896280 usecs/op
112406 ops/sec(112406 - 105574) * 100 / 105574 = ~6.5 %
Let me know if you need more details
Quoted 2 lines
I played a bit on my juno board on CA73 cores with the command:
IIRC, Juno is powered by a big.little SoC, then we need to ensure the benchmark is always running on big or little cores when testing, to ensure apple to apple comparison.
I also have a CA73 platform, but can't access it now. I'll test the patch next week.
Quoted 14 lines
perf bench sched all
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------+
Quoted 8 lines
Quoted 5 lines
+---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |If you check the result, this result variance is abnormal, it means your OS is noiser.
BTW: if you remove the abnormal run3 result, you'll find that the benchmark is improved by ~3.5% on CA73:
(23.159 + 23.702) / 2 = 23.43
(24.258 + 24.224) / 2 = 24.24
(24.24 - 23.43)*100 / 23.43 = ~3.5
Quoted 193 lines
Quoted 9 lines
|seccomp-notify (sec) | 46.848 | 46.938 | 46.973 | 46.920 | +2.98% | +---------------------+--------+--------+--------+--------+--------+With this patch, the messaging test shows a minor improvement (0.52%).
The pipe test performs worse (-4.15%) after applying the patch. However, one positive signal is that the minimum latency is 23.159, which is lower than without the change (24.017).
The perf bench is sensitive to OS noise, so I disable most devices, only uart, timer, irq chip, i2c and regulator is kept, and stop all daemons only keep a usespace process bash shell.
defconfig for reference:
#
CONFIG_LOCALVERSION_AUTOis not setCONFIG_SYSVIPC=yCONFIG_NO_HZ_IDLE=yCONFIG_HIGH_RES_TIMERS=yCONFIG_PREEMPT=y
#CONFIG_CPU_ISOLATIONis not setCONFIG_BLK_DEV_INITRD=y
#CONFIG_RD_GZIPis not set
#CONFIG_RD_BZIP2is not set
#CONFIG_RD_LZMAis not set
#CONFIG_RD_XZis not set
#CONFIG_RD_LZOis not set
#CONFIG_RD_LZ4is not set
#CONFIG_INITRAMFS_PRESERVE_MTIMEis not setCONFIG_EXPERT=y
#CONFIG_UID16is not set
#CONFIG_SYSFS_SYSCALLis not setCONFIG_PROFILING=yCONFIG_ARCH_BERLIN=y
#CONFIG_ARM64_ERRATUM_826319is not set
#CONFIG_ARM64_ERRATUM_827319is not set
#CONFIG_ARM64_ERRATUM_824069is not set
#CONFIG_ARM64_ERRATUM_819472is not set
#CONFIG_ARM64_ERRATUM_832075is not set
#CONFIG_ARM64_ERRATUM_845719is not set
#CONFIG_ARM64_ERRATUM_843419is not set
#CONFIG_ARM64_ERRATUM_1418040is not set #CONFIG_ARM64_ERRATUM_1165522is not set #CONFIG_ARM64_ERRATUM_1463225is not set #CONFIG_CAVIUM_ERRATUM_22375is not set
#CONFIG_CAVIUM_ERRATUM_23154is not set
#CONFIG_CAVIUM_ERRATUM_27456is not set
#CONFIG_CAVIUM_ERRATUM_30115is not set
#CONFIG_CAVIUM_TX2_ERRATUM_219is not set #CONFIG_FUJITSU_ERRATUM_010001is not set #CONFIG_HISILICON_ERRATUM_161600802is not set #CONFIG_QCOM_FALKOR_ERRATUM_1003is not set #CONFIG_QCOM_FALKOR_ERRATUM_1009is not set #CONFIG_QCOM_QDF2400_ERRATUM_0065is not set #CONFIG_QCOM_FALKOR_ERRATUM_E1041is not set #CONFIG_SOCIONEXT_SYNQUACER_PREITSis not setCONFIG_NR_CPUS=4CONFIG_HOTPLUG_CPU=yCONFIG_HZ_100=y
#CONFIG_UNMAP_KERNEL_AT_EL0is not setCONFIG_COMPAT=yCONFIG_ARMV8_DEPRECATED=yCONFIG_SWP_EMULATION=yCONFIG_CP15_BARRIER_EMULATION=yCONFIG_SETEND_EMULATION=y
#CONFIG_ARM64_HW_AFDBMis not set
#CONFIG_ARM64_PTR_AUTHis not set
#CONFIG_ARM64_AMU_EXTNis not set
#CONFIG_ARM64_SVEis not set
#CONFIG_EFIis not set
#CONFIG_SUSPENDis not setCONFIG_CPU_IDLE=yCONFIG_CPU_IDLE_GOV_TEO=yCONFIG_ARM_PSCI_CPUIDLE=yCONFIG_CPU_FREQ=yCONFIG_CPU_FREQ_GOV_USERSPACE=yCONFIG_CPUFREQ_DT=yCONFIG_JUMP_LABEL=y
#CONFIG_VMAP_STACKis not set
#CONFIG_BLOCKis not set
#CONFIG_COMPAT_BRKis not set
#CONFIG_COMPACTIONis not setCONFIG_CMA=y
#CONFIG_ZONE_DMA32is not set
#CONFIG_VM_EVENT_COUNTERSis not setCONFIG_SYNA_DOLPHIN_PLL=yCONFIG_SYNA_CLK_BASE=yCONFIG_SYNA_MYNA2_CLK=yCONFIG_PINCTRL_MYNA2=yCONFIG_USB_DWC3_SYNA=y
#CONFIG_SYNAPTICS_PCIEis not setCONFIG_REGULATOR_HL7593=yCONFIG_PHY_SYNA_USB=y
#CONFIG_SYNAPTICS_I2Cis not setCONFIG_BERLIN_CHIPID=yCONFIG_BERLIN_AXI_METER=y
#CONFIG_SYNAPTICS_NETis not set
#CONFIG_SYNAPTICS_VIDEOis not set
#CONFIG_SYNAPTICS_SOUNDis not set
#CONFIG_SYNAPTICS_DEVFREQis not set
#CONFIG_SYNAPTICS_DMABUFis not set
#CONFIG_SYNAPTICS_IRQCHIPis not set
#CONFIG_SYNAPTICS_INPUTis not set
#CONFIG_SYNAPTICS_BLUETOOTHis not set
#CONFIG_SYNAPTICS_CLOCKSOURCEis not set #CONFIG_SYNAPTICS_DMAis not set
#CONFIG_SYNAPTICS_TTYis not set
#CONFIG_SYNAPTICS_LEDSis not set
#CONFIG_SYNAPTICS_MFDis not set
#CONFIG_SYNAPTICS_MISCis not set
#CONFIG_SYNAPTICS_MTDis not set
#CONFIG_SYNAPTICS_PWMis not set
#CONFIG_SYNAPTICS_RTCis not set
#CONFIG_SYNAPTICS_SPIis not set
#CONFIG_SYNAPTICS_STAGINGis not set
#CONFIG_SYNAPTICS_WATCHDOGis not set
#CONFIG_SYNAPTICS_MAILBOXis not set
#CONFIG_SYNAPTICS_RPMSGis not set
#CONFIG_SYNAPTICS_CRYPTOis not setCONFIG_UEVENT_HELPER=yCONFIG_DEVTMPFS=yCONFIG_DEVTMPFS_MOUNT=y
#CONFIG_ALLOW_DEV_COREDUMPis not setCONFIG_INPUT_EVDEV=y
#CONFIG_INPUT_KEYBOARDis not set
#CONFIG_INPUT_MOUSEis not set
#CONFIG_SERIOis not set
#CONFIG_VTis not set
#CONFIG_LEGACY_PTYSis not set
#CONFIG_LEGACY_TIOCSTIis not setCONFIG_SERIAL_8250=y
#CONFIG_SERIAL_8250_DEPRECATED_OPTIONSis not setCONFIG_SERIAL_8250_CONSOLE=yCONFIG_SERIAL_8250_NR_UARTS=2CONFIG_SERIAL_8250_RUNTIME_UARTS=2CONFIG_SERIAL_8250_DW=y
#CONFIG_HW_RANDOMis not set
#CONFIG_DEVPORTis not setCONFIG_I2C=yCONFIG_I2C_CHARDEV=yCONFIG_I2C_DESIGNWARE_CORE=yCONFIG_GPIO_SYSFS=yCONFIG_GPIO_DWAPB=yCONFIG_GPIO_FXL6408=y
#CONFIG_HWMONis not setCONFIG_MFD_SYSCON=yCONFIG_REGULATOR=yCONFIG_REGULATOR_FIXED_VOLTAGE=yCONFIG_REGULATOR_GPIO=yCONFIG_REGULATOR_TPS6286X=y
#CONFIG_HID_SUPPORTis not set
#CONFIG_USB_SUPPORTis not setCONFIG_DMABUF_HEAPS=yCONFIG_DMABUF_SYSFS_STATS=yCONFIG_DMABUF_HEAPS_CMA=y
#CONFIG_VIRTIO_MENUis not set
#CONFIG_VHOST_MENUis not set
#CONFIG_SURFACE_PLATFORMSis not set
#CONFIG_ARM64_PLATFORM_DEVICESis not set #CONFIG_ARM_ARCH_TIMER_EVTSTREAMis not set #CONFIG_FSL_ERRATUM_A008585is not set
#CONFIG_HISILICON_ERRATUM_161010101is not set #CONFIG_ARM64_ERRATUM_858921is not set
#CONFIG_IOMMU_SUPPORTis not setCONFIG_RESET_CONTROLLER=y
#CONFIG_DNOTIFYis not set
#CONFIG_PROC_PAGE_MONITORis not setCONFIG_TMPFS=yCONFIG_CONFIGFS_FS=y
#CONFIG_MISC_FILESYSTEMSis not setCONFIG_DMA_CMA=yCONFIG_PRINTK_TIME=y
#CONFIG_SECTION_MISMATCH_WARN_ONLYis not setCONFIG_DEBUG_FS=y
#CONFIG_SLUB_DEBUGis not setCONFIG_SOFTLOCKUP_DETECTOR=yCONFIG_DEBUG_ATOMIC_SLEEP=yCONFIG_STACKTRACE=y
#CONFIG_RCU_TRACEis not set
#CONFIG_FTRACEis not set
#CONFIG_STRICT_DEVMEMis not set
#CONFIG_RUNTIME_TESTING_MENUis not setQuoted 9 lines
For seccomp, the results indicate a benefit (2.98%) from the change.
Hope this is helpful for maintainers to judge the change.
I'd leave maintainers to review the code.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
On Fri, Feb 20, 2026 at 09:34:14PM +0800, Jisheng Zhang wrote:
[...]
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------++---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |
Quoted 8 lines
Quoted 2 lines
If you check the result, this result variance is abnormal, it means your OS is noiser.
BTW: if you remove the abnormal run3 result, you'll find that the benchmark is improved by ~3.5% on CA73:
(23.159 + 23.702) / 2 = 23.43
(24.258 + 24.224) / 2 = 24.24
(24.24 - 23.43)*100 / 23.43 = ~3.5
TBH, I don't think we should subjectively select data. But I agree a clean test env is important to avoid noise, and I also agree that the current results already show positive signals.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Quoted 24 lines · Leo Yan
On Fri, Feb 20, 2026 at 09:34:14PM +0800, Jisheng Zhang wrote:
[...]
Quoted 20 lines
Quoted 19 lines
Quoted 11 lines
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------+Quoted 5 lines
+---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |If you check the result, this result variance is abnormal, it means your OS is noiser.
Quoted 7 lines
Quoted 5 lines
BTW: if you remove the abnormal run3 result, you'll find that the benchmark is improved by ~3.5% on CA73:
(23.159 + 23.702) / 2 = 23.43
(24.258 + 24.224) / 2 = 24.24
(24.24 - 23.43)*100 / 23.43 = ~3.5TBH, I don't think we should subjectively select data. But I agree a
The precondition of this is testing the benchmark properly. And I just tried perf bench sched in noisy OS, I didn't get the similar abnormal variance as you got, so I think your run3 result was CA53's result. This isn't an apple-to-apple comparison.
If possible, could you plz test after forcing CA53 offline or test on non big.little platform. Anyway, I will test CA73 next week too.
Quoted 5 lines
clean test env is important to avoid noise, and I also agree that the current results already show positive signals.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
On Sat, Feb 21, 2026 at 08:14:17AM +0800, Jisheng Zhang wrote:
[...]
Quoted 31 lines · Leo Yan
Quoted 30 lines · Jisheng Zhang
[...]
Quoted 25 lines
Quoted 11 lines
Quoted 11 lines
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------+Quoted 8 lines
Quoted 5 lines
+---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |If you check the result, this result variance is abnormal, it means your OS is noiser.
BTW: if you remove the abnormal run3 result, you'll find that the benchmark is improved by ~3.5% on CA73:
(23.159 + 23.702) / 2 = 23.43
(24.258 + 24.224) / 2 = 24.24
(24.24 - 23.43)*100 / 23.43 = ~3.5
Quoted 6 lines
Quoted 1 line
TBH, I don't think we should subjectively select data. But I agree a
The precondition of this is testing the benchmark properly. And I just tried perf bench sched in noisy OS, I didn't get the similar abnormal variance as you got, so I think your run3 result was CA53's result. This isn't an apple-to-apple comparison.
Not true. As said, I tested on CA73. I should say explicitly that I have hotplugged off CA53 CPUs and run test only on CA73 CPUs.
Quoted 8 lines
If possible, could you plz test after forcing CA53 offline or test on non big.little platform. Anyway, I will test CA73 next week too.
Quoted 5 lines
clean test env is important to avoid noise, and I also agree that the current results already show positive signals.
Thanks,
Leo
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Quoted 37 lines · Leo Yan
On Sat, Feb 21, 2026 at 08:14:17AM +0800, Jisheng Zhang wrote:
[...]
Quoted 33 lines · Leo Yan
Quoted 31 lines · Jisheng Zhang
[...]
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
Run 3 iterations, and measures three metrics (messaging/pipe/seccomp) and results in seconds. Less is better.
+---------------------+--------+--------+--------+--------+ |Without change | run1 | run2 | run3 | avg | +---------------------+--------+--------+--------+--------+ |messaging (sec) | 4.546 | 4.508 | 4.591 | 4.548 | |pipe (sec) | 24.258 | 24.224 | 24.017 | 24.166 | |seccomp-notify (sec) | 48.393 | 48.457 | 48.232 | 48.361 | +---------------------+--------+--------+--------+--------++---------------------+--------+--------+--------+--------+--------+ |With change | run1 | run2 | run3 | avg | diff | +---------------------+--------+--------+--------+--------+--------+ |messaging (sec) | 4.493 | 4.523 | 4.556 | 4.524 | +0.52% | |pipe (sec) | 23.159 | 23.702 | 28.649 | 25.170 | -4.15% |Quoted 8 lines
Quoted 2 lines
If you check the result, this result variance is abnormal, it means your OS is noiser.
BTW: if you remove the abnormal run3 result, you'll find that the benchmark is improved by ~3.5% on CA73:
(23.159 + 23.702) / 2 = 23.43
(24.258 + 24.224) / 2 = 24.24
(24.24 - 23.43)*100 / 23.43 = ~3.5TBH, I don't think we should subjectively select data. But I agree a
Quoted 7 lines
Quoted 4 lines
The precondition of this is testing the benchmark properly. And I just tried perf bench sched in noisy OS, I didn't get the similar abnormal variance as you got, so I think your run3 result was CA53's result. This isn't an apple-to-apple comparison.
Not true. As said, I tested on CA73. I should say explicitly that I have hotplugged off CA53 CPUs and run test only on CA73 CPUs.
I tested on quad CA73 platform, I can reproduce the abnormal variance as you got. This means the series may not alway improve performance as I expected for *all* CPUs. So I'd like to drop it now.
Quoted 9 lines
Quoted 9 lines
If possible, could you plz test after forcing CA53 offline or test on non big.little platform. Anyway, I will test CA73 next week too.
Quoted 5 lines
clean test env is important to avoid noise, and I also agree that the current results already show positive signals.
Thanks,
Leo
Mark Rutland
reviewerRe: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
In reply to Jisheng ZhangQuoted 7 lines · Jisheng Zhang
Currently, on arm64 platforms, the
handle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
That is a surprisingly large impact. :/
Does this meaningfully actually affect any real workload?
Quoted 19 lines
Signed-off-by Jisheng Zhang <[email protected]>--- arch/arm64/kernel/entry-common.c | 4 +++- arch/arm64/kernel/irq.c | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-)arch/arm64/kernel/entry-common.c +1
-0@@ -25,6 +25,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/processor.h> +#include <asm/runtime-const.h> #include <asm/sdei.h> #include <asm/stacktrace.h> #include <asm/sysreg.h>
arch/arm64/kernel/entry-common.c static void do_interrupt_handler(struct pt_regs *regs, · line 145
Quoted 7 lines
arch/arm64/kernel/entry-common.c +2
-1@@ -139,7 +140,8 @@ static void do_interrupt_handler(struct pt_regs *regs,set_irq_regs (old_regs); } -extern void (*handle_arch_irq)(struct pt_regs *); +extern void (*_handle_arch_irq)(struct pt_regs *); +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)extern void (*handle_arch_fiq)(struct pt_regs *);
We should treat handle_arch_irq and handle_arch_fiq the same way. Either both get this, or neither do.
Quoted 14 lines
arch/arm64/kernel/entry-common.c +0
-0static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,arch/arm64/kernel/irq.c +1
-0@@ -23,6 +23,7 @@#include <asm/daifflags.h> #include <asm/exception.h> #include <asm/numa.h> +#include <asm/runtime-const.h> #include <asm/softirq_stack.h> #include <asm/stacktrace.h> #include <asm/vmap_stack.h>
arch/arm64/kernel/irq.c static void default_handle_fiq(struct pt_regs *regs) · line 89
Quoted 6 lines
arch/arm64/kernel/irq.c +2
-1@@ -84,15 +85,17 @@ static void default_handle_fiq(struct pt_regs *regs)panic ("FIQ taken without a root FIQ handler\n" ); } -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)
This breaks the default case, since handle_arch_irq is initialized to a bunch of garbage hex bytes (0x0123456789abcdef).
That means that if set_handle_irq() isn't called, an IRQ will result in a call to that bogus address rather than default_handle_irq(), which'll be more difficult to debug.
Mark.
Quoted 18 lines
arch/arm64/kernel/irq.c +3
-2void (*handle_arch_fiq)(struct pt_regs *)__ro_after_init = default_handle_fiq;int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq != default_handle_irq) +if (_handle_arch_irq != default_handle_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);pr_info ("Root IRQ handler: %ps\n" , handle_irq);return 0 ; }Signature · 1 line
2.51.0
Re: [PATCH 3/3] arm64: use runtime constant to optimize handle_arch_irq access
Quoted 6 lines · Mark Rutland
Quoted 5 lines · Jisheng Zhang
Currently, on arm64 platforms, the
handle_arch_irqis a pointer which is set during booting, and every irq processing needs to access it, so it sits in hot code path. We can use the runtime constant mechanism which was introduced by Linus to speed up its accessing.
Quoted 6 lines
Quoted 2 lines
Tested on Quad CA55 platform, the perf sched benchmark is improved by ~6.5%
That is a surprisingly large impact. :/
Does this meaningfully actually affect any real workload?
all irqs' processing is improved to this extent. The perf sched bench(an existing and good benchmark to measure IPI) is used to show how much will be the improvement.
Quoted 55 lines
Quoted 20 lines
Signed-off-by Jisheng Zhang <[email protected]>--- arch/arm64/kernel/entry-common.c | 4 +++- arch/arm64/kernel/irq.c | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-)arch/arm64/kernel/entry-common.c +1
-0@@ -25,6 +25,7 @@#include <asm/kprobes.h> #include <asm/mmu.h> #include <asm/processor.h> +#include <asm/runtime-const.h> #include <asm/sdei.h> #include <asm/stacktrace.h> #include <asm/sysreg.h> arch/arm64/kernel/entry-common.c static void do_interrupt_handler(struct pt_regs *regs, · line 145
Quoted 7 lines
arch/arm64/kernel/entry-common.c +2
-1@@ -139,7 +140,8 @@ static void do_interrupt_handler(struct pt_regs *regs,set_irq_regs (old_regs); } -extern void (*handle_arch_irq)(struct pt_regs *); +extern void (*_handle_arch_irq)(struct pt_regs *); +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)extern void (*handle_arch_fiq)(struct pt_regs *);We should treat
handle_arch_irqandhandle_arch_fiqthe same way. Either both get this, or neither do.Quoted 14 lines
arch/arm64/kernel/entry-common.c +0
-0static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,arch/arm64/kernel/irq.c +1
-0@@ -23,6 +23,7 @@#include <asm/daifflags.h> #include <asm/exception.h> #include <asm/numa.h> +#include <asm/runtime-const.h> #include <asm/softirq_stack.h> #include <asm/stacktrace.h> #include <asm/vmap_stack.h> arch/arm64/kernel/irq.c static void default_handle_fiq(struct pt_regs *regs) · line 89
Quoted 6 lines
arch/arm64/kernel/irq.c +2
-1@@ -84,15 +85,17 @@ static void default_handle_fiq(struct pt_regs *regs)panic ("FIQ taken without a root FIQ handler\n" ); } -void (*handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +void (*_handle_arch_irq)(struct pt_regs *)__ro_after_init = default_handle_irq; +#define handle_arch_irq runtime_const_ptr(_handle_arch_irq)This breaks the default case, since
handle_arch_irqis initialized to a bunch of garbage hex bytes (0x0123456789abcdef).
Quoted 3 lines
That means that if
set_handle_irq()isn't called, an IRQ will result in a call to that bogus address rather thandefault_handle_irq(), which'll be more difficult to debug.
Oops, you're right. If runtime constants is chosen, I will address this comment. While Thomas suggested static call instead. I found you concerned with the arm64's static call implementation 5 years ago, I mentioned this in another email a few seconds ago. Could you plz comment the thread?
Quoted 21 lines
Mark.
Quoted 18 lines
arch/arm64/kernel/irq.c +3
-2void (*handle_arch_fiq)(struct pt_regs *)__ro_after_init = default_handle_fiq;int __init set_handle_irq (void (*handle_irq)(struct pt_regs *)) { -if (handle_arch_irq != default_handle_irq) +if (_handle_arch_irq != default_handle_irq)return -EBUSY ; - handle_arch_irq = handle_irq; + _handle_arch_irq = handle_irq; +runtime_const_init (ptr, _handle_arch_irq);pr_info ("Root IRQ handler: %ps\n" , handle_irq);return 0 ; }Signature · 1 line
2.51.0