x86/alternatives: remove false sharing in poke_int3_handler()
PATCH v2 1/1 x86/alternatives: remove false sharing in poke_int3_handler()
eBPF programs can be run 50,000,000 times per second on busy servers.
Whenever /proc/sys/kernel/bpf_stats_enabled is turned off, hundreds of calls sites are patched from text_poke_bp_batch() and we see a huge loss of performance due to false sharing on bp_desc.refs lasting up to three seconds.
51.30% server_bin [kernel.kallsyms] [k] poke_int3_handler
|
|--46.45%--poke_int3_handler
| exc_int3
| asm_exc_int3
| |
| |--24.26%--cls_bpf_classify
| | tcf_classify
| | __dev_queue_xmit
| | ip6_finish_output2
| | ip6_output
| | ip6_xmit
| | inet6_csk_xmit
| | __tcp_transmit_skb
| | |
| | |--9.00%--tcp_v6_do_rcv
| | | tcp_v6_rcv
| | | ip6_protocol_deliver_rcu
| | | ip6_rcv_finish
| | | ipv6_rcv
| | | __netif_receive_skb
| | | process_backlog
| | | __napi_poll
| | | net_rx_action
| | | __softirqentry_text_start
| | | asm_call_sysvec_on_stack
| | | do_softirq_own_stackFix this by replacing bp_desc.refs with a per-cpu bp_refs.
Before the patch, on a host with 240 cores (480 threads):
$ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 3009063719 run_cnt 82757845
-> average cost is 36 nsec per call
echo 0 >/proc/sys/kernel/bpf_stats_enabledtext_poke_bp_batch(nr_entries=2)
text_poke_bp_batch+1
text_poke_finish+27
arch_jump_label_transform_apply+22
jump_label_update+98
__static_key_slow_dec_cpuslocked+64
static_key_slow_dec+31
bpf_stats_handler+236
proc_sys_call_handler+396
vfs_write+761
ksys_write+102
do_syscall_64+107
entry_SYSCALL_64_after_hwframe+103Took 324 usec
text_poke_bp_batch(nr_entries=164)
text_poke_bp_batch+1
text_poke_finish+27
arch_jump_label_transform_apply+22
jump_label_update+98
__static_key_slow_dec_cpuslocked+64
static_key_slow_dec+31
bpf_stats_handler+236
proc_sys_call_handler+396
vfs_write+761
ksys_write+102
do_syscall_64+107
entry_SYSCALL_64_after_hwframe+103Took 2655300 usec
After this patch:
$ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 1928223019 run_cnt 67682728
-> average cost is 28 nsec per call
echo 0 >/proc/sys/kernel/bpf_stats_enabledtext_poke_bp_batch(nr_entries=2)
text_poke_bp_batch+1
text_poke_finish+27
arch_jump_label_transform_apply+22
jump_label_update+98
__static_key_slow_dec_cpuslocked+64
static_key_slow_dec+31
bpf_stats_handler+236
proc_sys_call_handler+396
vfs_write+761
ksys_write+102
do_syscall_64+107
entry_SYSCALL_64_after_hwframe+103Took 519 usec
text_poke_bp_batch(nr_entries=164)
text_poke_bp_batch+1
text_poke_finish+27
arch_jump_label_transform_apply+22
jump_label_update+98
__static_key_slow_dec_cpuslocked+64
static_key_slow_dec+31
bpf_stats_handler+236
proc_sys_call_handler+396
vfs_write+761
ksys_write+102
do_syscall_64+107
entry_SYSCALL_64_after_hwframe+103Took 702 usec
--- arch/x86/kernel/alternative.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-)
arch/x86/kernel/alternative.c +18 -12
@@ -2137,28 +2137,29 @@ struct text_poke_loc {struct bp_patching_desc {struct text_poke_loc *vec;int nr_entries; -atomic_t refs; }; +static DEFINE_PER_CPU(atomic_t, bp_refs); +static struct bp_patching_desc bp_desc;static __always_inline struct bp_patching_desc *try_get_desc(void) { -struct bp_patching_desc *desc = &bp_desc; +atomic_t *refs =this_cpu_ptr (&bp_refs); -if (!raw_atomic_inc_not_zero (&desc->refs)) +if (!raw_atomic_inc_not_zero (refs))return NULL ; -return desc; +return &bp_desc; } static __always_inline void put_desc(void) { -struct bp_patching_desc *desc = &bp_desc; +atomic_t *refs =this_cpu_ptr (&bp_refs);smp_mb__before_atomic (); -raw_atomic_dec (&desc->refs); +raw_atomic_dec (refs); } static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
@@ -2191,9 +2192,9 @@ noinstr int poke_int3_handler(struct pt_regs *regs)* Having observed our INT3 instruction, we now must observe * bp_desc with non-zero refcount: * -* bp_desc.refs = 1 INT3 -* WMB RMB -* write INT3 if (bp_desc.refs != 0) +* bp_refs = 1 INT3 +* WMB RMB +* write INT3 if (bp_refs != 0) */ smp_rmb ();
@@ -2299,7 +2300,8 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries* Corresponds to the implicit memory barrier in try_get_desc() to * ensure reading a non-zero refcount provides up to date bp_desc data. */ -atomic_set_release (&bp_desc.refs,1 ); +for_each_possible_cpu (i) +atomic_set_release (per_cpu_ptr (&bp_refs, i),1 );/* * Function tracing can enable thousands of places that need to be
@@ -2413,8 +2415,12 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries/* * Remove and wait for refs to be zero. */ -if (!atomic_dec_and_test (&bp_desc.refs)) -atomic_cond_read_acquire (&bp_desc.refs, !VAL ); +for_each_possible_cpu (i) { +atomic_t *refs =per_cpu_ptr (&bp_refs, i); + +if (unlikely (!atomic_dec_and_test (refs))) +atomic_cond_read_acquire (refs, !VAL ); + } } static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
Signature · 1 line
2.49.0.395.g12beb8f557-goog
Quoted 115 lines · Eric Dumazet
eBPF programs can be run 50,000,000 times per second on busy servers.
Whenever
/proc/sys/kernel/bpf_stats_enabledis turned off, hundreds of calls sites are patched fromtext_poke_bp_batch()and we see a huge loss of performance due to false sharing on bp_desc.refs lasting up to three seconds.51.30%
server_bin[kernel.kallsyms] [k]poke_int3_handler| |--46.45%--poke_int3_handler | exc_int3 | asm_exc_int3 | | | |--24.26%--cls_bpf_classify | | tcf_classify | | __dev_queue_xmit | | ip6_finish_output2 | | ip6_output | | ip6_xmit | | inet6_csk_xmit | | __tcp_transmit_skb | | | | | |--9.00%--tcp_v6_do_rcv | | | tcp_v6_rcv | | | ip6_protocol_deliver_rcu | | | ip6_rcv_finish | | | ipv6_rcv | | | __netif_receive_skb | | | process_backlog | | | __napi_poll | | | net_rx_action | | | __softirqentry_text_start | | | asm_call_sysvec_on_stack | | | do_softirq_own_stackFix this by replacing bp_desc.refs with a per-cpu
bp_refs.Before the patch, on a host with 240 cores (480 threads):
$ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 3009063719 run_cnt 82757845-> average cost is 36 nsec per call
echo 0 >/proc/sys/kernel/bpf_stats_enabled
text_poke_bp_batch(nr_entries=2)text_poke_bp_batch+1 text_poke_finish+27 arch_jump_label_transform_apply+22 jump_label_update+98 __static_key_slow_dec_cpuslocked+64 static_key_slow_dec+31 bpf_stats_handler+236 proc_sys_call_handler+396 vfs_write+761 ksys_write+102 do_syscall_64+107 entry_SYSCALL_64_after_hwframe+103Took 324 usec
text_poke_bp_batch(nr_entries=164)text_poke_bp_batch+1 text_poke_finish+27 arch_jump_label_transform_apply+22 jump_label_update+98 __static_key_slow_dec_cpuslocked+64 static_key_slow_dec+31 bpf_stats_handler+236 proc_sys_call_handler+396 vfs_write+761 ksys_write+102 do_syscall_64+107 entry_SYSCALL_64_after_hwframe+103Took 2655300 usec
After this patch:
$ bpftool prog | grep run_time_ns ... 105: sched_cls name hn_egress tag 699fc5eea64144e3 gpl run_time_ns 1928223019 run_cnt 67682728-> average cost is 28 nsec per call
echo 0 >/proc/sys/kernel/bpf_stats_enabled
text_poke_bp_batch(nr_entries=2)text_poke_bp_batch+1 text_poke_finish+27 arch_jump_label_transform_apply+22 jump_label_update+98 __static_key_slow_dec_cpuslocked+64 static_key_slow_dec+31 bpf_stats_handler+236 proc_sys_call_handler+396 vfs_write+761 ksys_write+102 do_syscall_64+107 entry_SYSCALL_64_after_hwframe+103Took 519 usec
text_poke_bp_batch(nr_entries=164)text_poke_bp_batch+1 text_poke_finish+27 arch_jump_label_transform_apply+22 jump_label_update+98 __static_key_slow_dec_cpuslocked+64 static_key_slow_dec+31 bpf_stats_handler+236 proc_sys_call_handler+396 vfs_write+761 ksys_write+102 do_syscall_64+107 entry_SYSCALL_64_after_hwframe+103
Quoted 1 line
Took 702 usec
This is unreadable due to the amount of pointless repetition.
Also, urgh.
Quoted 4 lines · Peter Zijlstra
This is unreadable due to the amount of pointless repetition.
Thanks, I am trimming the changelog in v3.