bpf: add bpf_get_cpu_cycles kfunc
Subsystems (7)
PATCH v5 1/4 bpf: add bpf_get_cpu_cycles kfunc
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to __arch_get_hw_counter().
---
v4 -> v5:
* use if instead of ifdef with IS_ENABLED
v3 -> v4:
* change name of the helper to bpf_get_cpu_cycles (Andrii)
* Hide the helper behind CONFIG_GENERIC_GETTIMEOFDAY to avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper to bpf_get_cpu_cycles_counter to explicitly mention what counter it provides (Andrii)
* move kfunc definition to bpf.h to use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii)
v1 -> v2:
* Fix incorrect function return value type to u64
* Introduce bpf_jit_inlines_kfunc_call() and use it in mark_fastcall_pattern_for_call() to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
--- arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++ arch/x86/net/bpf_jit_comp32.c | 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 + kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++- 7 files changed, 101 insertions(+), 1 deletion(-)
arch/x86/net/bpf_jit_comp.c +28 -0
@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off))case BPF_JMP |BPF_CALL : {u8 *ip = image + addrs[i -1 ]; +if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE (); +EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 )); +/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG ); + +break ; + } + func = (u8 *) __bpf_call_base + imm32;if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR (bpf_prog->aux->stack_depth);
@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) {return 0 ; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ +if (imm ==BPF_CALL_IMM (bpf_get_cpu_cycles)) +return true; +return false; +}
arch/x86/net/bpf_jit_comp32.c +14 -0
@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL ) {int err; +if (imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT3 (0x0F ,0xAE ,0xE8 ); +EMIT2 (0x0F ,0x31 ); +break ; + } + err =emit_kfunc_call (bpf_prog, image + addrs[i], insn, &prog);
@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) {return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ +if (imm ==BPF_CALL_IMM (bpf_get_cpu_cycles)) +return true; +return false; +}
include/linux/bpf.h +5 -0
@@ -3328,6 +3328,11 @@ void bpf_user_rnd_init_once(void);u64 bpf_user_rnd_u32 (u64 r1,u64 r2,u64 r3,u64 r4,u64 r5);u64 bpf_get_raw_cpu_id (u64 r1,u64 r2,u64 r3,u64 r4,u64 r5); +/* Inlined kfuncs */ +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +u64 bpf_get_cpu_cycles (void ); +#endif +#if defined(CONFIG_NET ) bool bpf_sock_common_is_valid_access(int off, int size, enum bpf_access_type type,
include/linux/filter.h +1 -0
@@ -1111,6 +1111,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);void bpf_jit_compile (struct bpf_prog *prog);bool bpf_jit_needs_zext (void );bool bpf_jit_inlines_helper_call (s32 imm); +bool bpf_jit_inlines_kfunc_call (s32 imm);bool bpf_jit_supports_subprog_tailcalls (void );bool bpf_jit_supports_percpu_insn (void );bool bpf_jit_supports_kfunc_call (void );
kernel/bpf/core.c +11 -0
@@ -2965,6 +2965,17 @@ bool __weak bpf_jit_inlines_helper_call(s32 imm)return false; } +/* Return true if the JIT inlines the call to the kfunc corresponding to +* the imm. +* +* The verifier will not patch the insn->imm for the call to the helper if +* this returns true. +*/ +bool __weak bpf_jit_inlines_kfunc_call (s32 imm) +{ +return false; +} +/* Return TRUE if the JIT backend supports mixing bpf2bpf and tailcalls. */ bool __weak bpf_jit_supports_subprog_tailcalls (void ) {
kernel/bpf/helpers.c +13 -0
@@ -23,6 +23,9 @@#include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"
@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userreturn ret +1 ; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ +return __arch_get_hw_counter (1 ,NULL ); +} +#endif +__bpf_kfunc_end_defs ();BTF_KFUNCS_START (generic_btf_ids)
@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END (common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {
kernel/bpf/verifier.c +29 -1
@@ -16273,6 +16273,24 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) } } +/* True if fixup_kfunc_call() replaces calls to kfunc number 'imm', +* replacement patch is presumed to follow bpf_fastcall contract +* (see mark_fastcall_pattern_for_call() below). +*/ +static bool verifier_inlines_kfunc_call (struct bpf_verifier_env *env,s32 imm) +{ +const struct bpf_kfunc_desc *desc =find_kfunc_desc (env->prog, imm,0 ); + +if (!env->prog->jit_requested) +return false; + +if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) +return true; + +return false; +} +/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */ static u32 kfunc_fastcall_clobber_mask (struct bpf_kfunc_call_arg_meta *meta) {
@@ -16400,7 +16418,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,return ; clobbered_regs_mask =kfunc_fastcall_clobber_mask (&meta); - can_be_inlined =is_fastcall_kfunc_call (&meta); + can_be_inlined =is_fastcall_kfunc_call (&meta) && + (verifier_inlines_kfunc_call (env, call->imm) || + (meta.btf == btf_vmlinux && +bpf_jit_inlines_kfunc_call (call->imm))); }if (clobbered_regs_mask ==ALL_CALLER_SAVED_REGS )
@@ -20402,6 +20423,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { const struct bpf_kfunc_desc *desc; +s32 imm = insn->imm;if (!insn->imm) {verbose (env,"invalid kernel function call not eliminated in verifier pass\n" );
@@ -20488,6 +20510,12 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, node_offset_reg, insn, insn_buf, cnt); }else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { +if (!verifier_inlines_kfunc_call (env, imm)) { +verbose (env,"verifier internal error: kfunc id %d is not defined in checker\n" , + desc->func_id); +return -EFAULT ; + } + insn_buf[0 ] =BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ); *cnt =1 ; }else if (is_bpf_wq_set_callback_impl_kfunc (desc->func_id)) {
Signature · 1 line
2.43.5
PATCH v5 2/4 bpf: add bpf_cpu_cycles_to_ns helper
The new helper should be used to convert cycles received by bpf_get_cpu_cycle() into nanoseconds.
---
v5:
* no changes
v4:
* change helper name to bpf_cpu_cycles_to_ns.
* hide it behind CONFIG_GENERIC_GETTIMEOFDAY to avoid exposing on unsupported architectures.
--- arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++ arch/x86/net/bpf_jit_comp32.c | 19 +++++++++++++++++++ include/linux/bpf.h | 1 + kernel/bpf/helpers.c | 10 +++++++++- 4 files changed, 51 insertions(+), 1 deletion(-)
arch/x86/net/bpf_jit_comp.c +22 -0
@@ -11,6 +11,7 @@#include <linux/bpf.h> #include <linux/memory.h> #include <linux/sort.h> +#include <linux/clocksource.h> #include <asm/extable.h> #include <asm/ftrace.h> #include <asm/set_memory.h>
@@ -2146,6 +2147,24 @@ st: if (is_imm8(insn->off))break ; } +if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_cpu_cycles_to_ns) && +boot_cpu_has (X86_FEATURE_CONSTANT_TSC )) { +u32 mult, shift; + +clocks_calc_mult_shift (&mult, &shift, tsc_khz,USEC_PER_SEC ,0 ); +/* imul RAX, RDI, mult */ +maybe_emit_mod (&prog,BPF_REG_1 ,BPF_REG_0 , true); +EMIT2_off32 (0x69 ,add_2reg (0xC0 ,BPF_REG_1 ,BPF_REG_0 ), + mult); + +/* shr RAX, shift (which is less than 64) */ +maybe_emit_1mod (&prog,BPF_REG_0 , true); +EMIT3 (0xC1 ,add_1reg (0xE8 ,BPF_REG_0 ), shift); + +break ; + } + func = (u8 *) __bpf_call_base + imm32;if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR (bpf_prog->aux->stack_depth);
@@ -3678,5 +3697,8 @@ bool bpf_jit_inlines_kfunc_call(s32 imm) {if (imm ==BPF_CALL_IMM (bpf_get_cpu_cycles))return true; +if (imm ==BPF_CALL_IMM (bpf_cpu_cycles_to_ns) && +boot_cpu_has (X86_FEATURE_CONSTANT_TSC )) +return true;return false; }
arch/x86/net/bpf_jit_comp32.c +19 -0
@@ -12,6 +12,7 @@#include <linux/netdevice.h> #include <linux/filter.h> #include <linux/if_vlan.h> +#include <linux/clocksource.h> #include <asm/cacheflush.h> #include <asm/set_memory.h> #include <asm/nospec-branch.h>
@@ -2100,6 +2101,24 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,EMIT2 (0x0F ,0x31 );break ; } +if (imm32 ==BPF_CALL_IMM (bpf_cpu_cycles_to_ns) && +boot_cpu_has (X86_FEATURE_CONSTANT_TSC )) { +u32 mult, shift; + +clocks_calc_mult_shift (&mult, &shift, tsc_khz, +USEC_PER_SEC ,0 ); + +/* move parameter to BPF_REG_0 */ +emit_ia32_mov_r64 (true, bpf2ia32[BPF_REG_0 ], + bpf2ia32[BPF_REG_1 ], true, true, + &prog, bpf_prog->aux); +/* multiply parameter by mut */ +emit_ia32_mul_i64 (bpf2ia32[BPF_REG_0 ], + mult, true, &prog); +/* shift parameter by shift which is less than 64 */ +emit_ia32_rsh_i64 (bpf2ia32[BPF_REG_0 ], + shift, true, &prog); + } err = emit_kfunc_call(bpf_prog, image + addrs[i],
include/linux/bpf.h +1 -0
@@ -3331,6 +3331,7 @@ u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);/* Inlined kfuncs */ #if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY )u64 bpf_get_cpu_cycles (void ); +u64 bpf_cpu_cycles_to_ns (u64 cycles);#endif #if defined(CONFIG_NET )
kernel/bpf/helpers.c +9 -1
@@ -25,6 +25,7 @@#include <linux/kasan.h> #if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY )#include <vdso/datapage.h> +#include <asm/vdso/vsyscall.h> #endif #include "../../lib/kstrtox.h"
@@ -3031,8 +3032,14 @@ __bpf_kfunc u64 bpf_get_cpu_cycles(void) {return __arch_get_hw_counter (1 ,NULL ); } -#endif +__bpf_kfunc u64 bpf_cpu_cycles_to_ns(u64 cycles) +{ +const struct vdso_data *vd =__arch_get_k_vdso_data (); + +return mul_u64_u32_shr (cycles, vd->mult, vd->shift); +} +#endif __bpf_kfunc_end_defs ();BTF_KFUNCS_START (generic_btf_ids)
@@ -3127,6 +3134,7 @@ BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLE BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_cpu_cycles_to_ns, KF_FASTCALL)#endif BTF_KFUNCS_END (common_btf_ids)
Signature · 1 line
2.43.5
Re: [PATCH bpf-next v5 2/4] bpf: add bpf_cpu_cycles_to_ns helper
Quoted 5 lines · Vadim Fedorenko
The new helper should be used to convert cycles received by
bpf_get_cpu_cycle()into nanoseconds.Signed-off-by Vadim Fedorenko <[email protected]>---
Tried this with and without invtsc flag for qemu, the switch between call / inlined version occurs as expected.
In the off-list discussion Vadim explained that:
- mult and shift constants are not updated after boot if constant_tsc feature is present for CPU.
- despite gettimeofday.h:vdso_calc_ns() doing more complex calculations to avoid negative time motion, we again assume that this is not needed for BPF use-case (two measurements close in time). (If there would be a v6, it would be nice to have a comment about these differences, I think).
[...]
PATCH v5 3/4 selftests/bpf: add selftest to check rdtsc jit
In reply to Vadim Fedorenkoget_cpu_cycles() is replaced with rdtsc instruction on x86_64. Add tests to check it.
--- .../selftests/bpf/prog_tests/verifier.c | 2 + .../selftests/bpf/progs/verifier_cpu_cycles.c | 104 ++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_cpu_cycles.c
tools/testing/selftests/bpf/prog_tests/verifier.c +2 -0
@@ -97,6 +97,7 @@#include "verifier_xdp_direct_packet_access.skel.h" #include "verifier_bits_iter.skel.h" #include "verifier_lsm.skel.h" +#include "verifier_cpu_cycles.skel.h" #define MAX_ENTRIES 11
@@ -223,6 +224,7 @@ void test_verifier_xdp(void) { RUN(verifier_xdp); }void test_verifier_xdp_direct_packet_access (void ) {RUN (verifier_xdp_direct_packet_access); }void test_verifier_bits_iter (void ) {RUN (verifier_bits_iter); }void test_verifier_lsm (void ) {RUN (verifier_lsm); } +void test_verifier_cpu_cycles (void ) {RUN (verifier_cpu_cycles); }void test_verifier_mtu (void ) {
tools/testing/selftests/bpf/progs/verifier_cpu_cycles.c new file +104 -0
104 lines not on this page · raw message
Signature · 1 line
2.43.5
Re: [PATCH bpf-next v5 3/4] selftests/bpf: add selftest to check rdtsc jit
Quoted 5 lines · Vadim Fedorenko
get_cpu_cycles()is replaced with rdtsc instruction onx86_64. Add tests to check it.Signed-off-by Vadim Fedorenko <[email protected]>---
PATCH v5 4/4 selftests/bpf: add usage example for cpu cycles kfuncs
In reply to Vadim FedorenkoThe selftest provides an example of how to measure the latency of bpf kfunc/helper call in cycles and nanoseconds.
--- .../bpf/prog_tests/test_cpu_cycles.c | 35 +++++++++++++++++++ .../selftests/bpf/progs/test_cpu_cycles.c | 25 +++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_cpu_cycles.c create mode 100644 tools/testing/selftests/bpf/progs/test_cpu_cycles.c
tools/testing/selftests/bpf/prog_tests/test_cpu_cycles.c new file +35 -0
@@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Inc. */ + +#include <test_progs.h> +#include "test_cpu_cycles.skel.h" + +static void cpu_cycles (void ) +{ +LIBBPF_OPTS (bpf_test_run_opts, opts); +struct test_cpu_cycles *skel; +int err, pfd; + + skel =test_cpu_cycles__open_and_load (); +if (!ASSERT_OK_PTR (skel,"test_cpu_cycles open and load" )) +return ; + + pfd =bpf_program__fd (skel->progs.bpf_cpu_cycles); +if (!ASSERT_GT (pfd,0 ,"test_cpu_cycles fd" )) + goto fail; + + err =bpf_prog_test_run_opts (pfd, &opts); +if (!ASSERT_OK (err,"test_cpu_cycles test run" )) + goto fail; + +ASSERT_NEQ (skel->bss->cycles,0 ,"test_cpu_cycles 0 cycles" ); +ASSERT_NEQ (skel->bss->ns,0 ,"test_cpu_cycles 0 ns" ); +fail: +test_cpu_cycles__destroy (skel); +} + +void test_cpu_cycles (void ) +{ +if (test__start_subtest ("cpu_cycles" )) +cpu_cycles (); +}
tools/testing/selftests/bpf/progs/test_cpu_cycles.c new file +25 -0
@@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Inc. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +extern u64 bpf_cpu_cycles_to_ns(u64 cycles) __weak __ksym; +extern u64 bpf_get_cpu_cycles(void) __weak __ksym; + +__u64 cycles, ns; + +SEC("syscall") +int bpf_cpu_cycles (void ) +{ +struct bpf_pidns_info pidns; +__u64 start; + + start =bpf_get_cpu_cycles (); +bpf_get_ns_current_pid_tgid (0 ,0 , &pidns,sizeof (struct bpf_pidns_info )); + cycles =bpf_get_cpu_cycles () - start; + ns =bpf_cpu_cycles_to_ns (cycles); +return 0 ; +} + +char _license[] SEC("license") = "GPL";
Signature · 1 line
2.43.5
Quoted 8 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
nit: please add cover letter for the next revision, multi-patch sets generally should come with a cover letter, unless it's some set of trivial and mostly independent patches. Anyways...
I haven't yet looked through the code (yet), but I was curious to benchmark the perf benefit, so that's what I did for fun this evening.
(!!!) BTW, a complete aside, but I think it's interesting. It turned out that using bpf_test_prog_run() scales *VERY POORLY* with large number of CPUs, because we start spending tons of time in fdget()/fdput(), so I initially got pretty unscalable results, profiled a bit, and then switched to just doing syscall(syscall(__NR_getpgid); + SEC("raw_tp/sys_enter")). Anyways, the point is that microbenchmarking is tricky and we need to improve our existing bench setup for some benchmarks. Anyways, getting back to the main topic.
I wrote a quick two benchmarks testing what I see as intended use case for these kfuncs (batching amortizes the cost of triggering BPF program, batch_iters = 500 in my case):
SEC("?raw_tp/sys_enter")
int trigger_driver_ktime(void *ctx)
{
volatile __u64 total = 0;
int i;
for (i = 0; i < batch_iters; i++) {
__u64 start, end;
start = bpf_ktime_get_ns();
end = bpf_ktime_get_ns();
total += end - start;
}
inc_counter_batch(batch_iters);
return 0;
}extern __u64 bpf_get_cpu_cycles(void) __weak __ksym; extern __u64 bpf_cpu_cycles_to_ns(__u64 cycles) __weak __ksym;
SEC("?raw_tp/sys_enter")
int trigger_driver_cycles(void *ctx)
{
volatile __u64 total = 0;
int i;
for (i = 0; i < batch_iters; i++) {
__u64 start, end;
start = bpf_get_cpu_cycles();
end = bpf_get_cpu_cycles();
total += bpf_cpu_cycles_to_ns(end - start);
}
inc_counter_batch(batch_iters);
return 0;
}And here's what I got across multiple numbers of parallel CPUs on our production host.
# ./bench_timing.sh
ktime ( 1 cpus): 32.286 ± 0.309M/s ( 32.286M/s/cpu) ktime ( 2 cpus): 63.021 ± 0.538M/s ( 31.511M/s/cpu) ktime ( 3 cpus): 94.211 ± 0.686M/s ( 31.404M/s/cpu) ktime ( 4 cpus): 124.757 ± 0.691M/s ( 31.189M/s/cpu) ktime ( 5 cpus): 154.855 ± 0.693M/s ( 30.971M/s/cpu) ktime ( 6 cpus): 185.551 ± 2.304M/s ( 30.925M/s/cpu) ktime ( 7 cpus): 211.117 ± 4.755M/s ( 30.160M/s/cpu) ktime ( 8 cpus): 236.454 ± 0.226M/s ( 29.557M/s/cpu) ktime (10 cpus): 295.526 ± 0.126M/s ( 29.553M/s/cpu) ktime (12 cpus): 322.282 ± 0.153M/s ( 26.857M/s/cpu) ktime (14 cpus): 375.347 ± 0.087M/s ( 26.811M/s/cpu) ktime (16 cpus): 399.813 ± 0.181M/s ( 24.988M/s/cpu) ktime (24 cpus): 617.675 ± 7.053M/s ( 25.736M/s/cpu) ktime (32 cpus): 819.695 ± 0.231M/s ( 25.615M/s/cpu) ktime (40 cpus): 996.264 ± 0.290M/s ( 24.907M/s/cpu) ktime (48 cpus): 1180.201 ± 0.160M/s ( 24.588M/s/cpu) ktime (56 cpus): 1321.084 ± 0.099M/s ( 23.591M/s/cpu) ktime (64 cpus): 1482.061 ± 0.121M/s ( 23.157M/s/cpu) ktime (72 cpus): 1666.540 ± 0.460M/s ( 23.146M/s/cpu) ktime (80 cpus): 1851.419 ± 0.439M/s ( 23.143M/s/cpu)
cycles ( 1 cpus): 45.815 ± 0.018M/s ( 45.815M/s/cpu) cycles ( 2 cpus): 86.706 ± 0.068M/s ( 43.353M/s/cpu) cycles ( 3 cpus): 129.899 ± 0.101M/s ( 43.300M/s/cpu) cycles ( 4 cpus): 168.435 ± 0.073M/s ( 42.109M/s/cpu) cycles ( 5 cpus): 210.520 ± 0.164M/s ( 42.104M/s/cpu) cycles ( 6 cpus): 252.596 ± 0.050M/s ( 42.099M/s/cpu) cycles ( 7 cpus): 294.356 ± 0.159M/s ( 42.051M/s/cpu) cycles ( 8 cpus): 317.167 ± 0.163M/s ( 39.646M/s/cpu) cycles (10 cpus): 396.141 ± 0.208M/s ( 39.614M/s/cpu) cycles (12 cpus): 431.938 ± 0.511M/s ( 35.995M/s/cpu) cycles (14 cpus): 503.055 ± 0.070M/s ( 35.932M/s/cpu) cycles (16 cpus): 534.261 ± 0.107M/s ( 33.391M/s/cpu) cycles (24 cpus): 836.838 ± 0.141M/s ( 34.868M/s/cpu) cycles (32 cpus): 1099.689 ± 0.314M/s ( 34.365M/s/cpu) cycles (40 cpus): 1336.573 ± 0.015M/s ( 33.414M/s/cpu) cycles (48 cpus): 1571.734 ± 11.151M/s ( 32.744M/s/cpu) cycles (56 cpus): 1819.242 ± 4.627M/s ( 32.486M/s/cpu) cycles (64 cpus): 2046.285 ± 5.169M/s ( 31.973M/s/cpu) cycles (72 cpus): 2287.683 ± 0.787M/s ( 31.773M/s/cpu) cycles (80 cpus): 2505.414 ± 0.626M/s ( 31.318M/s/cpu)
So, from about +42% on a single CPU, to +36% at 80 CPUs. Not that bad. Scalability-wise, we still see some drop off in performance, but believe me, with bpf_prog_test_run() it was so much worse :) I also verified that now we spend cycles almost exclusively inside the BPF program (so presumably in those benchmarked kfuncs).
Quoted 7 lines · Andrii Nakryiko
Quoted 6 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().
Quoted 6 lines
Quoted 2 lines
Signed-off-by Vadim Fedorenko <[email protected]>---
nit: please add cover letter for the next revision, multi-patch sets generally should come with a cover letter, unless it's some set of trivial and mostly independent patches. Anyways...
Yeah, sure. This series has grown from the first small version...
Quoted 104 lines
I haven't yet looked through the code (yet), but I was curious to benchmark the perf benefit, so that's what I did for fun this evening.
(!!!) BTW, a complete aside, but I think it's interesting. It turned out that using
bpf_test_prog_run()scales *VERY POORLY* with large number of CPUs, because we start spending tons of time in fdget()/fdput(), so I initially got pretty unscalable results, profiled a bit, and then switched to just doingsyscall(syscall(__NR_getpgid); + SEC("raw_tp/sys_enter")). Anyways, the point is that microbenchmarking is tricky and we need to improve our existing bench setup for some benchmarks. Anyways, getting back to the main topic.I wrote a quick two benchmarks testing what I see as intended use case for these kfuncs (batching amortizes the cost of triggering BPF program,
batch_iters= 500 in my case):
SEC("?raw_tp/sys_enter")
inttrigger_driver_ktime(void *ctx){ volatile __u64 total = 0; int i; for (i = 0; i < batch_iters; i++) { __u64 start, end; start = bpf_ktime_get_ns(); end = bpf_ktime_get_ns(); total += end - start; } inc_counter_batch(batch_iters); return 0; }extern
__u64bpf_get_cpu_cycles(void)__weak__ksym; extern__u64bpf_cpu_cycles_to_ns(__u64 cycles)__weak__ksym;
SEC("?raw_tp/sys_enter")
inttrigger_driver_cycles(void *ctx){ volatile __u64 total = 0; int i; for (i = 0; i < batch_iters; i++) { __u64 start, end; start = bpf_get_cpu_cycles(); end = bpf_get_cpu_cycles(); total += bpf_cpu_cycles_to_ns(end - start); } inc_counter_batch(batch_iters); return 0; }And here's what I got across multiple numbers of parallel CPUs on our production host.
# ./bench_timing.shktime ( 1 cpus): 32.286 ± 0.309M/s ( 32.286M/s/cpu) ktime ( 2 cpus): 63.021 ± 0.538M/s ( 31.511M/s/cpu) ktime ( 3 cpus): 94.211 ± 0.686M/s ( 31.404M/s/cpu) ktime ( 4 cpus): 124.757 ± 0.691M/s ( 31.189M/s/cpu) ktime ( 5 cpus): 154.855 ± 0.693M/s ( 30.971M/s/cpu) ktime ( 6 cpus): 185.551 ± 2.304M/s ( 30.925M/s/cpu) ktime ( 7 cpus): 211.117 ± 4.755M/s ( 30.160M/s/cpu) ktime ( 8 cpus): 236.454 ± 0.226M/s ( 29.557M/s/cpu) ktime (10 cpus): 295.526 ± 0.126M/s ( 29.553M/s/cpu) ktime (12 cpus): 322.282 ± 0.153M/s ( 26.857M/s/cpu) ktime (14 cpus): 375.347 ± 0.087M/s ( 26.811M/s/cpu) ktime (16 cpus): 399.813 ± 0.181M/s ( 24.988M/s/cpu) ktime (24 cpus): 617.675 ± 7.053M/s ( 25.736M/s/cpu) ktime (32 cpus): 819.695 ± 0.231M/s ( 25.615M/s/cpu) ktime (40 cpus): 996.264 ± 0.290M/s ( 24.907M/s/cpu) ktime (48 cpus): 1180.201 ± 0.160M/s ( 24.588M/s/cpu) ktime (56 cpus): 1321.084 ± 0.099M/s ( 23.591M/s/cpu) ktime (64 cpus): 1482.061 ± 0.121M/s ( 23.157M/s/cpu) ktime (72 cpus): 1666.540 ± 0.460M/s ( 23.146M/s/cpu) ktime (80 cpus): 1851.419 ± 0.439M/s ( 23.143M/s/cpu)cycles ( 1 cpus): 45.815 ± 0.018M/s ( 45.815M/s/cpu) cycles ( 2 cpus): 86.706 ± 0.068M/s ( 43.353M/s/cpu) cycles ( 3 cpus): 129.899 ± 0.101M/s ( 43.300M/s/cpu) cycles ( 4 cpus): 168.435 ± 0.073M/s ( 42.109M/s/cpu) cycles ( 5 cpus): 210.520 ± 0.164M/s ( 42.104M/s/cpu) cycles ( 6 cpus): 252.596 ± 0.050M/s ( 42.099M/s/cpu) cycles ( 7 cpus): 294.356 ± 0.159M/s ( 42.051M/s/cpu) cycles ( 8 cpus): 317.167 ± 0.163M/s ( 39.646M/s/cpu) cycles (10 cpus): 396.141 ± 0.208M/s ( 39.614M/s/cpu) cycles (12 cpus): 431.938 ± 0.511M/s ( 35.995M/s/cpu) cycles (14 cpus): 503.055 ± 0.070M/s ( 35.932M/s/cpu) cycles (16 cpus): 534.261 ± 0.107M/s ( 33.391M/s/cpu) cycles (24 cpus): 836.838 ± 0.141M/s ( 34.868M/s/cpu) cycles (32 cpus): 1099.689 ± 0.314M/s ( 34.365M/s/cpu) cycles (40 cpus): 1336.573 ± 0.015M/s ( 33.414M/s/cpu) cycles (48 cpus): 1571.734 ± 11.151M/s ( 32.744M/s/cpu) cycles (56 cpus): 1819.242 ± 4.627M/s ( 32.486M/s/cpu) cycles (64 cpus): 2046.285 ± 5.169M/s ( 31.973M/s/cpu) cycles (72 cpus): 2287.683 ± 0.787M/s ( 31.773M/s/cpu) cycles (80 cpus): 2505.414 ± 0.626M/s ( 31.318M/s/cpu)
Quoted 5 lines
So, from about +42% on a single CPU, to +36% at 80 CPUs. Not that bad. Scalability-wise, we still see some drop off in performance, but believe me, with
bpf_prog_test_run()it was so much worse :) I also verified that now we spend cycles almost exclusively inside the BPF program (so presumably in those benchmarked kfuncs).
Am I right that the numbers show how many iterations were done during the very same amount of time? It would be also great to understand if we get more precise measurements - just in case you have your tests ready...
Quoted 122 lines · Vadim Fedorenko
Quoted 7 lines · Andrii Nakryiko
Quoted 6 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Quoted 6 lines
Quoted 2 lines
Signed-off-by Vadim Fedorenko <[email protected]>---
nit: please add cover letter for the next revision, multi-patch sets generally should come with a cover letter, unless it's some set of trivial and mostly independent patches. Anyways...
Yeah, sure. This series has grown from the first small version...
Quoted 104 lines
I haven't yet looked through the code (yet), but I was curious to benchmark the perf benefit, so that's what I did for fun this evening.
(!!!) BTW, a complete aside, but I think it's interesting. It turned out that using
bpf_test_prog_run()scales *VERY POORLY* with large number of CPUs, because we start spending tons of time in fdget()/fdput(), so I initially got pretty unscalable results, profiled a bit, and then switched to just doingsyscall(syscall(__NR_getpgid); + SEC("raw_tp/sys_enter")). Anyways, the point is that microbenchmarking is tricky and we need to improve our existing bench setup for some benchmarks. Anyways, getting back to the main topic.I wrote a quick two benchmarks testing what I see as intended use case for these kfuncs (batching amortizes the cost of triggering BPF program,
batch_iters= 500 in my case):
SEC("?raw_tp/sys_enter")
inttrigger_driver_ktime(void *ctx){ volatile __u64 total = 0; int i; for (i = 0; i < batch_iters; i++) { __u64 start, end; start = bpf_ktime_get_ns(); end = bpf_ktime_get_ns(); total += end - start; } inc_counter_batch(batch_iters); return 0; }extern
__u64bpf_get_cpu_cycles(void)__weak__ksym; extern__u64bpf_cpu_cycles_to_ns(__u64 cycles)__weak__ksym;
SEC("?raw_tp/sys_enter")
inttrigger_driver_cycles(void *ctx){ volatile __u64 total = 0; int i; for (i = 0; i < batch_iters; i++) { __u64 start, end; start = bpf_get_cpu_cycles(); end = bpf_get_cpu_cycles(); total += bpf_cpu_cycles_to_ns(end - start); } inc_counter_batch(batch_iters); return 0; }And here's what I got across multiple numbers of parallel CPUs on our production host.
# ./bench_timing.shktime ( 1 cpus): 32.286 ± 0.309M/s ( 32.286M/s/cpu) ktime ( 2 cpus): 63.021 ± 0.538M/s ( 31.511M/s/cpu) ktime ( 3 cpus): 94.211 ± 0.686M/s ( 31.404M/s/cpu) ktime ( 4 cpus): 124.757 ± 0.691M/s ( 31.189M/s/cpu) ktime ( 5 cpus): 154.855 ± 0.693M/s ( 30.971M/s/cpu) ktime ( 6 cpus): 185.551 ± 2.304M/s ( 30.925M/s/cpu) ktime ( 7 cpus): 211.117 ± 4.755M/s ( 30.160M/s/cpu) ktime ( 8 cpus): 236.454 ± 0.226M/s ( 29.557M/s/cpu) ktime (10 cpus): 295.526 ± 0.126M/s ( 29.553M/s/cpu) ktime (12 cpus): 322.282 ± 0.153M/s ( 26.857M/s/cpu) ktime (14 cpus): 375.347 ± 0.087M/s ( 26.811M/s/cpu) ktime (16 cpus): 399.813 ± 0.181M/s ( 24.988M/s/cpu) ktime (24 cpus): 617.675 ± 7.053M/s ( 25.736M/s/cpu) ktime (32 cpus): 819.695 ± 0.231M/s ( 25.615M/s/cpu) ktime (40 cpus): 996.264 ± 0.290M/s ( 24.907M/s/cpu) ktime (48 cpus): 1180.201 ± 0.160M/s ( 24.588M/s/cpu) ktime (56 cpus): 1321.084 ± 0.099M/s ( 23.591M/s/cpu) ktime (64 cpus): 1482.061 ± 0.121M/s ( 23.157M/s/cpu) ktime (72 cpus): 1666.540 ± 0.460M/s ( 23.146M/s/cpu) ktime (80 cpus): 1851.419 ± 0.439M/s ( 23.143M/s/cpu)cycles ( 1 cpus): 45.815 ± 0.018M/s ( 45.815M/s/cpu) cycles ( 2 cpus): 86.706 ± 0.068M/s ( 43.353M/s/cpu) cycles ( 3 cpus): 129.899 ± 0.101M/s ( 43.300M/s/cpu) cycles ( 4 cpus): 168.435 ± 0.073M/s ( 42.109M/s/cpu) cycles ( 5 cpus): 210.520 ± 0.164M/s ( 42.104M/s/cpu) cycles ( 6 cpus): 252.596 ± 0.050M/s ( 42.099M/s/cpu) cycles ( 7 cpus): 294.356 ± 0.159M/s ( 42.051M/s/cpu) cycles ( 8 cpus): 317.167 ± 0.163M/s ( 39.646M/s/cpu) cycles (10 cpus): 396.141 ± 0.208M/s ( 39.614M/s/cpu) cycles (12 cpus): 431.938 ± 0.511M/s ( 35.995M/s/cpu) cycles (14 cpus): 503.055 ± 0.070M/s ( 35.932M/s/cpu) cycles (16 cpus): 534.261 ± 0.107M/s ( 33.391M/s/cpu) cycles (24 cpus): 836.838 ± 0.141M/s ( 34.868M/s/cpu) cycles (32 cpus): 1099.689 ± 0.314M/s ( 34.365M/s/cpu) cycles (40 cpus): 1336.573 ± 0.015M/s ( 33.414M/s/cpu) cycles (48 cpus): 1571.734 ± 11.151M/s ( 32.744M/s/cpu) cycles (56 cpus): 1819.242 ± 4.627M/s ( 32.486M/s/cpu) cycles (64 cpus): 2046.285 ± 5.169M/s ( 31.973M/s/cpu) cycles (72 cpus): 2287.683 ± 0.787M/s ( 31.773M/s/cpu) cycles (80 cpus): 2505.414 ± 0.626M/s ( 31.318M/s/cpu)
Quoted 8 lines
Quoted 5 lines
So, from about +42% on a single CPU, to +36% at 80 CPUs. Not that bad. Scalability-wise, we still see some drop off in performance, but believe me, with
bpf_prog_test_run()it was so much worse :) I also verified that now we spend cycles almost exclusively inside the BPF program (so presumably in those benchmarked kfuncs).Am I right that the numbers show how many iterations were done during the very same amount of time? It would be also great to understand if
yeah, it's millions of operations per second
Quoted 3 lines
we get more precise measurements - just in case you have your tests ready...
not sure what precision you mean here?... I didn't plan to publish tests, but I can push what I have into a local branch if you'd like to play with this
Quoted 7 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
Aside from a note below, I think this patch is in good shape.
[...]
Quoted 14 lines
arch/x86/net/bpf_jit_comp.c +6
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off))case BPF_JMP |BPF_CALL : {u8 *ip = image + addrs[i -1 ]; +if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE ();
arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +10
-0+EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 )); +/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );
Note: The default implementation of this kfunc uses __arch_get_hw_counter(),
which is implemented as (u64)rdtsc_ordered() & S64_MAX. Here we don't do & S64_MAX.
The masking in __arch_get_hw_counter() was added by this commit:
77750f78b0b3 ("x86/vdso: Fix gettimeofday masking").
Also, the default implementation does not issue lfence. Not sure if this makes any real-world difference.
Quoted 7 lines
arch/x86/net/bpf_jit_comp.c +4
-0+ +break ; + } + func = (u8 *) __bpf_call_base + imm32;if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR (bpf_prog->aux->stack_depth);
[...]
arch/x86/net/bpf_jit_comp.c static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 20518
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +6
-0@@ -20488,6 +20510,12 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, node_offset_reg, insn, insn_buf, cnt); }else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { +if (!verifier_inlines_kfunc_call (env, imm)) { +verbose (env,"verifier internal error: kfunc id %d is not defined in checker\n" , + desc->func_id); +return -EFAULT ; + } +
Nit: still think that moving this check as the first conditional would
have been better:
if (verifier_inlines_kfunc_call(env, imm)) {
if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
// ...
} else {
// report error
}
} else if (...) {
// ... rest of the cases
}Quoted 3 lines
arch/x86/net/bpf_jit_comp.c +0
-0insn_buf[0 ] =BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ); *cnt =1 ; }else if (is_bpf_wq_set_callback_impl_kfunc (desc->func_id)) {
Quoted 37 lines · Eduard Zingerman
Quoted 7 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
Aside from a note below, I think this patch is in good shape.
Acked-by Eduard Zingerman <[email protected]>[...]
Quoted 22 lines
arch/x86/net/bpf_jit_comp.c +14
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off))case BPF_JMP |BPF_CALL : {u8 *ip = image + addrs[i -1 ]; +if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE (); +EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 ));
Quoted 8 lines
arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 2 lines
arch/x86/net/bpf_jit_comp.c +2
-0+/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").
I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.
Well, it actually does. rdtsc_ordered is translated into lfence; rdtsc or rdtscp (which is rdtsc + lfence + u32 cookie) depending on the cpu features.
Quoted 22 lines
Quoted 7 lines
arch/x86/net/bpf_jit_comp.c +4
-0+ +break ; + } + func = (u8 *) __bpf_call_base + imm32;if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR (bpf_prog->aux->stack_depth);[...]
arch/x86/net/bpf_jit_comp.c static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 20518
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +6
-0@@ -20488,6 +20510,12 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, node_offset_reg, insn, insn_buf, cnt); }else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { +if (!verifier_inlines_kfunc_call (env, imm)) { +verbose (env,"verifier internal error: kfunc id %d is not defined in checker\n" , + desc->func_id); +return -EFAULT ; + } +Nit: still think that moving this check as the first conditional would
Quoted 12 lines
have been better:
if (verifier_inlines_kfunc_call(env, imm)) {
if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
// ...
} else {
// report error
}
} else if (...) {
// ... rest of the cases
}I can change it in the next iteration (if it's needed) if you insist
Quoted 5 lines
Quoted 3 lines
arch/x86/net/bpf_jit_comp.c +0
-0insn_buf[0 ] =BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ); *cnt =1 ; }else if (is_bpf_wq_set_callback_impl_kfunc (desc->func_id)) {
Eduard Zingerman
maintainerOn Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 14 lines
Quoted 14 lines
arch/x86/net/bpf_jit_comp.c +14
-0+if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE (); +EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 ));Quoted 8 lines
arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 2 lines
arch/x86/net/bpf_jit_comp.c +2
-0+/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.
I see the following disassembly:
0000000000008980 <bpf_get_cpu_cycles>:
; {
8980: f3 0f 1e fa endbr64
8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9>
0000000000008985: R_X86_64_PLT32 __fentry__-0x4
; asm volatile(ALTERNATIVE_2("rdtsc",
8989: 0f 31 rdtsc
898b: 90 nop
898c: 90 nop
898d: 90 nop
; return EAX_EDX_VAL(val, low, high);
898e: 48 c1 e2 20 shlq $0x20, %rdx
8992: 48 09 d0 orq %rdx, %rax
8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF
; return (u64)rdtsc_ordered() & S64_MAX;
899f: 48 21 c8 andq %rcx, %rax
; return __arch_get_hw_counter(1, NULL);
89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
[...]
Quoted 25 lines
Quoted 12 lines
kernel/bpf/verifier.c static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 20518
Quoted 10 lines
kernel/bpf/verifier.c +6
-0@@ -20488,6 +20510,12 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, node_offset_reg, insn, insn_buf, cnt); }else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { +if (!verifier_inlines_kfunc_call (env, imm)) { +verbose (env,"verifier internal error: kfunc id %d is not defined in checker\n" , + desc->func_id); +return -EFAULT ; + } +Nit: still think that moving this check as the first conditional would
Quoted 12 lines
have been better: if (verifier_inlines_kfunc_call(env, imm)) { if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { // ... } else { // report error } } else if (...) { // ... rest of the cases }
Quoted 1 line
I can change it in the next iteration (if it's needed) if you insist
No need to change if there would be no next iteration.
[...]
Eduard Zingerman
maintainerQuoted 61 lines · Eduard Zingerman
On Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 14 lines
Quoted 14 lines
arch/x86/net/bpf_jit_comp.c +14
-0+if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE (); +EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 ));Quoted 8 lines
arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 2 lines
arch/x86/net/bpf_jit_comp.c +2
-0+/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.I see the following disassembly:
0000000000008980 <
bpf_get_cpu_cycles>:; { 8980: f3 0f 1e fa endbr64 8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9> 0000000000008985: R_X86_64_PLT32 __fentry__-0x4 ; asm volatile(ALTERNATIVE_2("rdtsc", 8989: 0f 31 rdtsc 898b: 90 nop 898c: 90 nop 898d: 90 nop ; return EAX_EDX_VAL(val, low, high); 898e: 48 c1 e2 20 shlq $0x20, %rdx 8992: 48 09 d0 orq %rdx, %rax 8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF ; return (u64)rdtsc_ordered() & S64_MAX; 899f: 48 21 c8 andq %rcx, %rax ; return __arch_get_hw_counter(1, NULL); 89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>
Quoted 3 lines
Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
Talked with Vadim off-list, he explained that 'rttsc nop nop nop' is indeed patched at kernel load. Regarding S64_MAX patching we just hope this should never be an issue for BPF use-case.
So, no more questions from my side.
[...]
Quoted 63 lines · Eduard Zingerman
Quoted 61 lines · Eduard Zingerman
On Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 17 lines
Quoted 6 lines
arch/x86/net/bpf_jit_comp.c +6
-0+if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE ();arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +10
-0+EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 )); +/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );Quoted 5 lines
Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.I see the following disassembly:
0000000000008980 <
bpf_get_cpu_cycles>:; { 8980: f3 0f 1e fa endbr64 8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9> 0000000000008985: R_X86_64_PLT32 __fentry__-0x4 ; asm volatile(ALTERNATIVE_2("rdtsc", 8989: 0f 31 rdtsc 898b: 90 nop 898c: 90 nop 898d: 90 nop ; return EAX_EDX_VAL(val, low, high); 898e: 48 c1 e2 20 shlq $0x20, %rdx 8992: 48 09 d0 orq %rdx, %rax 8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF ; return (u64)rdtsc_ordered() & S64_MAX; 899f: 48 21 c8 andq %rcx, %rax ; return __arch_get_hw_counter(1, NULL); 89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>
Quoted 8 lines
Quoted 3 lines
Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
Talked with Vadim off-list, he explained that 'rttsc nop nop nop' is indeed patched at kernel load. Regarding
S64_MAXpatching we just hope this should never be an issue for BPF use-case.
So, no more questions from my side.
since s64 question came up twice it should be a comment.
nop nop as well.
pw-bot: cr
Quoted 68 lines · Alexei Starovoitov
Quoted 67 lines · Eduard Zingerman
Quoted 64 lines · Eduard Zingerman
On Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 17 lines
Quoted 6 lines
arch/x86/net/bpf_jit_comp.c +6
-0+if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE ();arch/x86/net/bpf_jit_comp.c st: if (is_imm8(insn->off)) · line 2144
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +10
-0+EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 )); +/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG );Quoted 5 lines
Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.I see the following disassembly:
0000000000008980 <
bpf_get_cpu_cycles>:; { 8980: f3 0f 1e fa endbr64 8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9> 0000000000008985: R_X86_64_PLT32 __fentry__-0x4 ; asm volatile(ALTERNATIVE_2("rdtsc", 8989: 0f 31 rdtsc 898b: 90 nop 898c: 90 nop 898d: 90 nop ; return EAX_EDX_VAL(val, low, high); 898e: 48 c1 e2 20 shlq $0x20, %rdx 8992: 48 09 d0 orq %rdx, %rax 8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF ; return (u64)rdtsc_ordered() & S64_MAX; 899f: 48 21 c8 andq %rcx, %rax ; return __arch_get_hw_counter(1, NULL); 89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
Quoted 6 lines
Quoted 4 lines
Talked with Vadim off-list, he explained that 'rttsc nop nop nop' is indeed patched at kernel load. Regarding
S64_MAXpatching we just hope this should never be an issue for BPF use-case.
So, no more questions from my side.since s64 question came up twice it should be a comment.
sure, will do it.
Quoted 2 lines
nop nop as well.
do you mean why there are nop;nop instructions in the kernel's assembly?
Quoted 2 lines
pw-bot: cr
Quoted 75 lines · Vadim Fedorenko
Quoted 73 lines · Alexei Starovoitov
Quoted 71 lines · Eduard Zingerman
Quoted 61 lines · Eduard Zingerman
On Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 17 lines
Quoted 6 lines
(quoted diff) +6
-0+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX to return u64 */ + emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT_LFENCE();Review
Quoted 10 lines
(quoted diff) +10
-0+ EMIT2(0x0F, 0x31); + + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG);Quoted 5 lines
Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.I see the following disassembly:
0000000000008980 <
bpf_get_cpu_cycles>:; { 8980: f3 0f 1e fa endbr64 8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9> 0000000000008985: R_X86_64_PLT32 __fentry__-0x4 ; asm volatile(ALTERNATIVE_2("rdtsc", 8989: 0f 31 rdtsc 898b: 90 nop 898c: 90 nop 898d: 90 nop ; return EAX_EDX_VAL(val, low, high); 898e: 48 c1 e2 20 shlq $0x20, %rdx 8992: 48 09 d0 orq %rdx, %rax 8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF ; return (u64)rdtsc_ordered() & S64_MAX; 899f: 48 21 c8 andq %rcx, %rax ; return __arch_get_hw_counter(1, NULL); 89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>Quoted 3 lines
Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
Talked with Vadim off-list, he explained that 'rttsc nop nop nop' is indeed patched at kernel load. Regarding
S64_MAXpatching we just hope this should never be an issue for BPF use-case.
So, no more questions from my side.
Quoted 8 lines
Quoted 1 line
since s64 question came up twice it should be a comment.
sure, will do it.
Quoted 2 lines
nop nop as well.
do you mean why there are nop;nop instructions in the kernel's assembly?
Explanation on why JITed matches __arch_get_hw_counter.
Quoted 79 lines · Alexei Starovoitov
Quoted 77 lines · Vadim Fedorenko
Quoted 74 lines · Alexei Starovoitov
Quoted 63 lines · Eduard Zingerman
Quoted 61 lines · Eduard Zingerman
On Tue, 2024-11-12 at 21:39 +0000, Vadim Fedorenko wrote:
[...]
Quoted 29 lines
Quoted 17 lines
Quoted 6 lines
(quoted diff) +6
-0+ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX to return u64 */ + emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT_LFENCE();Review
Quoted 10 lines
(quoted diff) +10
-0+ EMIT2(0x0F, 0x31); + + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG);Quoted 5 lines
Note: The default implementation of this kfunc uses
__arch_get_hw_counter(),
which is implemented as(u64)rdtsc_ordered() & S64_MAX. Here we don't do& S64_MAX.
The masking in__arch_get_hw_counter()was added by this commit:
77750f78b0b3("x86/vdso: Fix gettimeofday masking").I think we already discussed it with Alexey in v1, we don't really need any masking here for BPF case. We can use values provided by CPU directly. It will never happen that within one BPF program we will have inlined and non-inlined implementation of this helper, hence the values to compare will be of the same source.
Quoted 6 lines
Quoted 2 lines
Also, the default implementation does not issue
lfence. Not sure if this makes any real-world difference.Well, it actually does.
rdtsc_orderedis translated intolfence; rdtscorrdtscp(which is rdtsc + lfence + u32 cookie) depending on the cpu features.I see the following disassembly:
0000000000008980 <
bpf_get_cpu_cycles>:; { 8980: f3 0f 1e fa endbr64 8984: e8 00 00 00 00 callq 0x8989 <bpf_get_cpu_cycles+0x9> 0000000000008985: R_X86_64_PLT32 __fentry__-0x4 ; asm volatile(ALTERNATIVE_2("rdtsc", 8989: 0f 31 rdtsc 898b: 90 nop 898c: 90 nop 898d: 90 nop ; return EAX_EDX_VAL(val, low, high); 898e: 48 c1 e2 20 shlq $0x20, %rdx 8992: 48 09 d0 orq %rdx, %rax 8995: 48 b9 ff ff ff ff ff ff ff 7f movabsq $0x7fffffffffffffff, %rcx # imm = 0x7FFFFFFFFFFFFFFF ; return (u64)rdtsc_ordered() & S64_MAX; 899f: 48 21 c8 andq %rcx, %rax ; return __arch_get_hw_counter(1, NULL); 89a2: 2e e9 00 00 00 00 jmp 0x89a8 <bpf_get_cpu_cycles+0x28>Quoted 8 lines
Quoted 3 lines
Is it patched when kernel is loaded to replace nops with lfence? By real-world difference I meant difference between default implementation and inlined assembly.
Talked with Vadim off-list, he explained that 'rttsc nop nop nop' is indeed patched at kernel load. Regarding
S64_MAXpatching we just hope this should never be an issue for BPF use-case.
So, no more questions from my side.since s64 question came up twice it should be a comment.
Quoted 8 lines
Quoted 6 lines
sure, will do it.
Quoted 2 lines
nop nop as well.
do you mean why there are nop;nop instructions in the kernel's assembly?
Explanation on why JITed matches
__arch_get_hw_counter.
Got it, will do
Yonghong Song
reviewerQuoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii)
v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures--- arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++ arch/x86/net/bpf_jit_comp32.c | 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 + kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++- 7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +28
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off))case BPF_JMP |BPF_CALL : {u8 *ip = image + addrs[i -1 ]; +if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && + imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +/* Save RDX because RDTSC will use EDX:EAX to return u64 */ +emit_mov_reg (&prog, true,AUX_REG ,BPF_REG_3 ); +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT_LFENCE (); +EMIT2 (0x0F ,0x31 ); + +/* shl RDX, 32 */ +maybe_emit_1mod (&prog,BPF_REG_3 , true); +EMIT3 (0xC1 ,add_1reg (0xE0 ,BPF_REG_3 ),32 ); +/* or RAX, RDX */ +maybe_emit_mod (&prog,BPF_REG_0 ,BPF_REG_3 , true); +EMIT2 (0x09 ,add_2reg (0xC0 ,BPF_REG_0 ,BPF_REG_3 )); +/* restore RDX from R11 */ +emit_mov_reg (&prog, true,BPF_REG_3 ,AUX_REG ); + +break ; + } + func = (u8 *) __bpf_call_base + imm32;if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR (bpf_prog->aux->stack_depth);@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) {return 0 ; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ +if (imm ==BPF_CALL_IMM (bpf_get_cpu_cycles)) +return true; +return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL ) {int err; +if (imm32 ==BPF_CALL_IMM (bpf_get_cpu_cycles)) { +if (boot_cpu_has (X86_FEATURE_LFENCE_RDTSC )) +EMIT3 (0x0F ,0xAE ,0xE8 ); +EMIT2 (0x0F ,0x31 ); +break ; + } + err =emit_kfunc_call (bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) {return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ +if (imm ==BPF_CALL_IMM (bpf_get_cpu_cycles)) +return true; +return false; +}
[...]
Quoted 15 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@#include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 7 lines
kernel/bpf/helpers.c +4
-0@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userreturn ret +1 ; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ +return __arch_get_hw_counter (1 ,NULL );
Some comment to explain what '1' mean in the above?
Quoted 16 lines
kernel/bpf/helpers.c +6
-0+} +#endif +__bpf_kfunc_end_defs ();BTF_KFUNCS_START (generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END (common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {
[...]
Quoted 132 lines · Yonghong Song
Quoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii) v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
---
arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++arch/x86/net/bpf_jit_comp32.c| 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 +
kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++-7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +3
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_CALL: { u8 *ip = image + addrs[i - 1]; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX to returnu64 */
+emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) +EMIT_LFENCE();
+EMIT2(0x0F, 0x31);arch/x86/net/bpf_jit_comp.c +13
-0+ + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG); + + break; + } +func = (u8 *)
__bpf_call_base+ imm32;
if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);arch/x86/net/bpf_jit_comp.c +8
-0@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) { return 0; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { int err; + if (imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT3(0x0F, 0xAE, 0xE8); + EMIT2(0x0F, 0x31); + break; + } + err = emit_kfunc_call(bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}[...]
Quoted 14 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@ #include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user
Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 6 lines
kernel/bpf/helpers.c +4
-0return ret + 1; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ + return __arch_get_hw_counter(1, NULL);Some comment to explain what '1' mean in the above?
That's arch-specific value which translates to HW implemented counter on all architectures which have vDSO gettimeofday() implementation.
For x86 it translates to VDSO_CLOCKMODE_TSC, while for aarch64/RISC-V it's VDSO_CLOCKMODE_ARCHTIMER. Actually, for RISC-V the value of the first parameter doesn't matter at all, for aarch64 it should be 0. The only arch which is more strict about this parameter is x86, but it has it's own special name...
Quoted 16 lines
Quoted 15 lines
kernel/bpf/helpers.c +6
-0+} +#endif + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {[...]
Yonghong Song
reviewerQuoted 142 lines · Vadim Fedorenko
Quoted 132 lines · Yonghong Song
Quoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii) v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
---
arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++arch/x86/net/bpf_jit_comp32.c| 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 +
kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++-7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +3
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_CALL: { u8 *ip = image + addrs[i - 1]; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX toreturn u64 */
+emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) +EMIT_LFENCE();
+EMIT2(0x0F, 0x31);arch/x86/net/bpf_jit_comp.c +13
-0+ + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG); + + break; + } +func = (u8 *)
__bpf_call_base+ imm32; if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);arch/x86/net/bpf_jit_comp.c +8
-0@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) { return 0; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { int err; + if (imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT3(0x0F, 0xAE, 0xE8); + EMIT2(0x0F, 0x31); + break; + } + err = emit_kfunc_call(bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}[...]
Quoted 14 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@ #include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userQuoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 6 lines
kernel/bpf/helpers.c +4
-0return ret + 1; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ + return __arch_get_hw_counter(1, NULL);Some comment to explain what '1' mean in the above?
Quoted 8 lines
That's arch-specific value which translates to HW implemented counter on all architectures which have vDSO
gettimeofday()implementation.For x86 it translates to
VDSO_CLOCKMODE_TSC, while for aarch64/RISC-V it'sVDSO_CLOCKMODE_ARCHTIMER. Actually, for RISC-V the value of the first parameter doesn't matter at all, for aarch64 it should be 0. The only arch which is more strict about this parameter is x86, but it has it's own special name...
So in the future, if we want add aarch64 support or other architecture, the argument could be different, right?
I think we should avoid to have arch specific control in helpers.c. How about we define a __weak func like bpf_arch_get_hw_counter() so we have
__bpf_kfunc u64 bpf_get_cpu_cycles(void)
{
return bpf_arch_get_hw_counter();
}Each arch can implement their own bpf_arch_get_hw_counter().
Do you think this will make more sense? This should not impact jit inlining of this kfunc.
Quoted 18 lines
Quoted 17 lines
Quoted 16 lines
kernel/bpf/helpers.c +6
-0+} +#endif + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {[...]
Quoted 149 lines · Yonghong Song
Quoted 145 lines · Vadim Fedorenko
Quoted 132 lines · Yonghong Song
Quoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii) v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
---
arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++arch/x86/net/bpf_jit_comp32.c| 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 +
kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++-7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +3
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_CALL: { u8 *ip = image + addrs[i - 1]; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX toreturn u64 */
+emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) +EMIT_LFENCE();
+EMIT2(0x0F, 0x31);arch/x86/net/bpf_jit_comp.c +13
-0+ + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG); + + break; + } +func = (u8 *)
__bpf_call_base+ imm32; if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);arch/x86/net/bpf_jit_comp.c +8
-0@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) { return 0; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { int err; + if (imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT3(0x0F, 0xAE, 0xE8); + EMIT2(0x0F, 0x31); + break; + } + err = emit_kfunc_call(bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}[...]
Quoted 14 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@ #include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userQuoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 6 lines
kernel/bpf/helpers.c +4
-0return ret + 1; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ + return __arch_get_hw_counter(1, NULL);Some comment to explain what '1' mean in the above?
That's arch-specific value which translates to HW implemented counter on all architectures which have vDSO
gettimeofday()implementation.
Quoted 8 lines
Quoted 5 lines
For x86 it translates to
VDSO_CLOCKMODE_TSC, while for aarch64/RISC-V it'sVDSO_CLOCKMODE_ARCHTIMER. Actually, for RISC-V the value of the first parameter doesn't matter at all, for aarch64 it should be 0. The only arch which is more strict about this parameter is x86, but it has it's own special name...So in the future, if we want add aarch64 support or other architecture, the argument could be different, right?
No, that's the point. This value will be the same for all architectures. I'll do the implementation for aarch64 once this series is in.
Quoted 33 lines
I think we should avoid to have arch specific control in
helpers.c. How about we define a__weakfunc likebpf_arch_get_hw_counter()so we have
__bpf_kfuncu64bpf_get_cpu_cycles(void)
{
returnbpf_arch_get_hw_counter();
}Each arch can implement their own
bpf_arch_get_hw_counter().
Do you think this will make more sense? This should not impact jit inlining of this kfunc.Quoted 18 lines
Quoted 17 lines
Quoted 16 lines
kernel/bpf/helpers.c +6
-0+} +#endif + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {[...]
Yonghong Song
reviewerQuoted 156 lines · Vadim Fedorenko
Quoted 155 lines · Yonghong Song
Quoted 150 lines · Vadim Fedorenko
Quoted 132 lines · Yonghong Song
Quoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii) v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
---
arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++arch/x86/net/bpf_jit_comp32.c| 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 +
kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++-7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +3
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_CALL: { u8 *ip = image + addrs[i - 1]; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX toreturn u64 */
+emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) +EMIT_LFENCE();
+EMIT2(0x0F, 0x31);arch/x86/net/bpf_jit_comp.c +13
-0+ + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG); + + break; + } +func = (u8 *)
__bpf_call_base+ imm32; if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);arch/x86/net/bpf_jit_comp.c +8
-0@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) { return 0; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { int err; + if (imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT3(0x0F, 0xAE, 0xE8); + EMIT2(0x0F, 0x31); + break; + } + err = emit_kfunc_call(bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}[...]
Quoted 14 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@ #include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userQuoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 6 lines
kernel/bpf/helpers.c +4
-0return ret + 1; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ + return __arch_get_hw_counter(1, NULL);Some comment to explain what '1' mean in the above?
That's arch-specific value which translates to HW implemented counter on all architectures which have vDSO
gettimeofday()implementation.For x86 it translates to
VDSO_CLOCKMODE_TSC, while for aarch64/RISC-V it'sVDSO_CLOCKMODE_ARCHTIMER. Actually, for RISC-V the value of the first parameter doesn't matter at all, for aarch64 it should be 0. The only arch which is more strict about this parameter is x86, but it has it's own special name...
Quoted 5 lines
Quoted 2 lines
So in the future, if we want add aarch64 support or other architecture, the argument could be different, right?
No, that's the point. This value will be the same for all architectures. I'll do the implementation for aarch64 once this series is in.
I did a little bit research and the following are two callsites for __arch_get_hw_counter:
0 lib/vdso/gettimeofday.c do_hres_timens 96 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
1 lib/vdso/gettimeofday.c do_hres 164 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
Let us pick func do_hres_timens():
vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
vd = __arch_get_timens_vdso_data(vd);
if (clk != CLOCK_MONOTONIC_RAW)
vd = &vd[CS_HRES_COARSE];
else
vd = &vd[CS_RAW];
..
cycles = __arch_get_hw_counter(vd->clock_mode, vd);So 'vd' is supplied by arch specific func (__arch_get_timens_vdso_data ()), so theoretically vd->clock_mode in __arch_get_hw_counter(vd->clock_mode, vd) could be different for different archs. The other case in do_hres() is similar.
But if you are sure that the first argument is the same for all architectures, please add a comment right above __arch_get_hw_counter() to say
All architectures have the same parameters as below
Quoted 35 lines
Quoted 34 lines
I think we should avoid to have arch specific control in
helpers.c. How about we define a__weakfunc likebpf_arch_get_hw_counter()so we have
__bpf_kfuncu64bpf_get_cpu_cycles(void)
{
returnbpf_arch_get_hw_counter();
}Each arch can implement their own
bpf_arch_get_hw_counter().
Do you think this will make more sense? This should not impact jit inlining of this kfunc.Quoted 18 lines
Quoted 17 lines
Quoted 16 lines
kernel/bpf/helpers.c +6
-0+} +#endif + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {[...]
Quoted 182 lines · Yonghong Song
Quoted 156 lines · Vadim Fedorenko
Quoted 155 lines · Yonghong Song
Quoted 150 lines · Vadim Fedorenko
Quoted 132 lines · Yonghong Song
Quoted 111 lines · Vadim Fedorenko
New kfunc to return ARCH-specific timecounter. For x86 BPF JIT converts it into rdtsc ordered call. Other architectures will get JIT implementation too if supported. The fallback is to
__arch_get_hw_counter().Signed-off-by Vadim Fedorenko <[email protected]>---
v4 -> v5:
* use if instead of ifdef withIS_ENABLED
v3 -> v4:
* change name of the helper tobpf_get_cpu_cycles(Andrii)
* Hide the helper behindCONFIG_GENERIC_GETTIMEOFDAYto avoid exposing it on architectures which do not have vDSO functions and data
* reduce the scope of check of inlined functions in verifier to only 2, which are actually inlined.
v2 -> v3:
* change name of the helper tobpf_get_cpu_cycles_counterto explicitly mention what counter it provides (Andrii)
* move kfunc definition tobpf.hto use it in JIT.
* introduce another kfunc to convert cycles into nanoseconds as more meaningful time units for generic tracing use case (Andrii) v1 -> v2:
* Fix incorrect function return value type to u64
* Introducebpf_jit_inlines_kfunc_call()and use it inmark_fastcall_pattern_for_call()to avoid clobbering in case of running programs with no JIT (Eduard)
* Avoid rewriting instruction and check function pointer directly in JIT (Alexei)
* Change includes to fix compile issues on non x86 architectures
---
arch/x86/net/bpf_jit_comp.c | 28 ++++++++++++++++++++++++++++arch/x86/net/bpf_jit_comp32.c| 14 ++++++++++++++ include/linux/bpf.h | 5 +++++ include/linux/filter.h | 1 +
kernel/bpf/core.c | 11 +++++++++++ kernel/bpf/helpers.c | 13 +++++++++++++ kernel/bpf/verifier.c | 30 +++++++++++++++++++++++++++++-7 files changed, 101 insertions(+), 1 deletion(-)arch/x86/net/bpf_jit_comp.c +3
-0@@ -2126,6 +2126,26 @@ st: if (is_imm8(insn->off)) case BPF_JMP | BPF_CALL: { u8 *ip = image + addrs[i - 1]; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + /* Save RDX because RDTSC will use EDX:EAX toreturn u64 */
+emit_mov_reg(&prog, true, AUX_REG, BPF_REG_3); + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) +EMIT_LFENCE();
+EMIT2(0x0F, 0x31);arch/x86/net/bpf_jit_comp.c +13
-0+ + /* shl RDX, 32 */ + maybe_emit_1mod(&prog, BPF_REG_3, true); + EMIT3(0xC1, add_1reg(0xE0, BPF_REG_3), 32); + /* or RAX, RDX */ + maybe_emit_mod(&prog, BPF_REG_0, BPF_REG_3, true); + EMIT2(0x09, add_2reg(0xC0, BPF_REG_0, BPF_REG_3)); + /* restore RDX from R11 */ + emit_mov_reg(&prog, true, BPF_REG_3, AUX_REG); + + break; + } +func = (u8 *)
__bpf_call_base+ imm32; if (tail_call_reachable) {LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);arch/x86/net/bpf_jit_comp.c +8
-0@@ -3652,3 +3672,11 @@ u64 bpf_arch_uaddress_limit(void) { return 0; } + +/* x86-64 JIT can inline kfunc */ +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}arch/x86/net/bpf_jit_comp32.c +14
-0@@ -2094,6 +2094,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { int err; + if (imm32 == BPF_CALL_IMM(bpf_get_cpu_cycles)) { + if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) + EMIT3(0x0F, 0xAE, 0xE8); + EMIT2(0x0F, 0x31); + break; + } + err = emit_kfunc_call(bpf_prog, image + addrs[i], insn, &prog);@@ -2621,3 +2628,10 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_inlines_kfunc_call (s32 imm) +{ + if (imm == BPF_CALL_IMM(bpf_get_cpu_cycles)) + return true; + return false; +}[...]
Quoted 14 lines
kernel/bpf/helpers.c +3
-0@@ -23,6 +23,9 @@ #include <linux/btf_ids.h> #include <linux/bpf_mem_alloc.h> #include <linux/kasan.h> +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +#include <vdso/datapage.h> +#endif #include "../../lib/kstrtox.h"@@ -3023,6 +3026,13 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __userQuoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · line 3032
Quoted 6 lines
kernel/bpf/helpers.c +4
-0return ret + 1; } +#if IS_ENABLED (CONFIG_GENERIC_GETTIMEOFDAY ) +__bpf_kfunc u64 bpf_get_cpu_cycles(void) +{ + return __arch_get_hw_counter(1, NULL);Some comment to explain what '1' mean in the above?
That's arch-specific value which translates to HW implemented counter on all architectures which have vDSO
gettimeofday()implementation.For x86 it translates to
VDSO_CLOCKMODE_TSC, while for aarch64/RISC-V it'sVDSO_CLOCKMODE_ARCHTIMER. Actually, for RISC-V the value of the first parameter doesn't matter at all, for aarch64 it should be 0. The only arch which is more strict about this parameter is x86, but it has it's own special name...Quoted 5 lines
Quoted 2 lines
So in the future, if we want add aarch64 support or other architecture, the argument could be different, right?
No, that's the point. This value will be the same for all architectures. I'll do the implementation for aarch64 once this series is in.
I did a little bit research and the following are two callsites for
__arch_get_hw_counter:
0lib/vdso/gettimeofday.cdo_hres_timens 96 cycles =__arch_get_hw_counter(vd->clock_mode, vd); 1lib/vdso/gettimeofday.cdo_hres 164 cycles =__arch_get_hw_counter(vd->clock_mode, vd);Let us pick func
do_hres_timens():vd = vdns - (clk ==
CLOCK_MONOTONIC_RAW?CS_RAW:CS_HRES_COARSE); vd =__arch_get_timens_vdso_data(vd);
if (clk !=CLOCK_MONOTONIC_RAW)
vd = &vd[CS_HRES_COARSE];
else
vd = &vd[CS_RAW];
..
cycles =__arch_get_hw_counter(vd->clock_mode, vd);
Quoted 4 lines
So 'vd' is supplied by arch specific func (
__arch_get_timens_vdso_data()), so theoreticallyvd->clock_modein __arch_get_hw_counter(vd-Quoted 1 line
clock_mode, vd)could be different for different archs. The other case in
do_hres()is similar.
Well, ok, even though I'm pretty sure all the implememtations will end up with 1 as a value, let's be on the safe side and implement it the same way with reading vdso data structure. Now with cycles2ns kfunc we include all the needed pieces, should be no blockers on this implementations. Thanks for bringing it up.
Quoted 41 lines
But if you are sure that the first argument is the same for all architectures, please add a comment right above
__arch_get_hw_counter()to say
All architectures have the same parameters as belowQuoted 35 lines
Quoted 34 lines
I think we should avoid to have arch specific control in
helpers.c. How about we define a__weakfunc likebpf_arch_get_hw_counter()so we have
__bpf_kfuncu64bpf_get_cpu_cycles(void)
{
returnbpf_arch_get_hw_counter();
}Each arch can implement their own
bpf_arch_get_hw_counter().
Do you think this will make more sense? This should not impact jit inlining of this kfunc.Quoted 18 lines
Quoted 17 lines
Quoted 16 lines
kernel/bpf/helpers.c +6
-0+} +#endif + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(generic_btf_ids)@@ -3115,6 +3125,9 @@ BTF_ID_FLAGS(func, bpf_get_kmem_cache) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) +#if IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) +BTF_ID_FLAGS(func, bpf_get_cpu_cycles, KF_FASTCALL) +#endif BTF_KFUNCS_END(common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {[...]