bpf: Fix dispatcher patchable function entry to 5 bytes nop
Subsystems (4)
Jiri Olsa
reviewerPATCH 1/1 bpf: Fix dispatcher patchable function entry to 5 bytes nop
The patchable_function_entry(5) might output 5 single nop instructions (depends on toolchain), which will clash with bpf_arch_text_poke check for 5 bytes nop instruction.
Adding early init call for dispatcher that checks and change the patchable entry into expected 5 nop instruction if needed.
There's no need to take text_mutex, because we are using it in early init call which is called at pre-smp time.
--- arch/x86/net/bpf_jit_comp.c | 13 +++++++++++++ include/linux/bpf.h | 14 +++++++++++++- kernel/bpf/dispatcher.c | 6 ++++++ 3 files changed, 32 insertions(+), 1 deletion(-)
arch/x86/net/bpf_jit_comp.c +13 -0
@@ -11,6 +11,7 @@#include <linux/bpf.h> #include <linux/memory.h> #include <linux/sort.h> +#include <linux/init.h> #include <asm/extable.h> #include <asm/set_memory.h> #include <asm/nospec-branch.h>
@@ -388,6 +389,18 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,return ret; } +int __init bpf_arch_init_dispatcher_early (void *ip) +{ +const u8 *nop_insn = x86_nops[5 ]; + +if (is_endbr (*(u32 *)ip)) + ip +=ENDBR_INSN_SIZE ; + +if (memcmp (ip, nop_insn,X86_PATCH_SIZE )) +text_poke_early (ip, nop_insn,X86_PATCH_SIZE ); +return 0 ; +} +int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t,void *old_addr,void *new_addr) {
include/linux/bpf.h +13 -1
@@ -27,6 +27,7 @@#include <linux/bpfptr.h> #include <linux/btf.h> #include <linux/rcupdate_trace.h> +#include <linux/init.h> struct bpf_verifier_env ;struct bpf_verifier_log ;
@@ -970,6 +971,8 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info);void bpf_trampoline_put (struct bpf_trampoline *tr);int arch_prepare_bpf_dispatcher (void *image,void *buf,s64 *funcs,int num_funcs); +int __init bpf_arch_init_dispatcher_early (void *ip); +#define BPF_DISPATCHER_INIT (_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \
@@ -983,6 +986,13 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func }, \ } +#define BPF_DISPATCHER_INIT_CALL (_name) \ + static int __init _name##_init(void) \ + { \ + return bpf_arch_init_dispatcher_early(_name##_func); \ + } \ + early_initcall(_name##_init) +#ifdef CONFIG_X86_64 #define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5)))#else
@@ -1000,7 +1010,9 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func } \ EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ struct bpf_dispatcher bpf_dispatcher_##name = \ - BPF_DISPATCHER_INIT(bpf_dispatcher_##name); + BPF_DISPATCHER_INIT(bpf_dispatcher_##name); \ + BPF_DISPATCHER_INIT_CALL(bpf_dispatcher_##name); +#define DECLARE_BPF_DISPATCHER (name) \ unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \
kernel/bpf/dispatcher.c +6 -0
@@ -4,6 +4,7 @@#include <linux/hash.h> #include <linux/bpf.h> #include <linux/filter.h> +#include <linux/init.h> /* The BPF dispatcher is a multiway branch code generator. The * dispatcher is a mechanism to avoid the performance penalty of an
@@ -90,6 +91,11 @@ int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int nreturn -ENOTSUPP ; } +int __weak __init bpf_arch_init_dispatcher_early (void *ip) +{ +return -ENOTSUPP ; +} +static int bpf_dispatcher_prepare (struct bpf_dispatcher *d,void *image,void *buf) {s64 ips[BPF_DISPATCHER_MAX ] = {}, *ipsp = &ips[0 ];
Signature · 1 line
2.37.3
Quoted 4 lines · Jiri Olsa
The
patchable_function_entry(5)might output 5 single nop instructions (depends on toolchain), which will clash withbpf_arch_text_pokecheck for 5 bytes nop instruction.
Quoted 8 lines
Adding early init call for dispatcher that checks and change the patchable entry into expected 5 nop instruction if needed.
There's no need to take
text_mutex, because we are using it in early init call which is called at pre-smp time.Fixes ceea991a019c ("bpf: Move bpf_dispatcher function out of ftrace locations")Signed-off-by Jiri Olsa <[email protected]>
This patch was applied to bpf/bpf.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This patch was applied to bpf/bpf.git (master)
by Alexei Starovoitov <[email protected]>:
Quoted 8 lines · On Tue, 18 Oct 2022 09:59:34 +0200 you wrote:
The
patchable_function_entry(5)might output 5 single nop instructions (depends on toolchain), which will clash withbpf_arch_text_pokecheck for 5 bytes nop instruction.Adding early init call for dispatcher that checks and change the patchable entry into expected 5 nop instruction if needed.
[...]
Here is the summary with links:
- [bpf] bpf: Fix dispatcher patchable function entry to 5 bytes nop
https://git.kernel.org/bpf/bpf/c/dbe69b299884
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html