bpf: Fix prog_array_map_poke_run map poke update
Jiri Olsa
reviewerv2 cover bpf: Fix prog_array_map_poke_run map poke update
hi,
this patchset fixes the issue reported in [0].
For the actual fix in patch 2 I'm changing bpf_arch_text_poke to allow to skip ip address check in patch 1. I considered adding separate function for that, but because each arch implementation is bit different, adding extra arg seemed like better option.
v2 changes:
- make it work for other archs
thanks,
jirka
[0] https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument to bpf_arch_text_poke bpf, x64: Fix prog_array_map_poke_run map poke update
arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++-------------------- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 39 insertions(+), 43 deletions(-)
Jiri Olsa
reviewerPATCH v2 1/2 bpf: Add checkip argument to bpf_arch_text_poke
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)
arch/arm64/net/bpf_jit_comp.c +2 -1
@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) +void *old_addr,void *new_addr, +bool checkip__maybe_unused ) {int ret;u32 old_insn;
arch/riscv/net/bpf_jit_comp64.c +3 -2
@@ -667,13 +667,14 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) }int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) +void *old_addr,void *new_addr,bool checkip) {u32 old_insns[RV_FENTRY_NINSNS ], new_insns[RV_FENTRY_NINSNS ];bool is_call = poke_type ==BPF_MOD_CALL ;int ret; -if (!is_kernel_text ((unsigned long )ip) && +if (checkip && + !is_kernel_text ((unsigned long )ip) && !is_bpf_text_address ((unsigned long )ip))return -ENOTSUPP ;
arch/s390/net/bpf_jit_comp.c +2 -1
@@ -2146,7 +2146,8 @@ bool bpf_jit_supports_far_kfunc_call(void) }int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t, -void *old_addr,void *new_addr) +void *old_addr,void *new_addr, +bool checkip__maybe_unused ) {struct {u16 opc;
arch/x86/net/bpf_jit_comp.c +13 -11
@@ -435,19 +435,21 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, }int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t, -void *old_addr,void *new_addr) +void *old_addr,void *new_addr,bool checkip) { -if (!is_kernel_text ((long )ip) && - !is_bpf_text_address ((long )ip)) -/* BPF poking in modules is not supported */ -return -EINVAL ; +if (checkip) { +if (!is_kernel_text ((long )ip) && + !is_bpf_text_address ((long )ip)) +/* BPF poking in modules is not supported */ +return -EINVAL ; -/* -* See emit_prologue(), for IBT builds the trampoline hook is preceded -* with an ENDBR instruction. -*/ -if (is_endbr (*(u32 *)ip)) - ip +=ENDBR_INSN_SIZE ; +/* +* See emit_prologue(), for IBT builds the trampoline hook is preceded +* with an ENDBR instruction. +*/ +if (is_endbr (*(u32 *)ip)) + ip +=ENDBR_INSN_SIZE ; + }return __bpf_arch_text_poke (ip, t, old_addr, new_addr); }
include/linux/bpf.h +1 -1
@@ -3173,7 +3173,7 @@ enum bpf_text_poke_type { };int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t, -void *addr1,void *addr2); +void *addr1,void *addr2,bool checkip);void *bpf_arch_text_copy (void *dst,void *src,size_t len);int bpf_arch_text_invalidate (void *dst,size_t len);
kernel/bpf/arraymap.c +4 -4
@@ -1075,20 +1075,20 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key,if (new) { ret =bpf_arch_text_poke (poke->tailcall_target,BPF_MOD_JUMP , - old_addr, new_addr); + old_addr, new_addr, true);BUG_ON (ret <0 && ret != -EINVAL );if (!old) { ret =bpf_arch_text_poke (poke->tailcall_bypass,BPF_MOD_JUMP , poke->bypass_addr, -NULL ); +NULL , true);BUG_ON (ret <0 && ret != -EINVAL ); } }else { ret =bpf_arch_text_poke (poke->tailcall_bypass,BPF_MOD_JUMP , old_bypass_addr, - poke->bypass_addr); + poke->bypass_addr, true);BUG_ON (ret <0 && ret != -EINVAL );/* let other CPUs finish the execution of program * so that it will not possible to expose them
@@ -1098,7 +1098,7 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key,synchronize_rcu (); ret =bpf_arch_text_poke (poke->tailcall_target,BPF_MOD_JUMP , - old_addr,NULL ); + old_addr,NULL , true);BUG_ON (ret <0 && ret != -EINVAL ); } }
kernel/bpf/core.c +1 -1
@@ -2903,7 +2903,7 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, }int __weak bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t, -void *addr1,void *addr2) +void *addr1,void *addr2,bool checkip) {return -ENOTSUPP ; }
kernel/bpf/trampoline.c +6 -6
@@ -179,7 +179,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)if (tr->func.ftrace_managed) ret =unregister_ftrace_direct (tr->fops, (long )old_addr, false);else - ret =bpf_arch_text_poke (ip,BPF_MOD_CALL , old_addr,NULL ); + ret =bpf_arch_text_poke (ip,BPF_MOD_CALL , old_addr,NULL , true);return ret; }
@@ -196,7 +196,7 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_adelse ret =modify_ftrace_direct_nolock (tr->fops, (long )new_addr); }else { - ret =bpf_arch_text_poke (ip,BPF_MOD_CALL , old_addr, new_addr); + ret =bpf_arch_text_poke (ip,BPF_MOD_CALL , old_addr, new_addr, true); }return ret; }
@@ -219,7 +219,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)ftrace_set_filter_ip (tr->fops, (unsigned long )ip,0 ,1 ); ret =register_ftrace_direct (tr->fops, (long )new_addr); }else { - ret =bpf_arch_text_poke (ip,BPF_MOD_CALL ,NULL , new_addr); + ret =bpf_arch_text_poke (ip,BPF_MOD_CALL ,NULL , new_addr, true); }return ret;
@@ -331,7 +331,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)*/ if (im->ip_after_call) {int err =bpf_arch_text_poke (im->ip_after_call,BPF_MOD_JUMP , -NULL , im->ip_epilogue); +NULL , im->ip_epilogue, true);WARN_ON (err);if (IS_ENABLED (CONFIG_PREEMPTION ))call_rcu_tasks (&im->rcu, __bpf_tramp_image_put_rcu_tasks);
@@ -533,7 +533,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trreturn -EBUSY ; tr->extension_prog = link->link.prog;return bpf_arch_text_poke (tr->func.addr,BPF_MOD_JUMP ,NULL , - link->link.prog->bpf_func); + link->link.prog->bpf_func, true); }if (cnt >=BPF_MAX_TRAMP_LINKS )return -E2BIG ;
@@ -576,7 +576,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_if (kind ==BPF_TRAMP_REPLACE ) {WARN_ON_ONCE (!tr->extension_prog); err =bpf_arch_text_poke (tr->func.addr,BPF_MOD_JUMP , - tr->extension_prog->bpf_func,NULL ); + tr->extension_prog->bpf_func,NULL , true); tr->extension_prog =NULL ;return err; }
Signature · 1 line
2.43.0
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 25 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) + void *old_addr, void *new_addr,
[..]
arch/arm64/net/bpf_jit_comp.c
Quoted 1 line
arch/arm64/net/bpf_jit_comp.c +1
-0+ bool checkip __maybe_unused)
Any idea why only riscv and x86 do this check?
Asking because maybe it makes sense to move this check into some new generic bpf_text_poke and call it in the places where you currently call checkip=true (and keep using bpf_arch_text_poke for checkip=false case).
(don't see any issues with the current approach btw, just interested..)
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 29 lines · Stanislav Fomichev
Quoted 25 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) + void *old_addr, void *new_addr,[..]
Quoted 3 lines
arch/arm64/net/bpf_jit_comp.c
Quoted 1 line
arch/arm64/net/bpf_jit_comp.c +1
-0+ bool checkip __maybe_unused)Any idea why only riscv and x86 do this check?
so arm does the check as well, but needs the data from the lookup to patch things properly.. but IIUC it does not suffer the same issue because it does not implement direct tail calls [1] which is used only on x86
Quoted 7 lines
Asking because maybe it makes sense to move this check into some new generic
bpf_text_pokeand call it in the places where you currently call checkip=true (and keep usingbpf_arch_text_pokefor checkip=false case).(don't see any issues with the current approach btw, just interested..)
I tried to add new function for that, but it did not look good for arm because it needs to do the lookup anyway
hm maybe we could use new arch function that would cover the single tail call 'text poke' update in prog_array_map_poke_run and would be implemented only on x86 ... using __bpf_arch_text_poke directly
jirka
[1] 428d5df1fa4f bpf, x86: Emit patchable direct jump as tail call
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 45 lines · Jiri Olsa
Quoted 29 lines · Stanislav Fomichev
Quoted 25 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) + void *old_addr, void *new_addr,[..]
Quoted 3 lines
arch/arm64/net/bpf_jit_comp.c
Quoted 1 line
arch/arm64/net/bpf_jit_comp.c +1
-0+ bool checkip __maybe_unused)Any idea why only riscv and x86 do this check?
so arm does the check as well, but needs the data from the lookup to patch things properly.. but IIUC it does not suffer the same issue because it does not implement direct tail calls [1] which is used only on x86
Quoted 6 lines
Asking because maybe it makes sense to move this check into some new generic
bpf_text_pokeand call it in the places where you currently call checkip=true (and keep usingbpf_arch_text_pokefor checkip=false case).
Quoted 8 lines
Quoted 1 line
(don't see any issues with the current approach btw, just interested..)
I tried to add new function for that, but it did not look good for arm because it needs to do the lookup anyway
hm maybe we could use new arch function that would cover the single tail call 'text poke' update in
prog_array_map_poke_runand would be implemented only on x86 ... using__bpf_arch_text_pokedirectly
looks like below change would be enough, I'll test and send new version
jirka
---
arch/x86/net/bpf_jit_comp.c +41 -0
@@ -3025,3 +3025,44 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp#endif WARN (1 ,"verification of programs using bpf_throw should have failed\n" ); } + +void bpf_arch_poke_desc_update (struct bpf_jit_poke_descriptor *poke, +struct bpf_prog *new,struct bpf_prog *old) +{ +u8 *old_addr, *new_addr, *old_bypass_addr; +int ret; + + old_bypass_addr = old ?NULL : poke->bypass_addr; + old_addr = old ? (u8 *)old->bpf_func + poke->adj_off :NULL ; + new_addr = new ? (u8 *)new->bpf_func + poke->adj_off :NULL ; + +if (new) { + ret =__bpf_arch_text_poke (poke->tailcall_target, +BPF_MOD_JUMP , + old_addr, new_addr); +BUG_ON (ret <0 ); +if (!old) { + ret =__bpf_arch_text_poke (poke->tailcall_bypass, +BPF_MOD_JUMP , + poke->bypass_addr, +NULL ); +BUG_ON (ret <0 ); + } + }else { + ret =__bpf_arch_text_poke (poke->tailcall_bypass, +BPF_MOD_JUMP , + old_bypass_addr, + poke->bypass_addr); +BUG_ON (ret <0 ); +/* let other CPUs finish the execution of program +* so that it will not possible to expose them +* to invalid nop, stack unwind, nop state +*/ +if (!ret) +synchronize_rcu (); + ret =__bpf_arch_text_poke (poke->tailcall_target, +BPF_MOD_JUMP , + old_addr,NULL ); +BUG_ON (ret <0 ); + } +}
kernel/bpf/arraymap.c +8 -35
65 lines not on this page · raw message
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 49 lines · Jiri Olsa
Quoted 47 lines · Jiri Olsa
Quoted 29 lines · Stanislav Fomichev
Quoted 25 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) + void *old_addr, void *new_addr,[..]
Quoted 3 lines
Quoted 1 line
+ bool checkip
__maybe_unused)Any idea why only riscv and x86 do this check?
so arm does the check as well, but needs the data from the lookup to patch things properly.. but IIUC it does not suffer the same issue because it does not implement direct tail calls [1] which is used only on x86
Quoted 7 lines
Asking because maybe it makes sense to move this check into some new generic
bpf_text_pokeand call it in the places where you currently call checkip=true (and keep usingbpf_arch_text_pokefor checkip=false case).(don't see any issues with the current approach btw, just interested..)
Quoted 8 lines
Quoted 6 lines
I tried to add new function for that, but it did not look good for arm because it needs to do the lookup anyway
hm maybe we could use new arch function that would cover the single tail call 'text poke' update in
prog_array_map_poke_runand would be implemented only on x86 ... using__bpf_arch_text_pokedirectlylooks like below change would be enough, I'll test and send new version
sg. I'm still not 100% sure why it's x86 only, I was (probably wrongly?) assuming that at least arm64 jit is mostly on par with x86 :-)
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 57 lines · Stanislav Fomichev
Quoted 56 lines · Jiri Olsa
Quoted 53 lines · Jiri Olsa
Quoted 29 lines · Stanislav Fomichev
Quoted 25 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 3 ++- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------ 8 files changed, 32 insertions(+), 27 deletions(-)arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2167,7 +2167,8 @@ static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip,* locations during the patching process, making the patching process easier. */ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, -void *old_addr,void *new_addr) + void *old_addr, void *new_addr,[..]
Quoted 3 lines
Quoted 1 line
+ bool checkip
__maybe_unused)Any idea why only riscv and x86 do this check?
so arm does the check as well, but needs the data from the lookup to patch things properly.. but IIUC it does not suffer the same issue because it does not implement direct tail calls [1] which is used only on x86
Quoted 7 lines
Asking because maybe it makes sense to move this check into some new generic
bpf_text_pokeand call it in the places where you currently call checkip=true (and keep usingbpf_arch_text_pokefor checkip=false case).(don't see any issues with the current approach btw, just interested..)
I tried to add new function for that, but it did not look good for arm because it needs to do the lookup anyway
hm maybe we could use new arch function that would cover the single tail call 'text poke' update in
prog_array_map_poke_runand would be implemented only on x86 ... using__bpf_arch_text_pokedirectly
Quoted 5 lines
Quoted 1 line
looks like below change would be enough, I'll test and send new version
sg. I'm still not 100% sure why it's x86 only, I was (probably wrongly?) assuming that at least arm64 jit is mostly on par with x86 :-)
AFAICS the direct tail calls are on x86, CI also seems to be ok with that change.. I'll send it as formal patch
jirka
Ilya Leoshkevich
maintainerRe: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
In reply to Jiri OlsaQuoted 14 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>---
arch/arm64/net/bpf_jit_comp.c | 3 ++-arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------8 files changed, 32 insertions(+), 27 deletions(-)
[...]
Quoted 26 lines
arch/x86/net/bpf_jit_comp.c +0
-0@@ -435,19 +435,21 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, }
int bpf_arch_text_poke(void *ip, enumbpf_text_poke_typet, - void*old_addr, void*new_addr)
+ void*old_addr, void*new_addr, bool checkip) {
- if (!is_kernel_text((long)ip) &&
- !is_bpf_text_address((long)ip))
- /* BPF poking in modules is not supported */ - return-EINVAL;
+ if (checkip) {
+ if (!is_kernel_text((long)ip) &&
+ !is_bpf_text_address((long)ip))
+ /* BPF poking in modules is not supported */ + return-EINVAL;
- /*
- * Seeemit_prologue(), for IBT builds the trampoline hook is preceded
- * with an ENDBR instruction.
- */
- if (is_endbr(*(u32 *)ip))
Quoted 8 lines
- ip +=
ENDBR_INSN_SIZE;
+ /*
+ * Seeemit_prologue(), for IBT builds the trampoline hook is preceded
+ * with an ENDBR instruction.
+ */
+ if (is_endbr(*(u32 *)ip))
+ ip +=ENDBR_INSN_SIZE;
Do we really want to skip the IP adjustment too?
Quoted 4 lines
+ }
return__bpf_arch_text_poke(ip, t, old_addr, new_addr); }
[...]
Re: [PATCHv2 bpf 1/2] bpf: Add checkip argument to bpf_arch_text_poke
Quoted 46 lines · Ilya Leoshkevich
Quoted 14 lines · Jiri Olsa
We need to be able to skip ip address check for caller in following changes. Adding checkip argument to allow that.
Signed-off-by Jiri Olsa <[email protected]>---
arch/arm64/net/bpf_jit_comp.c | 3 ++-arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 8 ++++---- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------8 files changed, 32 insertions(+), 27 deletions(-)[...]
Quoted 28 lines
arch/x86/net/bpf_jit_comp.c +0
-0@@ -435,19 +435,21 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, }
int bpf_arch_text_poke(void *ip, enumbpf_text_poke_typet, - void*old_addr, void*new_addr)
+ void*old_addr, void*new_addr, bool checkip) {
- if (!is_kernel_text((long)ip) &&
- !is_bpf_text_address((long)ip))
- /* BPF poking in modules is not supported */ - return-EINVAL;
+ if (checkip) {
+ if (!is_kernel_text((long)ip) &&
+ !is_bpf_text_address((long)ip))
+ /* BPF poking in modules is not supported */ + return-EINVAL;
- /*
- * Seeemit_prologue(), for IBT builds the trampoline hook is preceded
- * with an ENDBR instruction.
- */
- if (is_endbr(*(u32 *)ip))
- ip +=ENDBR_INSN_SIZE;
+ /*
Quoted 8 lines
Quoted 6 lines
+ * See
emit_prologue(), for IBT builds the trampoline hook is preceded
+ * with an ENDBR instruction.
+ */
+ if (is_endbr(*(u32 *)ip))
+ ip +=ENDBR_INSN_SIZE;Do we really want to skip the IP adjustment too?
the idea was that with __bpf_arch_text_poke you are aware of what you are updating, so there's no need for extra checking
anyway this version got deprecated and I just sent v3 which is bit different without this change
thanks,
jirka
Quoted 7 lines
Quoted 5 lines
+ }
return__bpf_arch_text_poke(ip, t, old_addr, new_addr); }[...]
Jiri Olsa
reviewerPATCH v2 2/2 bpf: Fix prog_array_map_poke_run map poke update
In reply to Jiri OlsaLee pointed out issue found by syscaller [0] hitting BUG in prog array map poke update in prog_array_map_poke_run function due to error value returned from bpf_arch_text_poke function.
There's race window where bpf_arch_text_poke can fail due to missing bpf program kallsym symbols, which is accounted for with check for -EINVAL in that BUG_ON call.
The problem is that in such case we won't update the tail call jump and cause imballance for the next tail call update check which will fail with -EBUSY in bpf_arch_text_poke.
I'm hitting following race during the program load:
CPU 0 CPU 1
bpf_prog_load
bpf_check
do_misc_fixups
prog_array_map_poke_track
map_update_elem
bpf_fd_array_map_update_elem
prog_array_map_poke_run
bpf_arch_text_poke returns -EINVAL
bpf_prog_kallsyms_add
After bpf_arch_text_poke (CPU 1) fails to update the tail call jump, the next poke update fails on expected jump instruction check in bpf_arch_text_poke with -EBUSY and triggers the BUG_ON in prog_array_map_poke_run.
Similar race exists on the program unload.
Fixing this by calling bpf_arch_text_poke with 'checkip=false' to skip the bpf symbol check like we do in bpf_tail_call_direct_fixup. This way the prog_array_map_poke_run does not depend on bpf program having the kallsym symbol in place.
[0] https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
--- kernel/bpf/arraymap.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-)
kernel/bpf/arraymap.c +11 -20
@@ -1044,20 +1044,11 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key,* activated, so tail call updates can arrive from here * while JIT is still finishing its final fixup for * non-activated poke entries. -* 3) On program teardown, the program's kallsym entry gets -* removed out of RCU callback, but we can only untrack -* from sleepable context, therefore bpf_arch_text_poke() -* might not see that this is in BPF text section and -* bails out with -EINVAL. As these are unreachable since -* RCU grace period already passed, we simply skip them. -* 4) Also programs reaching refcount of zero while patching +* 3) Also programs reaching refcount of zero while patching * is in progress is okay since we're protected under * poke_mutex and untrack the programs before the JIT -* buffer is freed. When we're still in the middle of -* patching and suddenly kallsyms entry of the program -* gets evicted, we just skip the rest which is fine due -* to point 3). -* 5) Any other error happening below from bpf_arch_text_poke() +* buffer is freed. +* 4) Any error happening below from bpf_arch_text_poke() * is a unexpected bug. */ if (!READ_ONCE (poke->tailcall_target_stable))
@@ -1075,21 +1066,21 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key,if (new) { ret =bpf_arch_text_poke (poke->tailcall_target,BPF_MOD_JUMP , - old_addr, new_addr, true); -BUG_ON (ret <0 && ret != -EINVAL ); + old_addr, new_addr, false); +BUG_ON (ret <0 );if (!old) { ret =bpf_arch_text_poke (poke->tailcall_bypass,BPF_MOD_JUMP , poke->bypass_addr, -NULL , true); -BUG_ON (ret <0 && ret != -EINVAL ); +NULL , false); +BUG_ON (ret <0 ); } }else { ret =bpf_arch_text_poke (poke->tailcall_bypass,BPF_MOD_JUMP , old_bypass_addr, - poke->bypass_addr, true); -BUG_ON (ret <0 && ret != -EINVAL ); + poke->bypass_addr, false); +BUG_ON (ret <0 );/* let other CPUs finish the execution of program * so that it will not possible to expose them * to invalid nop, stack unwind, nop state
@@ -1098,8 +1089,8 @@ static void prog_array_map_poke_run(struct bpf_map *map, u32 key,synchronize_rcu (); ret =bpf_arch_text_poke (poke->tailcall_target,BPF_MOD_JUMP , - old_addr,NULL , true); -BUG_ON (ret <0 && ret != -EINVAL ); + old_addr,NULL , false); +BUG_ON (ret <0 ); } } }
Signature · 1 line
2.43.0
Ilya Leoshkevich
maintainerQuoted 32 lines · Jiri Olsa
hi,
this patchset fixes the issue reported in [0].For the actual fix in patch 2 I'm changing
bpf_arch_text_poketo allow to skip
ip address check in patch 1. I considered adding separate function for that,
but because each arch implementation is bit different, adding extra arg seemed
like better option.v2 changes:
- make it work for other archsthanks,
jirka[0] https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument tobpf_arch_text_pokebpf, x64: Fixprog_array_map_poke_runmap poke updatearch/arm64/net/bpf_jit_comp.c | 3 ++-
arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++-------------------- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------
Quoted 1 line
8 files changed, 39 insertions(+), 43 deletions(-)
Would it be possible to add a minimized version of the reproducer as a testcase?
Quoted 35 lines · Ilya Leoshkevich
Quoted 32 lines · Jiri Olsa
hi,
this patchset fixes the issue reported in [0].For the actual fix in patch 2 I'm changing
bpf_arch_text_poketo allow to skip
ip address check in patch 1. I considered adding separate function for that,
but because each arch implementation is bit different, adding extra arg seemed
like better option.v2 changes:
- make it work for other archsthanks,
jirka[0] https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument tobpf_arch_text_pokebpf, x64: Fixprog_array_map_poke_runmap poke updatearch/arm64/net/bpf_jit_comp.c | 3 ++-
arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++-------------------- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------Quoted 1 line
8 files changed, 39 insertions(+), 43 deletions(-)
Quoted 2 lines
Would it be possible to add a minimized version of the reproducer as a testcase?
there's reproducer I used in here:
https://syzkaller.appspot.com/text?tag=ReproC&x=1397180f680000
I can try, but not sure I'll be able to come up with something that would fit as testcase.. I'll check
jirka
Quoted 36 lines · Jiri Olsa
Quoted 35 lines · Ilya Leoshkevich
Quoted 33 lines · Jiri Olsa
hi,
this patchset fixes the issue reported in [0].For the actual fix in patch 2 I'm changing
bpf_arch_text_poketo allow to skip
ip address check in patch 1. I considered adding separate function for that,
but because each arch implementation is bit different, adding extra arg seemed
like better option.v2 changes:
- make it work for other archsthanks,
jirka[0] https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument tobpf_arch_text_pokebpf, x64: Fixprog_array_map_poke_runmap poke updatearch/arm64/net/bpf_jit_comp.c | 3 ++-
arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++-------------------- kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------8 files changed, 39 insertions(+), 43 deletions(-)
Quoted 8 lines
Quoted 2 lines
Would it be possible to add a minimized version of the reproducer as a testcase?
there's reproducer I used in here:
https://syzkaller.appspot.com/text?tag=ReproC&x=1397180f680000I can try, but not sure I'll be able to come up with something that would fit as testcase.. I'll check
the test below reproduces it for me.. the only tricky part is that I need to repeat the loop 10 times to trigger that on my setup.. which is not terrible, but not great for a test I think
jirka
---
tools/testing/selftests/bpf/prog_tests/tailcall_poke.c new file +77 -0
@@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <unistd.h> +#include <test_progs.h> +#include "tailcall_poke.skel.h" + +#define JMP_TABLE "/sys/fs/bpf/jmp_table" + +static int thread_exit; + +static void *update (void *arg) +{ +__u32 zero =0 , prog1_fd, prog2_fd, map_fd; +struct tailcall_poke *call = arg; + + map_fd =bpf_map__fd (call->maps.jmp_table); + prog1_fd =bpf_program__fd (call->progs.call1); + prog2_fd =bpf_program__fd (call->progs.call2); + +while (!thread_exit) { +bpf_map_update_elem (map_fd, &zero, &prog1_fd,BPF_ANY ); +bpf_map_update_elem (map_fd, &zero, &prog2_fd,BPF_ANY ); + } + +return NULL ; +} + +void test_tailcall_poke (void ) +{ +struct tailcall_poke *call, *test; +int err, cnt =10 ; +pthread_t thread; + +unlink (JMP_TABLE ); + + call =tailcall_poke__open_and_load (); +if (!ASSERT_OK_PTR (call,"tailcall_poke__open" )) +return ; + + err =bpf_map__pin (call->maps.jmp_table,JMP_TABLE ); +if (!ASSERT_OK (err,"bpf_map__pin" )) + goto out; + + err =pthread_create (&thread,NULL , update, call); +if (!ASSERT_OK (err,"new toggler" )) + goto out; + +while (cnt--) { + test =tailcall_poke__open (); +if (!ASSERT_OK_PTR (test,"tailcall_poke__open" )) +break ; + + err =bpf_map__set_pin_path (test->maps.jmp_table,JMP_TABLE ); +if (!ASSERT_OK (err,"bpf_map__pin" )) { +tailcall_poke__destroy (test); +break ; + } + +bpf_program__set_autoload (test->progs.test, true); +bpf_program__set_autoload (test->progs.call1, false); +bpf_program__set_autoload (test->progs.call2, false); + + err =tailcall_poke__load (test); +if (!ASSERT_OK (err,"tailcall_poke__load" )) { +tailcall_poke__destroy (test); +break ; + } + +tailcall_poke__destroy (test); + } + + thread_exit =1 ; +ASSERT_OK (pthread_join (thread,NULL ),"pthread_join" ); + +out: +bpf_map__unpin (call->maps.jmp_table,JMP_TABLE ); +tailcall_poke__destroy (call); +}
tools/testing/selftests/bpf/progs/tailcall_poke.c new file +32 -0
@@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG (test ,int a) +{ +bpf_tail_call_static (ctx, &jmp_table,0 ); +return 0 ; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG (call1 ,int a) +{ +return 0 ; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG (call2 ,int a) +{ +return 0 ; +}
Quoted 49 lines · Jiri Olsa
Quoted 48 lines · Jiri Olsa
Quoted 40 lines · Ilya Leoshkevich
Quoted 37 lines · Jiri Olsa
hi,
this patchset fixes the issue reported in [0].For the actual fix in patch 2 I'm changing
bpf_arch_text_poketo
allow to skip
ip address check in patch 1. I considered adding separate function
for that,
but because each arch implementation is bit different, adding extra
arg seemed
like better option.v2 changes:
- make it work for other archsthanks,
jirka[0]
https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument tobpf_arch_text_pokebpf, x64: Fixprog_array_map_poke_runmap poke updatearch/arm64/net/bpf_jit_comp.c | 3 ++-
arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++--------------
------
kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------Quoted 1 line
8 files changed, 39 insertions(+), 43 deletions(-)Quoted 3 lines
Would it be possible to add a minimized version of the reproducer as a
testcase?there's reproducer I used in here:
https://syzkaller.appspot.com/text?tag=ReproC&x=1397180f680000
Quoted 8 lines
Quoted 2 lines
I can try, but not sure I'll be able to come up with something that would fit as testcase.. I'll check
the test below reproduces it for me.. the only tricky part is that I need to repeat the loop 10 times to trigger that on my setup.. which is not terrible, but not great for a test I think
jirka
The test looks useful to me. I think having magic repetition counts like this 10 here is almost inevitable when trying to reproduce race conditions. The test also runs quickly for me. You can have my
in case you decide to make a formal patch.
Quoted 59 lines · Ilya Leoshkevich
Quoted 49 lines · Jiri Olsa
Quoted 48 lines · Jiri Olsa
Quoted 40 lines · Ilya Leoshkevich
Quoted 37 lines · Jiri Olsa
hi,
this patchset fixes the issue reported in [0].For the actual fix in patch 2 I'm changing
bpf_arch_text_poketo
allow to skip
ip address check in patch 1. I considered adding separate function
for that,
but because each arch implementation is bit different, adding extra
arg seemed
like better option.v2 changes:
- make it work for other archsthanks,
jirka[0]
https://syzkaller.appspot.com/bug?extid=97a4fe20470e9bc30810
---
Jiri Olsa (2):
bpf: Add checkip argument tobpf_arch_text_pokebpf, x64: Fixprog_array_map_poke_runmap poke updatearch/arm64/net/bpf_jit_comp.c | 3 ++-
arch/riscv/net/bpf_jit_comp64.c| 5 +++-- arch/s390/net/bpf_jit_comp.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 24 +++++++++++++----------- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 31 +++++++++++--------------
------
kernel/bpf/core.c | 2 +- kernel/bpf/trampoline.c | 12 ++++++------Quoted 1 line
8 files changed, 39 insertions(+), 43 deletions(-)Quoted 3 lines
Would it be possible to add a minimized version of the reproducer as a
testcase?there's reproducer I used in here:
https://syzkaller.appspot.com/text?tag=ReproC&x=1397180f680000Quoted 8 lines
Quoted 2 lines
I can try, but not sure I'll be able to come up with something that would fit as testcase.. I'll check
the test below reproduces it for me.. the only tricky part is that I need to repeat the loop 10 times to trigger that on my setup.. which is not terrible, but not great for a test I think
jirka
Quoted 7 lines
The test looks useful to me. I think having magic repetition counts like this 10 here is almost inevitable when trying to reproduce race conditions. The test also runs quickly for me. You can have my
Acked-by Ilya Leoshkevich <[email protected]>in case you decide to make a formal patch.
great, thanks
jirka