bpf: Disable migrate when kprobe_multi attach to access bpf_prog_active
Subsystems (4)
PATCH 1/1 bpf: Disable migrate when kprobe_multi attach to access bpf_prog_active
The syscall link_create not protected by bpf_disable_instrumentation, accessing percpu data bpf_prog_active should use cpu local_lock when kprobe_multi program attach.
--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/trace/bpf_trace.c +2 -2
@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err; +migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active); +migrate_enable ();return err; }
Signature · 1 line
2.48.1
Quoted 15 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)kernel/trace/bpf_trace.c +0
-0
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2732
Quoted 5 lines
kernel/trace/bpf_trace.c +1
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err; +migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {
this is called all the way from graph tracer, which disables preemption in function_graph_enter_regs, so I think we can safely use __this_cpu_inc_return
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2737
Quoted 6 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();
hum, but now I'm not sure why we disable migration in here then
jirka
Quoted 17 lines
kernel/trace/bpf_trace.c +1
-1rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active); +migrate_enable ();return err; }Signature · 1 line
2.48.1
Quoted 26 lines · Jiri Olsa
Quoted 15 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)kernel/trace/bpf_trace.c +0
-0kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2732
Quoted 5 lines
kernel/trace/bpf_trace.c +1
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err; +migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {this is called all the way from graph tracer, which disables preemption in
function_graph_enter_regs, so I think we can safely use__this_cpu_inc_return
Quoted 9 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2737
Quoted 6 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();hum, but now I'm not sure why we disable migration in here then
It seems that there is a cant_migrate() check in bpf_prog_run, so it should be disabled before run.
Quoted 19 lines
jirka
Quoted 17 lines
kernel/trace/bpf_trace.c +1
-1rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active); +migrate_enable ();return err; }Signature · 1 line
2.48.1
Signature · 2 lines
Best Regards Tao Chen
Yonghong Song
reviewerQuoted 19 lines · Jiri Olsa
Quoted 18 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)kernel/trace/bpf_trace.c +0
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err;
Quoted 4 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2732
Quoted 2 lines
kernel/trace/bpf_trace.c +1
-0+migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {this is called all the way from graph tracer, which disables preemption in
function_graph_enter_regs, so I think we can safely use__this_cpu_inc_return
Agree. migrate_disable() is not needed here. But it would be great to add some comments here since for most other prog_run, they typically have migrate_disable/enable.
Quoted 9 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2737
Quoted 8 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();hum, but now I'm not sure why we disable migration in here then
Probably a oversight.
Quoted 20 lines
jirka
Quoted 17 lines
kernel/trace/bpf_trace.c +1
-1rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active); +migrate_enable ();return err; }Signature · 1 line
2.48.1
Quoted 35 lines · Tao Chen
Quoted 25 lines · Jiri Olsa
Quoted 15 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)kernel/trace/bpf_trace.c +0
-0kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2731
Quoted 4 lines
kernel/trace/bpf_trace.c +1
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err; +migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {this is called all the way from graph tracer, which disables preemption in
function_graph_enter_regs, so I think we can safely use__this_cpu_inc_returnQuoted 8 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2735
Quoted 5 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();hum, but now I'm not sure why we disable migration in here then
Quoted 2 lines
It seems that there is a
cant_migrate()check inbpf_prog_run, so it should be disabled before run.
yes, but disabled preemption will take care of that
I think we can do change below plus some comment that Yonghong is suggesting in the other reply
jirka
---
kernel/trace/bpf_trace.c +0 -2
@@ -2734,14 +2734,12 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, goto out; } -migrate_disable ();rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active);
Quoted 36 lines · Jiri Olsa
Quoted 35 lines · Tao Chen
Quoted 33 lines · Jiri Olsa
Quoted 15 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>--- kernel/trace/bpf_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)kernel/trace/bpf_trace.c +0
-0kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2731
Quoted 4 lines
kernel/trace/bpf_trace.c +1
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,struct pt_regs *regs;int err; +migrate_disable ();if (unlikely (__this_cpu_inc_return (bpf_prog_active) !=1 )) {this is called all the way from graph tracer, which disables preemption in
function_graph_enter_regs, so I think we can safely use__this_cpu_inc_returnkernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2737
Quoted 5 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter (link->link.prog); err =1 ; goto out; } -migrate_disable ();hum, but now I'm not sure why we disable migration in here then
Quoted 5 lines
Quoted 2 lines
It seems that there is a
cant_migrate()check inbpf_prog_run, so it should be disabled before run.yes, but disabled preemption will take care of that
I see, you are right, preempt will pass the check, thanks.
void __cant_migrate(const char *file, int line)
{
static unsigned long prev_jiffy;
if (irqs_disabled())
return;
if (is_migration_disabled(current))
return;
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
if (preempt_count() > 0)
return;...
Quoted 3 lines
I think we can do change below plus some comment that Yonghong is suggesting in the other reply
Yes, i will remove the migrate_disable and add some comment as you and Yonghong suggested.
Quoted 23 lines
jirka
---
kernel/trace/bpf_trace.c +0
-2@@ -2734,14 +2734,12 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, goto out; } -migrate_disable ();rcu_read_lock (); regs =ftrace_partial_regs (fregs,bpf_kprobe_multi_pt_regs_ptr ()); old_run_ctx =bpf_set_run_ctx (&run_ctx.session_ctx.run_ctx); err =bpf_prog_run (link->link.prog, regs);bpf_reset_run_ctx (old_run_ctx);rcu_read_unlock (); -migrate_enable (); out:__this_cpu_dec (bpf_prog_active);
Signature · 2 lines
Best Regards Tao Chen
Quoted 32 lines · Yonghong Song
Quoted 11 lines · Jiri Olsa
Quoted 10 lines · Tao Chen
The syscall
link_createnot protected bybpf_disable_instrumentation, accessing percpu databpf_prog_activeshould use cpulocal_lockwhenkprobe_multiprogram attach.Fixes 0dcac2725406 ("bpf: Add multi kprobe link")Signed-off-by Tao Chen <[email protected]>---
kernel/trace/bpf_trace.c| 4 ++--1 file changed, 2 insertions(+), 2 deletions(-)Quoted 12 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct · line 2732
Quoted 10 lines
kernel/trace/bpf_trace.c +1
-0@@ -2728,23 +2728,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, struct pt_regs *regs; int err; + migrate_disable(); if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {this is called all the way from graph tracer, which disables preemption in
function_graph_enter_regs, so I think we can safely use__this_cpu_inc_returnAgree.
migrate_disable()is not needed here. But it would be great to add some comments here since for most otherprog_run, they typically have migrate_disable/enable.Quoted 2 lines
Quoted 28 lines
Quoted 6 lines
kernel/trace/bpf_trace.c kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · line 2737
Quoted 5 lines
kernel/trace/bpf_trace.c +0
-1bpf_prog_inc_misses_counter(link->link.prog); err = 1; goto out; } - migrate_disable();hum, but now I'm not sure why we disable migration in here then
Probably a oversight.
Quoted 18 lines
jirka
Quoted 15 lines
kernel/trace/bpf_trace.c +0
-1rcu_read_lock(); regs = ftrace_partial_regs(fregs, bpf_kprobe_multi_pt_regs_ptr()); old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); err = bpf_prog_run(link->link.prog, regs); bpf_reset_run_ctx(old_run_ctx); rcu_read_unlock(); - migrate_enable();out:
__this_cpu_dec(bpf_prog_active);
+migrate_enable();
return err;
}Signature · 1 line
2.48.1
Hi Jiri, Yonghong,
I send another patch as you suggested, pls review it, thanks.
https://lore.kernel.org/bpf/[email protected]
Signature · 2 lines
Best Regards Tao Chen