bpf: support nested rcu critical sections
Subsystems (4)
PATCH 1/1 bpf: support nested rcu critical sections
Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.
--- include/linux/bpf_verifier.h | 2 +- kernel/bpf/verifier.c | 34 ++++++++-------- .../selftests/bpf/prog_tests/rcu_read_lock.c | 4 +- .../selftests/bpf/progs/rcu_read_lock.c | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 19 deletions(-)
include/linux/bpf_verifier.h +1 -1
@@ -441,7 +441,7 @@ struct bpf_verifier_state {u32 active_irq_id;u32 active_lock_id;void *active_lock_ptr; -bool active_rcu_lock; +u32 active_rcu_locks;bool speculative;bool in_sleepable;
kernel/bpf/verifier.c +17 -17
@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf dst->acquired_refs = src->acquired_refs; dst->active_locks = src->active_locks; dst->active_preempt_locks = src->active_preempt_locks; - dst->active_rcu_lock = src->active_rcu_lock; + dst->active_rcu_locks = src->active_rcu_locks; dst->active_irq_id = src->active_irq_id; dst->active_lock_id = src->active_lock_id; dst->active_lock_ptr = src->active_lock_ptr;
@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)*/ static bool in_rcu_cs (struct bpf_verifier_env *env) { -return env->cur_state->active_rcu_lock || +return env->cur_state->active_rcu_locks || env->cur_state->active_locks || !in_sleepable (env); }
@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (env->subprog_info[subprog].might_sleep && - (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks || + (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks || env->cur_state->active_irq_id || !in_sleepable (env))) {verbose (env,"global functions that may sleep are not allowed in non-sleepable context,\n" "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exitreturn -EINVAL ; } -if (check_lock && env->cur_state->active_rcu_lock) { +if (check_lock && env->cur_state->active_rcu_locks) {verbose (env,"%s cannot be used inside bpf_rcu_read_lock-ed region\n" , prefix);return -EINVAL ; }
@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insnreturn err; } -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {if (fn->might_sleep) {verbose (env,"sleepable helper %s#%d in rcu_read_lock region\n" ,func_id_name (func_id), func_id);
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) { -bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ -if (reg->type &MEM_RCU ) { - reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); - reg->type |=PTR_UNTRUSTED ; - } - })); - env->cur_state->active_rcu_lock = false; +if (--env->cur_state->active_rcu_locks ==0 ) { +bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ +if (reg->type &MEM_RCU ) { + reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); + reg->type |=PTR_UNTRUSTED ; + } + })); + } }else if (sleepable) {verbose (env,"kernel func %s is sleepable within rcu_read_lock region\n" , func_name);return -EACCES ; } }else if (rcu_lock) { - env->cur_state->active_rcu_lock = true; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) {verbose (env,"unmatched rcu read unlock (kernel function %s)\n" , func_name);return -EINVAL ;
@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cif (old->active_preempt_locks != cur->active_preempt_locks)return false; -if (old->active_rcu_lock != cur->active_rcu_lock) +if (old->active_rcu_locks != cur->active_rcu_locks)return false;if (!check_ids (old->active_irq_id, cur->active_irq_id, idmap))
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +3 -1
@@ -28,6 +28,7 @@ static void test_success(void)bpf_program__set_autoload (skel->progs.two_regions, true);bpf_program__set_autoload (skel->progs.non_sleepable_1, true);bpf_program__set_autoload (skel->progs.non_sleepable_2, true); +bpf_program__set_autoload (skel->progs.nested_rcu_region, true);bpf_program__set_autoload (skel->progs.task_trusted_non_rcuptr, true);bpf_program__set_autoload (skel->progs.rcu_read_lock_subprog, true);bpf_program__set_autoload (skel->progs.rcu_read_lock_global_subprog, true);
@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {"non_sleepable_rcu_mismatch" ,"inproper_sleepable_helper" ,"inproper_sleepable_kfunc" , -"nested_rcu_region" , +"nested_rcu_region_unbalanced_1" , +"nested_rcu_region_unbalanced_2" ,"rcu_read_lock_global_subprog_lock" ,"rcu_read_lock_global_subprog_unlock" , "rcu_read_lock_sleepable_helper_global_subprog",
tools/testing/selftests/bpf/progs/rcu_read_lock.c +40 -0
@@ -278,6 +278,46 @@ int nested_rcu_region(void *ctx)return 0 ; } +SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep") +int nested_rcu_region_unbalanced_1 (void *ctx) +{ +struct task_struct *task, *real_parent; + +/* nested rcu read lock regions */ + task =bpf_get_current_task_btf (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); + real_parent = task->real_parent; +if (!real_parent) + goto out; + (void )bpf_task_storage_get (&map_a, real_parent,0 ,0 ); +out: +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +return 0 ; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep") +int nested_rcu_region_unbalanced_2 (void *ctx) +{ +struct task_struct *task, *real_parent; + +/* nested rcu read lock regions */ + task =bpf_get_current_task_btf (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); + real_parent = task->real_parent; +if (!real_parent) + goto out; + (void )bpf_task_storage_get (&map_a, real_parent,0 ,0 ); +out: +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +return 0 ; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")int task_trusted_non_rcuptr (void *ctx) {
Signature · 1 line
2.47.3
Quoted 93 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>--- include/linux/bpf_verifier.h | 2 +- kernel/bpf/verifier.c | 34 ++++++++-------- .../selftests/bpf/prog_tests/rcu_read_lock.c | 4 +- .../selftests/bpf/progs/rcu_read_lock.c | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 19 deletions(-)include/linux/bpf_verifier.h +1
-1@@ -441,7 +441,7 @@ struct bpf_verifier_state {u32 active_irq_id;u32 active_lock_id;void *active_lock_ptr; -bool active_rcu_lock; +u32 active_rcu_locks;bool speculative;bool in_sleepable;kernel/bpf/verifier.c +6
-8@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf dst->acquired_refs = src->acquired_refs; dst->active_locks = src->active_locks; dst->active_preempt_locks = src->active_preempt_locks; - dst->active_rcu_lock = src->active_rcu_lock; + dst->active_rcu_locks = src->active_rcu_locks; dst->active_irq_id = src->active_irq_id; dst->active_lock_id = src->active_lock_id; dst->active_lock_ptr = src->active_lock_ptr;@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)*/ static bool in_rcu_cs (struct bpf_verifier_env *env) { -return env->cur_state->active_rcu_lock || +return env->cur_state->active_rcu_locks || env->cur_state->active_locks || !in_sleepable (env); }@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (env->subprog_info[subprog].might_sleep && - (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks || + (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks || env->cur_state->active_irq_id || !in_sleepable (env))) {verbose (env,"global functions that may sleep are not allowed in non-sleepable context,\n" "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exitreturn -EINVAL ; } -if (check_lock && env->cur_state->active_rcu_lock) { +if (check_lock && env->cur_state->active_rcu_locks) {verbose (env,"%s cannot be used inside bpf_rcu_read_lock-ed region\n" , prefix);return -EINVAL ; }@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insnreturn err; } -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {if (fn->might_sleep) {verbose (env,"sleepable helper %s#%d in rcu_read_lock region\n" ,func_id_name (func_id), func_id);@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ;
kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13879
Quoted 10 lines
kernel/bpf/verifier.c +2
-7+ env->cur_state->active_rcu_locks++; }else if (rcu_unlock) { -bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ -if (reg->type &MEM_RCU ) { - reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); - reg->type |=PTR_UNTRUSTED ; - } - })); - env->cur_state->active_rcu_lock = false; +if (--env->cur_state->active_rcu_locks ==0 ) {
hmm. can it go negative ?
nested_rcu_region_unbalanced_1 test suppose to check it, but what kind of error is returned?
kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13880
Quoted 1 line
kernel/bpf/verifier.c +1
-0+ bpf_for_each_reg_in_vstate_mask(env-
rewrite it to avoid adding extra ident?
Quoted 39 lines
kernel/bpf/verifier.c +7
-1cur_state, state, reg, clear_mask, ({ +if (reg->type &MEM_RCU ) { + reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); + reg->type |=PTR_UNTRUSTED ; + } + })); + } }else if (sleepable) {verbose (env,"kernel func %s is sleepable within rcu_read_lock region\n" , func_name);return -EACCES ; } }else if (rcu_lock) { - env->cur_state->active_rcu_lock = true; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) {verbose (env,"unmatched rcu read unlock (kernel function %s)\n" , func_name);return -EINVAL;kernel/bpf/verifier.c +1
-1@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cif (old->active_preempt_locks != cur->active_preempt_locks)return false; -if (old->active_rcu_lock != cur->active_rcu_lock) +if (old->active_rcu_locks != cur->active_rcu_locks)return false;if (!check_ids (old->active_irq_id, cur->active_irq_id, idmap))tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +1
-0@@ -28,6 +28,7 @@ static void test_success(void)bpf_program__set_autoload (skel->progs.two_regions, true);bpf_program__set_autoload (skel->progs.non_sleepable_1, true);bpf_program__set_autoload (skel->progs.non_sleepable_2, true); +bpf_program__set_autoload (skel->progs.nested_rcu_region, true);bpf_program__set_autoload (skel->progs.task_trusted_non_rcuptr, true);bpf_program__set_autoload (skel->progs.rcu_read_lock_subprog, true);bpf_program__set_autoload (skel->progs.rcu_read_lock_global_subprog, true);
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c static const char * const inproper_region_tests[] = { · line 81
Quoted 4 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +0
-1@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {"non_sleepable_rcu_mismatch" ,"inproper_sleepable_helper" , "inproper_sleepable_kfunc", - "nested_rcu_region",
should be deleted from progs/rcu_read_lock.c too ?
This selftests hunk should in the main patch, but below new tests need to go into another patch.
pw-bot: cr
Quoted 59 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +2
-0+"nested_rcu_region_unbalanced_1" , +"nested_rcu_region_unbalanced_2" ,"rcu_read_lock_global_subprog_lock" ,"rcu_read_lock_global_subprog_unlock" , "rcu_read_lock_sleepable_helper_global_subprog",tools/testing/selftests/bpf/progs/rcu_read_lock.c +40
-1@@ -278,6 +278,46 @@ int nested_rcu_region(void *ctx)return 0 ; } +SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep") +int nested_rcu_region_unbalanced_1 (void *ctx) +{ +struct task_struct *task, *real_parent; + +/* nested rcu read lock regions */ + task =bpf_get_current_task_btf (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); + real_parent = task->real_parent; +if (!real_parent) + goto out; + (void )bpf_task_storage_get (&map_a, real_parent,0 ,0 ); +out: +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +return 0 ; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep") +int nested_rcu_region_unbalanced_2 (void *ctx) +{ +struct task_struct *task, *real_parent; + +/* nested rcu read lock regions */ + task =bpf_get_current_task_btf (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); +bpf_rcu_read_lock (); + real_parent = task->real_parent; +if (!real_parent) + goto out; + (void )bpf_task_storage_get (&map_a, real_parent,0 ,0 ); +out: +bpf_rcu_read_unlock (); +bpf_rcu_read_unlock (); +return 0 ; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")int task_trusted_non_rcuptr (void *ctx) { --2.47.3
Quoted 107 lines · Alexei Starovoitov
Quoted 93 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>--- include/linux/bpf_verifier.h | 2 +- kernel/bpf/verifier.c | 34 ++++++++-------- .../selftests/bpf/prog_tests/rcu_read_lock.c | 4 +- .../selftests/bpf/progs/rcu_read_lock.c | 40 +++++++++++++++++++ 4 files changed, 61 insertions(+), 19 deletions(-)include/linux/bpf_verifier.h +1
-1@@ -441,7 +441,7 @@ struct bpf_verifier_state {u32 active_irq_id;u32 active_lock_id;void *active_lock_ptr; -bool active_rcu_lock; +u32 active_rcu_locks;bool speculative;bool in_sleepable;kernel/bpf/verifier.c +6
-8@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf dst->acquired_refs = src->acquired_refs; dst->active_locks = src->active_locks; dst->active_preempt_locks = src->active_preempt_locks; - dst->active_rcu_lock = src->active_rcu_lock; + dst->active_rcu_locks = src->active_rcu_locks; dst->active_irq_id = src->active_irq_id; dst->active_lock_id = src->active_lock_id; dst->active_lock_ptr = src->active_lock_ptr;@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)*/ static bool in_rcu_cs (struct bpf_verifier_env *env) { -return env->cur_state->active_rcu_lock || +return env->cur_state->active_rcu_locks || env->cur_state->active_locks || !in_sleepable (env); }@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (env->subprog_info[subprog].might_sleep && - (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks || + (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks || env->cur_state->active_irq_id || !in_sleepable (env))) {verbose (env,"global functions that may sleep are not allowed in non-sleepable context,\n" "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exitreturn -EINVAL ; } -if (check_lock && env->cur_state->active_rcu_lock) { +if (check_lock && env->cur_state->active_rcu_locks) {verbose (env,"%s cannot be used inside bpf_rcu_read_lock-ed region\n" , prefix);return -EINVAL ; }@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insnreturn err; } -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {if (fn->might_sleep) {verbose (env,"sleepable helper %s#%d in rcu_read_lock region\n" ,func_id_name (func_id), func_id);@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ;kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13879
Quoted 10 lines
kernel/bpf/verifier.c +2
-7+ env->cur_state->active_rcu_locks++; }else if (rcu_unlock) { -bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ -if (reg->type &MEM_RCU ) { - reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); - reg->type |=PTR_UNTRUSTED ; - } - })); - env->cur_state->active_rcu_lock = false; +if (--env->cur_state->active_rcu_locks ==0 ) {hmm. can it go negative ?
Quoted 2 lines
nested_rcu_region_unbalanced_1test suppose to check it, but what kind of error is returned?
It can't go regative as active_rcu_locks is checked above before the line (-- == 0) is executed, nested_rcu_region_unbalanced_1 will return the following error:
unmatched rcu read unlock (kernel function bpf_rcu_read_unlock)
Quoted 5 lines
kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13880
Quoted 3 lines
kernel/bpf/verifier.c +1
-0+ bpf_for_each_reg_in_vstate_mask(env-rewrite it to avoid adding extra ident?
Ack!
Quoted 37 lines
Quoted 37 lines
kernel/bpf/verifier.c +7
-1cur_state, state, reg, clear_mask, ({ +if (reg->type &MEM_RCU ) { + reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); + reg->type |=PTR_UNTRUSTED ; + } + })); + } }else if (sleepable) {verbose (env,"kernel func %s is sleepable within rcu_read_lock region\n" , func_name);return -EACCES ; } }else if (rcu_lock) { - env->cur_state->active_rcu_lock = true; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) {verbose (env,"unmatched rcu read unlock (kernel function %s)\n" , func_name);return -EINVAL;kernel/bpf/verifier.c +1
-1@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cif (old->active_preempt_locks != cur->active_preempt_locks)return false; -if (old->active_rcu_lock != cur->active_rcu_lock) +if (old->active_rcu_locks != cur->active_rcu_locks)return false;if (!check_ids (old->active_irq_id, cur->active_irq_id, idmap))tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +1
-0@@ -28,6 +28,7 @@ static void test_success(void)bpf_program__set_autoload (skel->progs.two_regions, true);bpf_program__set_autoload (skel->progs.non_sleepable_1, true);bpf_program__set_autoload (skel->progs.non_sleepable_2, true); +bpf_program__set_autoload (skel->progs.nested_rcu_region, true);bpf_program__set_autoload (skel->progs.task_trusted_non_rcuptr, true);bpf_program__set_autoload (skel->progs.rcu_read_lock_subprog, true);
Quoted 8 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c static const char * const inproper_region_tests[] = { · line 81
Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +0
-1bpf_program__set_autoload (skel->progs.rcu_read_lock_global_subprog, true);@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {"non_sleepable_rcu_mismatch" ,"inproper_sleepable_helper" , "inproper_sleepable_kfunc", - "nested_rcu_region",should be deleted from
progs/rcu_read_lock.ctoo ?
This is not deleted, just moved above into test_success() because now it will succeed.
Quoted 2 lines
This selftests hunk should in the main patch, but below new tests need to go into another patch.
Sure, I will split the changes into two patches.
Thanks,
Puranjay
Eduard Zingerman
maintainerQuoted 9 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
[...]
Quoted 23 lines
kernel/bpf/verifier.c +9
-9@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) { -bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ -if (reg->type &MEM_RCU ) { - reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); - reg->type |=PTR_UNTRUSTED ; - } - })); - env->cur_state->active_rcu_lock = false; +if (--env->cur_state->active_rcu_locks ==0 ) { +bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ +if (reg->type &MEM_RCU ) { + reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); + reg->type |=PTR_UNTRUSTED ; + } + })); + }
kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13895
Quoted 10 lines
kernel/bpf/verifier.c +1
-1}else if (sleepable) {verbose (env,"kernel func %s is sleepable within rcu_read_lock region\n" , func_name);return -EACCES ; } }else if (rcu_lock) { - env->cur_state->active_rcu_lock = true; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) {verbose (env,"unmatched rcu read unlock (kernel function %s)\n" , func_name);return -EINVAL ;
Nit: active_rcu_locks increment in two places can be avoided e.g. as follows:
if (rcu_lock) {
env->cur_state->active_rcu_locks++;
} else if (rcu_unlock) {
struct bpf_func_state *state;
struct bpf_reg_state *reg;
u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
if (env->cur_state->active_rcu_locks == 0) {
verbose(private_data: env, fmt: "unmatched rcu read unlock (kernel function %s)\n", func_name);
return -EINVAL;
}
if (--env->cur_state->active_rcu_locks == 0) {
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
if (reg->type & MEM_RCU) {
reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
reg->type |= PTR_UNTRUSTED;
}
}));
}
} else if (sleepable) {
verbose(private_data: env, fmt: "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
return -EACCES;
}
if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
verbose(private_data: env, fmt: "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
return -EACCES;
}[...]
Quoted 78 lines · Eduard Zingerman
Quoted 9 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
[...]
Quoted 23 lines
kernel/bpf/verifier.c +9
-9@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) { -bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ -if (reg->type &MEM_RCU ) { - reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); - reg->type |=PTR_UNTRUSTED ; - } - })); - env->cur_state->active_rcu_lock = false; +if (--env->cur_state->active_rcu_locks ==0 ) { +bpf_for_each_reg_in_vstate_mask (env->cur_state, state, reg, clear_mask, ({ +if (reg->type &MEM_RCU ) { + reg->type &= ~(MEM_RCU |PTR_MAYBE_NULL ); + reg->type |=PTR_UNTRUSTED ; + } + })); + }kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13895
Quoted 10 lines
kernel/bpf/verifier.c +1
-1}else if (sleepable) {verbose (env,"kernel func %s is sleepable within rcu_read_lock region\n" , func_name);return -EACCES ; } }else if (rcu_lock) { - env->cur_state->active_rcu_lock = true; + env->cur_state->active_rcu_locks++; }else if (rcu_unlock) {verbose (env,"unmatched rcu read unlock (kernel function %s)\n" , func_name);return -EINVAL ;Nit:
active_rcu_locksincrement in two places can be avoided e.g. as follows:if (rcu_lock) { env->cur_state->active_rcu_locks++; } else if (rcu_unlock) { struct bpf_func_state *state; struct bpf_reg_state *reg; u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); if (env->cur_state->active_rcu_locks == 0) { verbose(private_data: env, fmt: "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } if (--env->cur_state->active_rcu_locks == 0) { bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({ if (reg->type & MEM_RCU) { reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; } })); } } else if (sleepable) { verbose(private_data: env, fmt: "kernel func %s is sleepable within rcu_read_lock region\n", func_name); return -EACCES; } if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(private_data: env, fmt: "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; }
Quoted 1 line
[...]
I agree, this looks better. Will use it in the next version.
Thanks,
Puranjay
Kumar Kartikeya Dwivedi
maintainerQuoted 11 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
Apart from other comments, we discussed this once offline as well, but it makes sense to upgrade the rcu_read_lock selftest to be higher fidelity and match the error strings instead of simply testing failure to load (which can lead to false positives). The other missing cases to test would be interaction with subprogs (both non-global and global), to make sure sleepable / non-sleepable summarization is working correctly.
The logic itself looks okay to me.
Quoted 9 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
[...]
Quoted 10 lines
kernel/bpf/verifier.c +1
-1@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );
kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877
Quoted 6 lines
kernel/bpf/verifier.c +1
-2@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++;
Could we add a check for the maximum of 'active_rcu_locks'?
From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath 'rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.
This scenario was already discussed in the thread:
"[BUG] Deadlock triggered by bpfsnoop funcgraph feature"[1].
[1]
https://lore.kernel.org/bpf/[email protected]/
Thanks,
Leon
[...]
Quoted 32 lines · Leon Hwang
Quoted 9 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
[...]
Quoted 10 lines
kernel/bpf/verifier.c +1
-1@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877
Quoted 6 lines
kernel/bpf/verifier.c +1
-2@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++;Could we add a check for the maximum of '
active_rcu_locks'?
Quoted 5 lines
From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath '
rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.
IIUC what you are saying is that if I attach a BPF tracing program to something under rcu_read_lock() and then call bpf_rcu_read_lock() in the BPF program then there could recursion? Wouldn't that be triggered even with a single call to bpf_rcu_read_lock() ?
Thanks,
Puranjay
Quoted 40 lines · Puranjay Mohan
Quoted 32 lines · Leon Hwang
Quoted 9 lines · Puranjay Mohan
Currently, nested rcu critical sections are rejected by the verifier and
rcu_lockstate is managed by a boolean variable. Add support for nested rcu critical sections by makeactive_rcu_locksa counter similar toactive_preempt_locks.bpf_rcu_read_lock()increments this counter andbpf_rcu_read_unlock()decrements it,MEM_RCU->PTR_UNTRUSTEDtransition happens whenactive_rcu_locksdrops to 0.Signed-off-by Puranjay Mohan <[email protected]>---
[...]
Quoted 10 lines
kernel/bpf/verifier.c +1
-1@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, preempt_disable =is_kfunc_bpf_preempt_disable (&meta); preempt_enable =is_kfunc_bpf_preempt_enable (&meta); -if (env->cur_state->active_rcu_lock) { +if (env->cur_state->active_rcu_locks) {struct bpf_func_state *state;struct bpf_reg_state *reg;u32 clear_mask = (1 <<STACK_SPILL ) | (1 <<STACK_ITER );kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877
Quoted 6 lines
kernel/bpf/verifier.c +1
-2@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, }if (rcu_lock) { -verbose (env,"nested rcu read lock (kernel function %s)\n" , func_name); -return -EINVAL ; + env->cur_state->active_rcu_locks++;Could we add a check for the maximum of '
active_rcu_locks'?Quoted 5 lines
From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath '
rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.
Quoted 5 lines
IIUC what you are saying is that if I attach a BPF tracing program to something under
rcu_read_lock()and then callbpf_rcu_read_lock()in the BPF program then there could recursion? Wouldn't that be triggered even with a single call tobpf_rcu_read_lock()?
The recursion can happen, yes. But BPF tracing programs won't run recursively.
However, the deadlock I ran into wasn’t caused directly by recursive BPF programs, and I haven’t yet tracked down the exact root cause.
That said, even a single call to bpf_rcu_read_lock() can potentially trigger a deadlock.
So I think it would be better to add bpf_rcu_read_lock() and bpf_rcu_read_unlock() to the btf_id_deny list, similar to __rcu_read_lock() and __rcu_read_unlock().
Thanks,
Leon