Skip to contentSkip to conversation

bpf: support nested rcu critical sections

PATCH Superseded Puranjay Mohan 5 participants 9 messages Sep 16, 2025 – Sep 18, 2025

Subsystems (4)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 bpf: support nested rcu critical sections

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---
 include/linux/bpf_verifier.h                  |  2 +-
 kernel/bpf/verifier.c                         | 34 ++++++++--------
 .../selftests/bpf/prog_tests/rcu_read_lock.c  |  4 +-
 .../selftests/bpf/progs/rcu_read_lock.c       | 40 +++++++++++++++++++
 4 files changed, 61 insertions(+), 19 deletions(-)
include/linux/bpf_verifier.h +1 -1
index 020de62bd09c..3fb4632d5eed 100644
@@ -441,7 +441,7 @@ struct bpf_verifier_state {
 	u32 active_irq_id;
 	u32 active_lock_id;
 	void *active_lock_ptr;
-	bool active_rcu_lock;
+	u32 active_rcu_locks;
 
 	bool speculative;
 	bool in_sleepable;
kernel/bpf/verifier.c +17 -17
index 1029380f84db..645af66e29ab 100644
@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
 	dst->acquired_refs = src->acquired_refs;
 	dst->active_locks = src->active_locks;
 	dst->active_preempt_locks = src->active_preempt_locks;
-	dst->active_rcu_lock = src->active_rcu_lock;
+	dst->active_rcu_locks = src->active_rcu_locks;
 	dst->active_irq_id = src->active_irq_id;
 	dst->active_lock_id = src->active_lock_id;
 	dst->active_lock_ptr = src->active_lock_ptr;
@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)
  */
 static bool in_rcu_cs(struct bpf_verifier_env *env)
 {
-	return env->cur_state->active_rcu_lock ||
+	return env->cur_state->active_rcu_locks ||
 	       env->cur_state->active_locks ||
 	       !in_sleepable(env);
 }
@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (env->subprog_info[subprog].might_sleep &&
-		    (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks ||
+		    (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||
 		     env->cur_state->active_irq_id || !in_sleepable(env))) {
 			verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
 				     "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit
 		return -EINVAL;
 	}
 
-	if (check_lock && env->cur_state->active_rcu_lock) {
+	if (check_lock && env->cur_state->active_rcu_locks) {
 		verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
 		return -EINVAL;
 	}
@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		return err;
 	}
 
-	if (env->cur_state->active_rcu_lock) {
+	if (env->cur_state->active_rcu_locks) {
 		if (fn->might_sleep) {
 			verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
 				func_id_name(func_id), func_id);
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-	if (env->cur_state->active_rcu_lock) {
+	if (env->cur_state->active_rcu_locks) {
 		struct bpf_func_state *state;
 		struct bpf_reg_state *reg;
 		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;
 		} else if (rcu_unlock) {
-			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
-				if (reg->type & MEM_RCU) {
-					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
-					reg->type |= PTR_UNTRUSTED;
-				}
-			}));
-			env->cur_state->active_rcu_lock = false;
+			if (--env->cur_state->active_rcu_locks == 0) {
+				bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
+					if (reg->type & MEM_RCU) {
+						reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+						reg->type |= PTR_UNTRUSTED;
+					}
+				}));
+			}
 		} else if (sleepable) {
 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
 			return -EACCES;
 		}
 	} else if (rcu_lock) {
-		env->cur_state->active_rcu_lock = true;
+		env->cur_state->active_rcu_locks++;
 	} else if (rcu_unlock) {
 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
 		return -EINVAL;
@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c
 	if (old->active_preempt_locks != cur->active_preempt_locks)
 		return false;
 
-	if (old->active_rcu_lock != cur->active_rcu_lock)
+	if (old->active_rcu_locks != cur->active_rcu_locks)
 		return false;
 
 	if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +3 -1
index c9f855e5da24..246eb259c08a 100644
@@ -28,6 +28,7 @@ static void test_success(void)
 	bpf_program__set_autoload(skel->progs.two_regions, true);
 	bpf_program__set_autoload(skel->progs.non_sleepable_1, true);
 	bpf_program__set_autoload(skel->progs.non_sleepable_2, true);
+	bpf_program__set_autoload(skel->progs.nested_rcu_region, true);
 	bpf_program__set_autoload(skel->progs.task_trusted_non_rcuptr, true);
 	bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog, true);
 	bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true);
@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {
 	"non_sleepable_rcu_mismatch",
 	"inproper_sleepable_helper",
 	"inproper_sleepable_kfunc",
-	"nested_rcu_region",
+	"nested_rcu_region_unbalanced_1",
+	"nested_rcu_region_unbalanced_2",
 	"rcu_read_lock_global_subprog_lock",
 	"rcu_read_lock_global_subprog_unlock",
 	"rcu_read_lock_sleepable_helper_global_subprog",
tools/testing/selftests/bpf/progs/rcu_read_lock.c +40 -0
index 3a868a199349..d70c28824bbe 100644
@@ -278,6 +278,46 @@ int nested_rcu_region(void *ctx)
 	return 0;
 }
 
+SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
+int nested_rcu_region_unbalanced_1(void *ctx)
+{
+	struct task_struct *task, *real_parent;
+
+	/* nested rcu read lock regions */
+	task = bpf_get_current_task_btf();
+	bpf_rcu_read_lock();
+	bpf_rcu_read_lock();
+	real_parent = task->real_parent;
+	if (!real_parent)
+		goto out;
+	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
+out:
+	bpf_rcu_read_unlock();
+	bpf_rcu_read_unlock();
+	bpf_rcu_read_unlock();
+	return 0;
+}
+
+SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
+int nested_rcu_region_unbalanced_2(void *ctx)
+{
+	struct task_struct *task, *real_parent;
+
+	/* nested rcu read lock regions */
+	task = bpf_get_current_task_btf();
+	bpf_rcu_read_lock();
+	bpf_rcu_read_lock();
+	bpf_rcu_read_lock();
+	real_parent = task->real_parent;
+	if (!real_parent)
+		goto out;
+	(void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
+out:
+	bpf_rcu_read_unlock();
+	bpf_rcu_read_unlock();
+	return 0;
+}
+
 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
 int task_trusted_non_rcuptr(void *ctx)
 {
Signature · 1 line
2.47.3
#1 lore raw
Quoted 93 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---
 include/linux/bpf_verifier.h                  |  2 +-
 kernel/bpf/verifier.c                         | 34 ++++++++--------
 .../selftests/bpf/prog_tests/rcu_read_lock.c  |  4 +-
 .../selftests/bpf/progs/rcu_read_lock.c       | 40 +++++++++++++++++++
 4 files changed, 61 insertions(+), 19 deletions(-)
include/linux/bpf_verifier.h +1 -1
index 020de62bd09c..3fb4632d5eed 100644
@@ -441,7 +441,7 @@ struct bpf_verifier_state {
        u32 active_irq_id;
        u32 active_lock_id;
        void *active_lock_ptr;
-       bool active_rcu_lock;
+       u32 active_rcu_locks;
 
        bool speculative;
        bool in_sleepable;
kernel/bpf/verifier.c +6 -8
index 1029380f84db..645af66e29ab 100644
@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
        dst->acquired_refs = src->acquired_refs;
        dst->active_locks = src->active_locks;
        dst->active_preempt_locks = src->active_preempt_locks;
-       dst->active_rcu_lock = src->active_rcu_lock;
+       dst->active_rcu_locks = src->active_rcu_locks;
        dst->active_irq_id = src->active_irq_id;
        dst->active_lock_id = src->active_lock_id;
        dst->active_lock_ptr = src->active_lock_ptr;
@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)
  */
 static bool in_rcu_cs(struct bpf_verifier_env *env)
 {
-       return env->cur_state->active_rcu_lock ||
+       return env->cur_state->active_rcu_locks ||
               env->cur_state->active_locks ||
               !in_sleepable(env);
 }
@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                }
 
                if (env->subprog_info[subprog].might_sleep &&
-                   (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks ||
+                   (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||
                     env->cur_state->active_irq_id || !in_sleepable(env))) {
                        verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
                                     "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit
                return -EINVAL;
        }
 
-       if (check_lock && env->cur_state->active_rcu_lock) {
+       if (check_lock && env->cur_state->active_rcu_locks) {
                verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
                return -EINVAL;
        }
@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                return err;
        }
 
-       if (env->cur_state->active_rcu_lock) {
+       if (env->cur_state->active_rcu_locks) {
                if (fn->might_sleep) {
                        verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
                                func_id_name(func_id), func_id);
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
        preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
        preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-       if (env->cur_state->active_rcu_lock) {
+       if (env->cur_state->active_rcu_locks) {
                struct bpf_func_state *state;
                struct bpf_reg_state *reg;
                u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                }
 
                if (rcu_lock) {
-                       verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-                       return -EINVAL;

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13879

Quoted 10 lines
kernel/bpf/verifier.c +2 -7
+                       env->cur_state->active_rcu_locks++;
                } else if (rcu_unlock) {
-                       bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
-                               if (reg->type & MEM_RCU) {
-                                       reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
-                                       reg->type |= PTR_UNTRUSTED;
-                               }
-                       }));
-                       env->cur_state->active_rcu_lock = false;
+                       if (--env->cur_state->active_rcu_locks == 0) {

hmm. can it go negative ?

nested_rcu_region_unbalanced_1 test suppose to check it, but what kind of error is returned?

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13880

Quoted 1 line
kernel/bpf/verifier.c +1 -0
+                               bpf_for_each_reg_in_vstate_mask(env-

rewrite it to avoid adding extra ident?

Quoted 39 lines
kernel/bpf/verifier.c +7 -1
 cur_state, state, reg, clear_mask, ({
+                                       if (reg->type & MEM_RCU) {
+                                               reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+                                               reg->type |= PTR_UNTRUSTED;
+                                       }
+                               }));
+                       }
                } else if (sleepable) {
                        verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
                        return -EACCES;
                }
        } else if (rcu_lock) {
-               env->cur_state->active_rcu_lock = true;
+               env->cur_state->active_rcu_locks++;
        } else if (rcu_unlock) {
                verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
                return -EINVAL;
kernel/bpf/verifier.c +1 -1
@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c
        if (old->active_preempt_locks != cur->active_preempt_locks)
                return false;
 
-       if (old->active_rcu_lock != cur->active_rcu_lock)
+       if (old->active_rcu_locks != cur->active_rcu_locks)
                return false;
 
        if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +1 -0
index c9f855e5da24..246eb259c08a 100644
@@ -28,6 +28,7 @@ static void test_success(void)
        bpf_program__set_autoload(skel->progs.two_regions, true);
        bpf_program__set_autoload(skel->progs.non_sleepable_1, true);
        bpf_program__set_autoload(skel->progs.non_sleepable_2, true);
+       bpf_program__set_autoload(skel->progs.nested_rcu_region, true);
        bpf_program__set_autoload(skel->progs.task_trusted_non_rcuptr, true);
        bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog, true);
        bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true);

tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c static const char * const inproper_region_tests[] = { · line 81

Quoted 4 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +0 -1
@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {
        "non_sleepable_rcu_mismatch",
        "inproper_sleepable_helper",
        "inproper_sleepable_kfunc",
-       "nested_rcu_region",

should be deleted from progs/rcu_read_lock.c too ?

This selftests hunk should in the main patch, but below new tests need to go into another patch.

pw-bot: cr

Quoted 59 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +2 -0
+       "nested_rcu_region_unbalanced_1",
+       "nested_rcu_region_unbalanced_2",
        "rcu_read_lock_global_subprog_lock",
        "rcu_read_lock_global_subprog_unlock",
        "rcu_read_lock_sleepable_helper_global_subprog",
tools/testing/selftests/bpf/progs/rcu_read_lock.c +40 -1
index 3a868a199349..d70c28824bbe 100644
@@ -278,6 +278,46 @@ int nested_rcu_region(void *ctx)
        return 0;
 }
 
+SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
+int nested_rcu_region_unbalanced_1(void *ctx)
+{
+       struct task_struct *task, *real_parent;
+
+       /* nested rcu read lock regions */
+       task = bpf_get_current_task_btf();
+       bpf_rcu_read_lock();
+       bpf_rcu_read_lock();
+       real_parent = task->real_parent;
+       if (!real_parent)
+               goto out;
+       (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
+out:
+       bpf_rcu_read_unlock();
+       bpf_rcu_read_unlock();
+       bpf_rcu_read_unlock();
+       return 0;
+}
+
+SEC("?fentry.s/" SYS_PREFIX "sys_nanosleep")
+int nested_rcu_region_unbalanced_2(void *ctx)
+{
+       struct task_struct *task, *real_parent;
+
+       /* nested rcu read lock regions */
+       task = bpf_get_current_task_btf();
+       bpf_rcu_read_lock();
+       bpf_rcu_read_lock();
+       bpf_rcu_read_lock();
+       real_parent = task->real_parent;
+       if (!real_parent)
+               goto out;
+       (void)bpf_task_storage_get(&map_a, real_parent, 0, 0);
+out:
+       bpf_rcu_read_unlock();
+       bpf_rcu_read_unlock();
+       return 0;
+}
+
 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
 int task_trusted_non_rcuptr(void *ctx)
 {
--

2.47.3

#2 lore raw
Quoted 107 lines · Alexei Starovoitov
Quoted 93 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---
 include/linux/bpf_verifier.h                  |  2 +-
 kernel/bpf/verifier.c                         | 34 ++++++++--------
 .../selftests/bpf/prog_tests/rcu_read_lock.c  |  4 +-
 .../selftests/bpf/progs/rcu_read_lock.c       | 40 +++++++++++++++++++
 4 files changed, 61 insertions(+), 19 deletions(-)
include/linux/bpf_verifier.h +1 -1
index 020de62bd09c..3fb4632d5eed 100644
@@ -441,7 +441,7 @@ struct bpf_verifier_state {
        u32 active_irq_id;
        u32 active_lock_id;
        void *active_lock_ptr;
-       bool active_rcu_lock;
+       u32 active_rcu_locks;
 
        bool speculative;
        bool in_sleepable;
kernel/bpf/verifier.c +6 -8
index 1029380f84db..645af66e29ab 100644
@@ -1438,7 +1438,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
        dst->acquired_refs = src->acquired_refs;
        dst->active_locks = src->active_locks;
        dst->active_preempt_locks = src->active_preempt_locks;
-       dst->active_rcu_lock = src->active_rcu_lock;
+       dst->active_rcu_locks = src->active_rcu_locks;
        dst->active_irq_id = src->active_irq_id;
        dst->active_lock_id = src->active_lock_id;
        dst->active_lock_ptr = src->active_lock_ptr;
@@ -5924,7 +5924,7 @@ static bool in_sleepable(struct bpf_verifier_env *env)
  */
 static bool in_rcu_cs(struct bpf_verifier_env *env)
 {
-       return env->cur_state->active_rcu_lock ||
+       return env->cur_state->active_rcu_locks ||
               env->cur_state->active_locks ||
               !in_sleepable(env);
 }
@@ -10684,7 +10684,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                }
 
                if (env->subprog_info[subprog].might_sleep &&
-                   (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks ||
+                   (env->cur_state->active_rcu_locks || env->cur_state->active_preempt_locks ||
                     env->cur_state->active_irq_id || !in_sleepable(env))) {
                        verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
                                     "i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
@@ -11231,7 +11231,7 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit
                return -EINVAL;
        }
 
-       if (check_lock && env->cur_state->active_rcu_lock) {
+       if (check_lock && env->cur_state->active_rcu_locks) {
                verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
                return -EINVAL;
        }
@@ -11426,7 +11426,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
                return err;
        }
 
-       if (env->cur_state->active_rcu_lock) {
+       if (env->cur_state->active_rcu_locks) {
                if (fn->might_sleep) {
                        verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
                                func_id_name(func_id), func_id);
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
        preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
        preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-       if (env->cur_state->active_rcu_lock) {
+       if (env->cur_state->active_rcu_locks) {
                struct bpf_func_state *state;
                struct bpf_reg_state *reg;
                u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                }
 
                if (rcu_lock) {
-                       verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-                       return -EINVAL;

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13879

Quoted 10 lines
kernel/bpf/verifier.c +2 -7
+                       env->cur_state->active_rcu_locks++;
                } else if (rcu_unlock) {
-                       bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
-                               if (reg->type & MEM_RCU) {
-                                       reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
-                                       reg->type |= PTR_UNTRUSTED;
-                               }
-                       }));
-                       env->cur_state->active_rcu_lock = false;
+                       if (--env->cur_state->active_rcu_locks == 0) {

hmm. can it go negative ?

Quoted 2 lines

nested_rcu_region_unbalanced_1 test suppose to check it, but what kind of error is returned?

It can't go regative as active_rcu_locks is checked above before the line (-- == 0) is executed, nested_rcu_region_unbalanced_1 will return the following error:

unmatched rcu read unlock (kernel function bpf_rcu_read_unlock)

Quoted 5 lines

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13880

Quoted 3 lines
kernel/bpf/verifier.c +1 -0
+                               bpf_for_each_reg_in_vstate_mask(env-

rewrite it to avoid adding extra ident?

Ack!

Quoted 37 lines
Quoted 37 lines
kernel/bpf/verifier.c +7 -1
 cur_state, state, reg, clear_mask, ({
+                                       if (reg->type & MEM_RCU) {
+                                               reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+                                               reg->type |= PTR_UNTRUSTED;
+                                       }
+                               }));
+                       }
                } else if (sleepable) {
                        verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
                        return -EACCES;
                }
        } else if (rcu_lock) {
-               env->cur_state->active_rcu_lock = true;
+               env->cur_state->active_rcu_locks++;
        } else if (rcu_unlock) {
                verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
                return -EINVAL;
kernel/bpf/verifier.c +1 -1
@@ -18887,7 +18887,7 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c
        if (old->active_preempt_locks != cur->active_preempt_locks)
                return false;
 
-       if (old->active_rcu_lock != cur->active_rcu_lock)
+       if (old->active_rcu_locks != cur->active_rcu_locks)
                return false;
 
        if (!check_ids(old->active_irq_id, cur->active_irq_id, idmap))
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +1 -0
index c9f855e5da24..246eb259c08a 100644
@@ -28,6 +28,7 @@ static void test_success(void)
        bpf_program__set_autoload(skel->progs.two_regions, true);
        bpf_program__set_autoload(skel->progs.non_sleepable_1, true);
        bpf_program__set_autoload(skel->progs.non_sleepable_2, true);
+       bpf_program__set_autoload(skel->progs.nested_rcu_region, true);
        bpf_program__set_autoload(skel->progs.task_trusted_non_rcuptr, true);
        bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog, true);
Quoted 8 lines

tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c static const char * const inproper_region_tests[] = { · line 81

Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +0 -1
        bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true);
@@ -78,7 +79,8 @@ static const char * const inproper_region_tests[] = {
        "non_sleepable_rcu_mismatch",
        "inproper_sleepable_helper",
        "inproper_sleepable_kfunc",
-       "nested_rcu_region",

should be deleted from progs/rcu_read_lock.c too ?

This is not deleted, just moved above into test_success() because now it will succeed.

Quoted 2 lines

This selftests hunk should in the main patch, but below new tests need to go into another patch.

Sure, I will split the changes into two patches.

Thanks,
Puranjay

maintainer
#3 lore raw
In reply to Puranjay Mohan
Quoted 9 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

[...]

Quoted 23 lines
kernel/bpf/verifier.c +9 -9
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;
 		} else if (rcu_unlock) {
-			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
-				if (reg->type & MEM_RCU) {
-					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
-					reg->type |= PTR_UNTRUSTED;
-				}
-			}));
-			env->cur_state->active_rcu_lock = false;
+			if (--env->cur_state->active_rcu_locks == 0) {
+				bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
+					if (reg->type & MEM_RCU) {
+						reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+						reg->type |= PTR_UNTRUSTED;
+					}
+				}));
+			}

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13895

Quoted 10 lines
kernel/bpf/verifier.c +1 -1
 		} else if (sleepable) {
 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
 			return -EACCES;
 		}
 	} else if (rcu_lock) {
-		env->cur_state->active_rcu_lock = true;
+		env->cur_state->active_rcu_locks++;
 	} else if (rcu_unlock) {
 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
 		return -EINVAL;

Nit: active_rcu_locks increment in two places can be avoided e.g. as follows:

        if (rcu_lock) {
                env->cur_state->active_rcu_locks++;
        } else if (rcu_unlock) {
                struct bpf_func_state *state;
                struct bpf_reg_state *reg;
                u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

                if (env->cur_state->active_rcu_locks == 0) {
                        verbose(private_data: env, fmt: "unmatched rcu read unlock (kernel function %s)\n", func_name);
                        return -EINVAL;
                }
                if (--env->cur_state->active_rcu_locks == 0) {
                        bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
                                if (reg->type & MEM_RCU) {
                                        reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
                                        reg->type |= PTR_UNTRUSTED;
                                }
                        }));
                }
        } else if (sleepable) {
                verbose(private_data: env, fmt: "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
                return -EACCES;
        }

        if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
                verbose(private_data: env, fmt: "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
                return -EACCES;
        }

[...]

#4 lore raw
Quoted 78 lines · Eduard Zingerman
Quoted 9 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

[...]

Quoted 23 lines
kernel/bpf/verifier.c +9 -9
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;
 		} else if (rcu_unlock) {
-			bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
-				if (reg->type & MEM_RCU) {
-					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
-					reg->type |= PTR_UNTRUSTED;
-				}
-			}));
-			env->cur_state->active_rcu_lock = false;
+			if (--env->cur_state->active_rcu_locks == 0) {
+				bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
+					if (reg->type & MEM_RCU) {
+						reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
+						reg->type |= PTR_UNTRUSTED;
+					}
+				}));
+			}

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13895

Quoted 10 lines
kernel/bpf/verifier.c +1 -1
 		} else if (sleepable) {
 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
 			return -EACCES;
 		}
 	} else if (rcu_lock) {
-		env->cur_state->active_rcu_lock = true;
+		env->cur_state->active_rcu_locks++;
 	} else if (rcu_unlock) {
 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
 		return -EINVAL;

Nit: active_rcu_locks increment in two places can be avoided e.g. as follows:

        if (rcu_lock) {
                env->cur_state->active_rcu_locks++;
        } else if (rcu_unlock) {
                struct bpf_func_state *state;
                struct bpf_reg_state *reg;
                u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

                if (env->cur_state->active_rcu_locks == 0) {
                        verbose(private_data: env, fmt: "unmatched rcu read unlock (kernel function %s)\n", func_name);
                        return -EINVAL;
                }
                if (--env->cur_state->active_rcu_locks == 0) {
                        bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
                                if (reg->type & MEM_RCU) {
                                        reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
                                        reg->type |= PTR_UNTRUSTED;
                                }
                        }));
                }
        } else if (sleepable) {
                verbose(private_data: env, fmt: "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
                return -EACCES;
        }

        if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
                verbose(private_data: env, fmt: "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
                return -EACCES;
        }
Quoted 1 line

[...]

I agree, this looks better. Will use it in the next version.

Thanks,
Puranjay

#5 lore raw
In reply to Puranjay Mohan
Quoted 11 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

Apart from other comments, we discussed this once offline as well, but it makes sense to upgrade the rcu_read_lock selftest to be higher fidelity and match the error strings instead of simply testing failure to load (which can lead to false positives). The other missing cases to test would be interaction with subprogs (both non-global and global), to make sure sleepable / non-sleepable summarization is working correctly.

The logic itself looks okay to me.

#6 lore raw
In reply to Puranjay Mohan
Quoted 9 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

[...]

Quoted 10 lines
kernel/bpf/verifier.c +1 -1
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-	if (env->cur_state->active_rcu_lock) {
+	if (env->cur_state->active_rcu_locks) {
 		struct bpf_func_state *state;
 		struct bpf_reg_state *reg;
 		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877

Quoted 6 lines
kernel/bpf/verifier.c +1 -2
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;

Could we add a check for the maximum of 'active_rcu_locks'?

From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath 'rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.

This scenario was already discussed in the thread:
"[BUG] Deadlock triggered by bpfsnoop funcgraph feature"[1].

[1]
https://lore.kernel.org/bpf/[email protected]/

Thanks,
Leon

[...]

#7 lore raw
Quoted 32 lines · Leon Hwang
Quoted 9 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

[...]

Quoted 10 lines
kernel/bpf/verifier.c +1 -1
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-	if (env->cur_state->active_rcu_lock) {
+	if (env->cur_state->active_rcu_locks) {
 		struct bpf_func_state *state;
 		struct bpf_reg_state *reg;
 		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877

Quoted 6 lines
kernel/bpf/verifier.c +1 -2
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;

Could we add a check for the maximum of 'active_rcu_locks'?

Quoted 5 lines

From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath 'rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.

IIUC what you are saying is that if I attach a BPF tracing program to something under rcu_read_lock() and then call bpf_rcu_read_lock() in the BPF program then there could recursion? Wouldn't that be triggered even with a single call to bpf_rcu_read_lock() ?

Thanks,
Puranjay

#8 lore raw
Quoted 40 lines · Puranjay Mohan
Quoted 32 lines · Leon Hwang
Quoted 9 lines · Puranjay Mohan

Currently, nested rcu critical sections are rejected by the verifier and rcu_lock state is managed by a boolean variable. Add support for nested rcu critical sections by make active_rcu_locks a counter similar to active_preempt_locks. bpf_rcu_read_lock() increments this counter and bpf_rcu_read_unlock() decrements it, MEM_RCU -> PTR_UNTRUSTED transition happens when active_rcu_locks drops to 0.

---

[...]

Quoted 10 lines
kernel/bpf/verifier.c +1 -1
@@ -13863,7 +13863,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	preempt_disable = is_kfunc_bpf_preempt_disable(&meta);
 	preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
 
-	if (env->cur_state->active_rcu_lock) {
+	if (env->cur_state->active_rcu_locks) {
 		struct bpf_func_state *state;
 		struct bpf_reg_state *reg;
 		u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);

kernel/bpf/verifier.c static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · line 13877

Quoted 6 lines
kernel/bpf/verifier.c +1 -2
@@ -13874,22 +13874,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (rcu_lock) {
-			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
-			return -EINVAL;
+			env->cur_state->active_rcu_locks++;

Could we add a check for the maximum of 'active_rcu_locks'?

Quoted 5 lines

From a cracker's perspective, this could potentially be abused to stall the kernel or trigger a deadlock. Underneath 'rcu_read_lock()', there are several RCU functions that tracing programs are able to attach to. If those functions are traced, a deadlock can be triggered.

Quoted 5 lines

IIUC what you are saying is that if I attach a BPF tracing program to something under rcu_read_lock() and then call bpf_rcu_read_lock() in the BPF program then there could recursion? Wouldn't that be triggered even with a single call to bpf_rcu_read_lock() ?

The recursion can happen, yes. But BPF tracing programs won't run recursively.

However, the deadlock I ran into wasn’t caused directly by recursive BPF programs, and I haven’t yet tracked down the exact root cause.

That said, even a single call to bpf_rcu_read_lock() can potentially trigger a deadlock.

So I think it would be better to add bpf_rcu_read_lock() and bpf_rcu_read_unlock() to the btf_id_deny list, similar to __rcu_read_lock() and __rcu_read_unlock().

Thanks,
Leon