bpf: consistently use bpf_rcu_lock_held() everywhere
Andrii Nakryiko
maintainerPATCH v2 1/1 bpf: consistently use bpf_rcu_lock_held() everywhere
We have many places which open-code what's now is bpf_rcu_lock_held() macro, so replace all those places with a clean and short macro invocation. For that, move bpf_rcu_lock_held() macro into include/linux/bpf.h.
---v1->v2:
- move bpf_rcu_lock_held() outside of #ifdef CONFIG_BPF_SYSCALL area (kernel test robot).
include/linux/bpf.h | 3 +++ include/linux/bpf_local_storage.h | 3 --- kernel/bpf/hashtab.c | 21 +++++++-------------- kernel/bpf/helpers.c | 12 ++++-------- 4 files changed, 14 insertions(+), 25 deletions(-)
include/linux/bpf.h +3 -0
@@ -2381,6 +2381,9 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,bool bpf_jit_bypass_spec_v1 (void );bool bpf_jit_bypass_spec_v4 (void ); +#define bpf_rcu_lock_held () \ + (rcu_read_lock_held() || rcu_read_lock_trace_held() || rcu_read_lock_bh_held()) +#ifdef CONFIG_BPF_SYSCALL DECLARE_PER_CPU (int, bpf_prog_active);extern struct mutex bpf_stats_enabled_mutex;
include/linux/bpf_local_storage.h +0 -3
@@ -18,9 +18,6 @@#define BPF_LOCAL_STORAGE_CACHE_SIZE 16 -#define bpf_rcu_lock_held () \ - (rcu_read_lock_held() || rcu_read_lock_trace_held() || \ - rcu_read_lock_bh_held())struct bpf_local_storage_map_bucket {struct hlist_head list;raw_spinlock_t lock;
kernel/bpf/hashtab.c +7 -14
@@ -657,8 +657,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)struct htab_elem *l;u32 hash, key_size; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1086,8 +1085,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1194,8 +1192,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1263,8 +1260,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1326,8 +1322,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1404,8 +1399,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)u32 hash, key_size;int ret; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
@@ -1440,8 +1434,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)u32 hash, key_size;int ret; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;
kernel/bpf/helpers.c +4 -8
@@ -42,8 +42,7 @@*/ BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return (unsigned long ) map->ops->map_lookup_elem (map, key); }
@@ -59,8 +58,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, void *, value, u64, flags) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return map->ops->map_update_elem (map, key, value, flags); }
@@ -77,8 +75,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return map->ops->map_delete_elem (map, key); }
@@ -134,8 +131,7 @@ const struct bpf_func_proto bpf_map_peek_elem_proto = { BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return (unsigned long ) map->ops->map_lookup_percpu_elem (map, key, cpu); }
Signature · 1 line
2.47.3
Quoted 5 lines · Andrii Nakryiko
We have many places which open-code what's now is
bpf_rcu_lock_held()macro, so replace all those places with a clean and short macro invocation. For that, movebpf_rcu_lock_held()macro intoinclude/linux/bpf.h.Signed-off-by Andrii Nakryiko <[email protected]>
jirka
Quoted 161 lines
---
v1->v2:
- movebpf_rcu_lock_held()outside of #ifdefCONFIG_BPF_SYSCALLarea (kernel test robot).include/linux/bpf.h | 3 +++ include/linux/bpf_local_storage.h | 3 --- kernel/bpf/hashtab.c | 21 +++++++-------------- kernel/bpf/helpers.c | 12 ++++-------- 4 files changed, 14 insertions(+), 25 deletions(-)include/linux/bpf.h +3
-0@@ -2381,6 +2381,9 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,bool bpf_jit_bypass_spec_v1 (void );bool bpf_jit_bypass_spec_v4 (void ); +#define bpf_rcu_lock_held () \ + (rcu_read_lock_held() || rcu_read_lock_trace_held() || rcu_read_lock_bh_held()) +#ifdef CONFIG_BPF_SYSCALL DECLARE_PER_CPU (int, bpf_prog_active);extern struct mutex bpf_stats_enabled_mutex;include/linux/bpf_local_storage.h +0
-3@@ -18,9 +18,6 @@#define BPF_LOCAL_STORAGE_CACHE_SIZE 16 -#define bpf_rcu_lock_held () \ - (rcu_read_lock_held() || rcu_read_lock_trace_held() || \ - rcu_read_lock_bh_held())struct bpf_local_storage_map_bucket {struct hlist_head list;raw_spinlock_t lock;kernel/bpf/hashtab.c +7
-14@@ -657,8 +657,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)struct htab_elem *l;u32 hash, key_size; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1086,8 +1085,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1194,8 +1192,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1263,8 +1260,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1326,8 +1322,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,/* unknown flags */ return -EINVAL ; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1404,8 +1399,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)u32 hash, key_size;int ret; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;@@ -1440,8 +1434,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)u32 hash, key_size;int ret; -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ()); key_size = map->key_size;kernel/bpf/helpers.c +4
-8@@ -42,8 +42,7 @@*/ BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return (unsigned long ) map->ops->map_lookup_elem (map, key); }@@ -59,8 +58,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, void *, value, u64, flags) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return map->ops->map_update_elem (map, key, value, flags); }@@ -77,8 +75,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return map->ops->map_delete_elem (map, key); }@@ -134,8 +131,7 @@ const struct bpf_func_proto bpf_map_peek_elem_proto = { BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu) { -WARN_ON_ONCE (!rcu_read_lock_held () && !rcu_read_lock_trace_held () && - !rcu_read_lock_bh_held ()); +WARN_ON_ONCE (!bpf_rcu_lock_held ());return (unsigned long ) map->ops->map_lookup_percpu_elem (map, key, cpu); }Signature · 1 line
2.47.3
This patch was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Hello:
This patch was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Quoted 11 lines · On Tue, 14 Oct 2025 13:14:03 -0700 you wrote:
We have many places which open-code what's now is
bpf_rcu_lock_held()macro, so replace all those places with a clean and short macro invocation. For that, movebpf_rcu_lock_held()macro intoinclude/linux/bpf.h.Signed-off-by Andrii Nakryiko <[email protected]>---
v1->v2:
- movebpf_rcu_lock_held()outside of #ifdefCONFIG_BPF_SYSCALLarea (kernel test robot).[...]
Here is the summary with links:
- [v2,bpf-next] bpf: consistently use bpf_rcu_lock_held() everywhere
https://git.kernel.org/bpf/bpf-next/c/48a97ffc6c82
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html