Skip to contentSkip to conversation

bpf: consistently use bpf_rcu_lock_held() everywhere

PATCH Superseded Andrii Nakryiko 2 participants 2 messages Oct 10, 2025 – Oct 11, 2025

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Andrii Nakryiko

author maintainer
#0 lore raw

PATCH 1/1 bpf: consistently use bpf_rcu_lock_held() everywhere

We have many places which open-code what's now is bpf_rcu_lock_held() macro, so replace all those places with a clean and short macro invocation. For that, move bpf_rcu_lock_held() macro into include/linux/bpf.h.

---
 include/linux/bpf.h               |  3 +++
 include/linux/bpf_local_storage.h |  3 ---
 kernel/bpf/hashtab.c              | 21 +++++++--------------
 kernel/bpf/helpers.c              | 12 ++++--------
 4 files changed, 14 insertions(+), 25 deletions(-)
include/linux/bpf.h +3 -0
index a98c83346134..0933b3ffe74e 100644
@@ -2417,6 +2417,9 @@ extern const struct bpf_prog_ops bpf_offload_prog_ops;
 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
 extern const struct bpf_verifier_ops xdp_analyzer_ops;
 
+#define bpf_rcu_lock_held() \
+	(rcu_read_lock_held() || rcu_read_lock_trace_held() || rcu_read_lock_bh_held())
+
 struct bpf_prog *bpf_prog_get(u32 ufd);
 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
 				       bool attach_drv);
include/linux/bpf_local_storage.h +0 -3
index ab7244d8108f..782f58feea35 100644
@@ -18,9 +18,6 @@
 
 #define BPF_LOCAL_STORAGE_CACHE_SIZE	16
 
-#define bpf_rcu_lock_held()                                                    \
-	(rcu_read_lock_held() || rcu_read_lock_trace_held() ||                 \
-	 rcu_read_lock_bh_held())
 struct bpf_local_storage_map_bucket {
 	struct hlist_head list;
 	raw_spinlock_t lock;
kernel/bpf/hashtab.c +7 -14
index c2fcd0cd51e5..2f07aaf4d732 100644
@@ -669,8 +669,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 	struct htab_elem *l;
 	u32 hash, key_size;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1098,8 +1097,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 		/* unknown flags */
 		return -EINVAL;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1206,8 +1204,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value
 		/* unknown flags */
 		return -EINVAL;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1275,8 +1272,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
 		/* unknown flags */
 		return -EINVAL;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1338,8 +1334,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 		/* unknown flags */
 		return -EINVAL;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1416,8 +1411,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key)
 	u32 hash, key_size;
 	int ret;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
@@ -1452,8 +1446,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)
 	u32 hash, key_size;
 	int ret;
 
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
 	key_size = map->key_size;
 
kernel/bpf/helpers.c +4 -8
index c9fab9a356df..500e141f6d94 100644
@@ -42,8 +42,7 @@
  */
 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
 {
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	return (unsigned long) map->ops->map_lookup_elem(map, key);
 }
 
@@ -59,8 +58,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = {
 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
 	   void *, value, u64, flags)
 {
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	return map->ops->map_update_elem(map, key, value, flags);
 }
 
@@ -77,8 +75,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = {
 
 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
 {
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	return map->ops->map_delete_elem(map, key);
 }
 
@@ -134,8 +131,7 @@ const struct bpf_func_proto bpf_map_peek_elem_proto = {
 
 BPF_CALL_3(bpf_map_lookup_percpu_elem, struct bpf_map *, map, void *, key, u32, cpu)
 {
-	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
-		     !rcu_read_lock_bh_held());
+	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	return (unsigned long) map->ops->map_lookup_percpu_elem(map, key, cpu);
 }
 
Signature · 1 line
2.47.3
#1 lore raw
kernel test robot noticed the following build errors:

Hi Andrii,

kernel test robot noticed the following build errors:

[auto build test ERROR on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Andrii-Nakryiko/bpf-consistently-use-bpf_rcu_lock_held-everywhere/20251011-013430
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20251010173021.3952776-1-andrii%40kernel.org
patch subject: [PATCH bpf-next] bpf: consistently use bpf_rcu_lock_held() everywhere
config: i386-buildonly-randconfig-001-20251011 (https://download.01.org/0day-ci/archive/20251011/[email protected]/config)
compiler: gcc-13 (Debian 13.3.0-16) 13.3.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251011/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from include/linux/workqueue.h:17,

                    from include/linux/bpf.h:11,
                    from include/linux/bpf_verifier.h:7,
                    from net/core/filter.c:21:
   include/linux/bpf_local_storage.h: In function 'bpf_local_storage_lookup':
>> include/linux/bpf_local_storage.h:149:39: error: implicit declaration of function 'bpf_rcu_lock_held'; did you mean 'rcu_read_lock_held'? [-Werror=implicit-function-declaration]
     149 |                                       bpf_rcu_lock_held());
         |                                       ^~~~~~~~~~~~~~~~~
   include/linux/rcupdate.h:399:53: note: in definition of macro 'RCU_LOCKDEP_WARN'
     399 |                 if (debug_lockdep_rcu_enabled() && (c) &&               \
         |                                                     ^
   include/linux/rcupdate.h:680:9: note: in expansion of macro '__rcu_dereference_check'
     680 |         __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
         |         ^~~~~~~~~~~~~~~~~~~~~~~
   include/linux/bpf_local_storage.h:148:17: note: in expansion of macro 'rcu_dereference_check'
     148 |         sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
         |                 ^~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors

vim +149 include/linux/bpf_local_storage.h

450af8d0f6be2e KP Singh         2020-08-25  129  
c83597fa5dc6b3 Yonghong Song    2022-10-25  130  struct bpf_map *
c83597fa5dc6b3 Yonghong Song    2022-10-25  131  bpf_local_storage_map_alloc(union bpf_attr *attr,
08a7ce384e33e5 Martin KaFai Lau 2023-03-22  132  			    struct bpf_local_storage_cache *cache,
08a7ce384e33e5 Martin KaFai Lau 2023-03-22  133  			    bool bpf_ma);
450af8d0f6be2e KP Singh         2020-08-25  134  
68bc61c26cacf1 Marco Elver      2024-02-07  135  void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,
68bc61c26cacf1 Marco Elver      2024-02-07  136  				      struct bpf_local_storage_map *smap,
68bc61c26cacf1 Marco Elver      2024-02-07  137  				      struct bpf_local_storage_elem *selem);
68bc61c26cacf1 Marco Elver      2024-02-07  138  /* If cacheit_lockit is false, this lookup function is lockless */
68bc61c26cacf1 Marco Elver      2024-02-07  139  static inline struct bpf_local_storage_data *
450af8d0f6be2e KP Singh         2020-08-25  140  bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
450af8d0f6be2e KP Singh         2020-08-25  141  			 struct bpf_local_storage_map *smap,
68bc61c26cacf1 Marco Elver      2024-02-07  142  			 bool cacheit_lockit)
68bc61c26cacf1 Marco Elver      2024-02-07  143  {
68bc61c26cacf1 Marco Elver      2024-02-07  144  	struct bpf_local_storage_data *sdata;
68bc61c26cacf1 Marco Elver      2024-02-07  145  	struct bpf_local_storage_elem *selem;
68bc61c26cacf1 Marco Elver      2024-02-07  146  
68bc61c26cacf1 Marco Elver      2024-02-07  147  	/* Fast path (cache hit) */
68bc61c26cacf1 Marco Elver      2024-02-07  148  	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
68bc61c26cacf1 Marco Elver      2024-02-07 @149  				      bpf_rcu_lock_held());
68bc61c26cacf1 Marco Elver      2024-02-07  150  	if (sdata && rcu_access_pointer(sdata->smap) == smap)
68bc61c26cacf1 Marco Elver      2024-02-07  151  		return sdata;
68bc61c26cacf1 Marco Elver      2024-02-07  152  
68bc61c26cacf1 Marco Elver      2024-02-07  153  	/* Slow path (cache miss) */
68bc61c26cacf1 Marco Elver      2024-02-07  154  	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
68bc61c26cacf1 Marco Elver      2024-02-07  155  				  rcu_read_lock_trace_held())
68bc61c26cacf1 Marco Elver      2024-02-07  156  		if (rcu_access_pointer(SDATA(selem)->smap) == smap)
68bc61c26cacf1 Marco Elver      2024-02-07  157  			break;
68bc61c26cacf1 Marco Elver      2024-02-07  158  
68bc61c26cacf1 Marco Elver      2024-02-07  159  	if (!selem)
68bc61c26cacf1 Marco Elver      2024-02-07  160  		return NULL;
68bc61c26cacf1 Marco Elver      2024-02-07  161  	if (cacheit_lockit)
68bc61c26cacf1 Marco Elver      2024-02-07  162  		__bpf_local_storage_insert_cache(local_storage, smap, selem);
68bc61c26cacf1 Marco Elver      2024-02-07  163  	return SDATA(selem);
68bc61c26cacf1 Marco Elver      2024-02-07  164  }
450af8d0f6be2e KP Singh         2020-08-25  165  
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki