bpf: Add cookie object to bpf maps
Daniel Borkmann
maintainerPATCH v2 1/4 bpf: Add cookie object to bpf maps
Add a cookie to BPF maps to uniquely identify BPF maps for the timespan when the node is up. This is different to comparing a pointer or BPF map id which could get rolled over and reused.
--- include/linux/bpf.h | 1 + kernel/bpf/syscall.c | 6 ++++++ 2 files changed, 7 insertions(+)
include/linux/bpf.h +1 -0
@@ -310,6 +310,7 @@ struct bpf_map {bool free_after_rcu_gp;atomic64_t sleepable_refcnt;s64 __percpu *elem_count; +u64 cookie;/* write-once */ };static inline const char *btf_field_type_name (enum btf_field_type type)
kernel/bpf/syscall.c +6 -0
@@ -37,6 +37,7 @@#include <linux/trace_events.h> #include <linux/tracepoint.h> #include <linux/overflow.h> +#include <linux/cookie.h> #include <net/netfilter/nf_bpf_link.h> #include <net/netkit.h>
@@ -53,6 +54,7 @@#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)DEFINE_PER_CPU (int, bpf_prog_active); +DEFINE_COOKIE (bpf_map_cookie);static DEFINE_IDR (prog_idr);static DEFINE_SPINLOCK (prog_idr_lock);static DEFINE_IDR (map_idr);
@@ -1487,6 +1489,10 @@ static int map_create(union bpf_attr *attr, bool kernel)if (err <0 ) goto free_map; +preempt_disable (); + map->cookie =gen_cookie_next (&bpf_map_cookie); +preempt_enable (); +atomic64_set (&map->refcnt,1 );atomic64_set (&map->usercnt,1 );mutex_init (&map->freeze_mutex);
Signature · 1 line
2.43.0
Daniel Borkmann
maintainerPATCH v2 2/4 bpf: Move bpf map owner out of common struct
Given this is only relevant for BPF tail call maps, it is adding up space and penalizing other map types. We also need to extend this with further objects to track / compare to. Therefore, lets move this out into a separate structure and dynamically allocate it only for BPF tail call maps.
--- include/linux/bpf.h | 36 ++++++++++++++++++++++++------------ kernel/bpf/core.c | 35 ++++++++++++++++++----------------- kernel/bpf/syscall.c | 13 +++++++------ 3 files changed, 49 insertions(+), 35 deletions(-)
include/linux/bpf.h +24 -12
@@ -260,6 +260,18 @@ struct bpf_list_node_kern {void *owner; } __attribute__((aligned(8))); +/* 'Ownership' of program-containing map is claimed by the first program +* that is going to use this map or by the first program which FD is +* stored in the map to make sure that all callers and callees have the +* same prog type, JITed flag and xdp_has_frags flag. +*/ +struct bpf_map_owner { +enum bpf_prog_type type; +bool jited; +bool xdp_has_frags; +const struct btf_type *attach_func_proto; +}; +struct bpf_map {const struct bpf_map_ops *ops;struct bpf_map *inner_map_meta;
@@ -292,18 +304,8 @@ struct bpf_map {struct rcu_head rcu; };atomic64_t writecnt; -/* 'Ownership' of program-containing map is claimed by the first program -* that is going to use this map or by the first program which FD is -* stored in the map to make sure that all callers and callees have the -* same prog type, JITed flag and xdp_has_frags flag. -*/ -struct { -const struct btf_type *attach_func_proto; -spinlock_t lock; -enum bpf_prog_type type; -bool jited; -bool xdp_has_frags; - } owner; +spinlock_t owner_lock; +struct bpf_map_owner *owner;bool bypass_spec_v1;bool frozen;/* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp;
@@ -2109,6 +2111,16 @@ static inline bool bpf_map_flags_access_ok(u32 access_flags) (BPF_F_RDONLY_PROG |BPF_F_WRONLY_PROG ); } +static inline struct bpf_map_owner *bpf_map_owner_alloc (struct bpf_map *map) +{ +return kzalloc (sizeof (*map->owner),GFP_ATOMIC ); +} + +static inline void bpf_map_owner_free (struct bpf_map *map) +{ +kfree (map->owner); +} +struct bpf_event_entry {struct perf_event *event;struct file *perf_file;
kernel/bpf/core.c +18 -17
@@ -2377,28 +2377,29 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp) {enum bpf_prog_type prog_type =resolve_prog_type (fp); -bool ret;struct bpf_prog_aux *aux = fp->aux; +bool ret = false;if (fp->kprobe_override) -return false; +return ret; -spin_lock (&map->owner.lock); -if (!map->owner.type) { -/* There's no owner yet where we could check for -* compatibility. -*/ - map->owner.type = prog_type; - map->owner.jited = fp->jited; - map->owner.xdp_has_frags = aux->xdp_has_frags; - map->owner.attach_func_proto = aux->attach_func_proto; +spin_lock (&map->owner_lock); +/* There's no owner yet where we could check for compatibility. */ +if (!map->owner) { + map->owner =bpf_map_owner_alloc (map); +if (!map->owner) + goto err; + map->owner->type = prog_type; + map->owner->jited = fp->jited; + map->owner->xdp_has_frags = aux->xdp_has_frags; + map->owner->attach_func_proto = aux->attach_func_proto; ret = true; }else { - ret = map->owner.type == prog_type && - map->owner.jited == fp->jited && - map->owner.xdp_has_frags == aux->xdp_has_frags; + ret = map->owner->type == prog_type && + map->owner->jited == fp->jited && + map->owner->xdp_has_frags == aux->xdp_has_frags;if (ret && - map->owner.attach_func_proto != aux->attach_func_proto) { + map->owner->attach_func_proto != aux->attach_func_proto) {switch (prog_type) {case BPF_PROG_TYPE_TRACING :case BPF_PROG_TYPE_LSM :
@@ -2411,8 +2412,8 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, } } } -spin_unlock (&map->owner.lock); - +err: +spin_unlock (&map->owner_lock);return ret; }
kernel/bpf/syscall.c +7 -6
@@ -887,6 +887,7 @@ static void bpf_map_free_deferred(struct work_struct *work)security_bpf_map_free (map);bpf_map_release_memcg (map); +bpf_map_owner_free (map);bpf_map_free (map); }
@@ -981,12 +982,12 @@ static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)struct bpf_map *map = filp->private_data;u32 type =0 , jited =0 ; -if (map_type_contains_progs (map)) { -spin_lock (&map->owner.lock); - type = map->owner.type; - jited = map->owner.jited; -spin_unlock (&map->owner.lock); +spin_lock (&map->owner_lock); +if (map->owner) { + type = map->owner->type; + jited = map->owner->jited; } +spin_unlock (&map->owner_lock);seq_printf (m, "map_type:\t%u\n"
@@ -1496,7 +1497,7 @@ static int map_create(union bpf_attr *attr, bool kernel)atomic64_set (&map->refcnt,1 );atomic64_set (&map->usercnt,1 );mutex_init (&map->freeze_mutex); -spin_lock_init (&map->owner.lock); +spin_lock_init (&map->owner_lock); if (attr->btf_key_type_id || attr->btf_value_type_id ||/* Even the map's value is a kernel's struct,
Signature · 1 line
2.43.0
Daniel Borkmann
maintainerPATCH v2 3/4 bpf: Move cgroup iterator helpers to bpf.h
In reply to Daniel BorkmannMove them into bpf.h given we also need them in core code.
---
v1 -> v2:
- Fixed kbuild bot for !CONFIG_CGROUP_BPF. Removing the defione for_each_cgroup_storage_type for this case I left for a future series.
- Removed for_each_cgroup_storage_type_cond
include/linux/bpf-cgroup.h | 5 ----- include/linux/bpf.h | 22 ++++++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-)
include/linux/bpf-cgroup.h +0 -5
@@ -77,9 +77,6 @@ to_cgroup_bpf_attach_type(enum bpf_attach_type attach_type)extern struct static_key_false cgroup_bpf_enabled_key[MAX_CGROUP_BPF_ATTACH_TYPE ];#define cgroup_bpf_enabled (atype) static_branch_unlikely(&cgroup_bpf_enabled_key[atype]) -#define for_each_cgroup_storage_type (stype) \ - for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) -struct bpf_cgroup_storage_map ;struct bpf_storage_buffer {
@@ -510,8 +507,6 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,#define BPF_CGROUP_RUN_PROG_SETSOCKOPT (sock, level, optname, optval, optlen, \ kernel_optval) ({ 0; }) -#define for_each_cgroup_storage_type (stype) for (; false; ) -#endif /* CONFIG_CGROUP_BPF */ #endif /* _BPF_CGROUP_H */
include/linux/bpf.h +14 -8
@@ -208,6 +208,20 @@ enum btf_field_type { BPF_RES_SPIN_LOCK = (1 << 12), }; +enum bpf_cgroup_storage_type { +BPF_CGROUP_STORAGE_SHARED , +BPF_CGROUP_STORAGE_PERCPU , + __BPF_CGROUP_STORAGE_MAX +#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX +}; + +#ifdef CONFIG_CGROUP_BPF +# define for_each_cgroup_storage_type (stype) \ + for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) +#else +# define for_each_cgroup_storage_type (stype) for (; false; ) +#endif /* CONFIG_CGROUP_BPF */ +typedef void (*btf_dtor_kfunc_t )(void *);struct btf_field_kptr {
@@ -1085,14 +1099,6 @@ struct bpf_prog_offload {u32 jited_len; }; -enum bpf_cgroup_storage_type { -BPF_CGROUP_STORAGE_SHARED , -BPF_CGROUP_STORAGE_PERCPU , - __BPF_CGROUP_STORAGE_MAX -}; - -#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX -/* The longest tracepoint has 12 args. * See include/trace/bpf_probe.h */
Signature · 1 line
2.43.0
Daniel Borkmann
maintainerPATCH v2 4/4 bpf: Fix oob access in cgroup local storage
In reply to Daniel BorkmannLonial reported that an out-of-bounds access in cgroup local storage can be crafted via tail calls. Given two programs each utilizing a cgroup local storage with a different value size, and one program doing a tail call into the other. The verifier will validate each of the indivial programs just fine. However, in the runtime context the bpf_cg_run_ctx holds an bpf_prog_array_item which contains the BPF program as well as any cgroup local storage flavor the program uses. Helpers such as bpf_get_local_storage() pick this up from the runtime context:
ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
storage = ctx->prog_item->cgroup_storage[stype];
if (stype == BPF_CGROUP_STORAGE_SHARED)
ptr = &READ_ONCE(storage->buf)->data[0];
else
ptr = this_cpu_ptr(storage->percpu_buf);For the second program which was called from the originally attached one, this means bpf_get_local_storage() will pick up the former program's map, not its own. With mismatching sizes, this can result in an unintended out-of-bounds access.
To fix this issue, we need to extend bpf_map_owner with an array of storage_cookie[] to match on i) the exact maps from the original program if the second program was using bpf_get_local_storage(), or ii) allow the tail call combination if the second program was not using any of the cgroup local storage maps.
--- include/linux/bpf.h | 1 + kernel/bpf/core.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+)
include/linux/bpf.h +1 -0
@@ -283,6 +283,7 @@ struct bpf_map_owner {enum bpf_prog_type type;bool jited;bool xdp_has_frags; +u64 storage_cookie[MAX_BPF_CGROUP_STORAGE_TYPE ];const struct btf_type *attach_func_proto; };
kernel/bpf/core.c +15 -0
@@ -2378,7 +2378,9 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, {enum bpf_prog_type prog_type =resolve_prog_type (fp);struct bpf_prog_aux *aux = fp->aux; +enum bpf_cgroup_storage_type i;bool ret = false; +u64 cookie;if (fp->kprobe_override)return ret;
@@ -2393,11 +2395,24 @@ static bool __bpf_prog_map_compatible(struct bpf_map *map, map->owner->jited = fp->jited; map->owner->xdp_has_frags = aux->xdp_has_frags; map->owner->attach_func_proto = aux->attach_func_proto; +for_each_cgroup_storage_type (i) { + map->owner->storage_cookie[i] = + aux->cgroup_storage[i] ? + aux->cgroup_storage[i]->cookie :0 ; + } ret = true; }else { ret = map->owner->type == prog_type && map->owner->jited == fp->jited && map->owner->xdp_has_frags == aux->xdp_has_frags; +for_each_cgroup_storage_type (i) { +if (!ret) +break ; + cookie = aux->cgroup_storage[i] ? + aux->cgroup_storage[i]->cookie :0 ; + ret = map->owner->storage_cookie[i] == cookie || + !cookie; + }if (ret && map->owner->attach_func_proto != aux->attach_func_proto) {switch (prog_type) {
Signature · 1 line
2.43.0
This series was applied to bpf/bpf.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This series was applied to bpf/bpf.git (master)
by Alexei Starovoitov <[email protected]>:
Quoted 9 lines · On Thu, 31 Jul 2025 01:47:30 +0200 you wrote:
Add a cookie to BPF maps to uniquely identify BPF maps for the timespan when the node is up. This is different to comparing a pointer or BPF map id which could get rolled over and reused.
Signed-off-by Daniel Borkmann <[email protected]>--- include/linux/bpf.h | 1 + kernel/bpf/syscall.c | 6 ++++++ 2 files changed, 7 insertions(+)
Here is the summary with links:
- [bpf,v2,1/4] bpf: Add cookie object to bpf maps
https://git.kernel.org/bpf/bpf/c/12df58ad2942 - [bpf,v2,2/4] bpf: Move bpf map owner out of common struct https://git.kernel.org/bpf/bpf/c/fd1c98f0ef5c - [bpf,v2,3/4] bpf: Move cgroup iterator helpers to bpf.h https://git.kernel.org/bpf/bpf/c/9621e60f59ea - [bpf,v2,4/4] bpf: Fix oob access in cgroup local storage https://git.kernel.org/bpf/bpf/c/abad3d0bad72
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html