bpf: remove __rcu annotations from bpf_prog_array
Subsystems (9)
PATCH v4 1/4 bpf: remove __rcu annotations from bpf_prog_array
Drop __rcu annotations and rcu read sections from bpf_prog_array helper functions. They are not needed since all existing callers call those helpers from the rcu update side while holding a mutex. This guarantees that use-after-free could not happen.
In the next patches I'll fix the callers with missing rcu_dereference_protected to make sparse/lockdep happy, the proper way to use these helpers is:
struct bpf_prog_array __rcu *progs = ...; struct bpf_prog_array *p; mutex_lock(&mtx); p = rcu_dereference_protected(progs, lockdep_is_held(&mtx)); bpf_prog_array_length(p); bpf_prog_array_copy_to_user(p, ...); bpf_prog_array_delete_safe(p, ...); bpf_prog_array_copy_info(p, ...); bpf_prog_array_copy(p, ...); bpf_prog_array_free(p); mutex_unlock(&mtx);
No functional changes! rcu_dereference_protected with lockdep_is_held should catch any cases where we update prog array without a mutex (I've looked at existing call sites and I think we hold a mutex everywhere).
Motivation is to fix sparse warnings:kernel/bpf/core.c:1803:9: warning: incorrect type in argument 1 (different address spaces)kernel/bpf/core.c:1803:9: expected struct callback_head *headkernel/bpf/core.c:1803:9: got struct callback_head [noderef] <asn:4> *kernel/bpf/core.c:1877:44: warning: incorrect type in initializer (different address spaces)kernel/bpf/core.c:1877:44: expected struct bpf_prog_array_item *itemkernel/bpf/core.c:1877:44: got struct bpf_prog_array_item [noderef] <asn:4> *kernel/bpf/core.c:1901:26: warning: incorrect type in assignment (different address spaces)kernel/bpf/core.c:1901:26: expected struct bpf_prog_array_item *existingkernel/bpf/core.c:1901:26: got struct bpf_prog_array_item [noderef] <asn:4> *kernel/bpf/core.c:1935:26: warning: incorrect type in assignment (different address spaces)kernel/bpf/core.c:1935:26: expected struct bpf_prog_array_item *[assigned] existingkernel/bpf/core.c:1935:26: got struct bpf_prog_array_item [noderef] <asn:4> *
v2:
* remove comment about potential race; that can't happen because all callers are in rcu-update section
--- include/linux/bpf.h | 12 ++++++------ kernel/bpf/core.c | 37 +++++++++++++------------------------ 2 files changed, 19 insertions(+), 30 deletions(-)
include/linux/bpf.h +6 -6
@@ -514,17 +514,17 @@ struct bpf_prog_array { };struct bpf_prog_array *bpf_prog_array_alloc (u32 prog_cnt,gfp_t flags); -void bpf_prog_array_free (struct bpf_prog_array __rcu *progs); -int bpf_prog_array_length (struct bpf_prog_array __rcu *progs); -int bpf_prog_array_copy_to_user (struct bpf_prog_array __rcu *progs, +void bpf_prog_array_free (struct bpf_prog_array *progs); +int bpf_prog_array_length (struct bpf_prog_array *progs); +int bpf_prog_array_copy_to_user (struct bpf_prog_array *progs,__u32 __user *prog_ids,u32 cnt); -void bpf_prog_array_delete_safe (struct bpf_prog_array __rcu *progs, +void bpf_prog_array_delete_safe (struct bpf_prog_array *progs,struct bpf_prog *old_prog); -int bpf_prog_array_copy_info (struct bpf_prog_array __rcu *array, +int bpf_prog_array_copy_info (struct bpf_prog_array *array,u32 *prog_ids,u32 request_cnt,u32 *prog_cnt); -int bpf_prog_array_copy (struct bpf_prog_array __rcu *old_array, +int bpf_prog_array_copy (struct bpf_prog_array *old_array,struct bpf_prog *exclude_prog,struct bpf_prog *include_prog,struct bpf_prog_array **new_array);
kernel/bpf/core.c +13 -24
@@ -1795,38 +1795,33 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)return &empty_prog_array.hdr; } -void bpf_prog_array_free (struct bpf_prog_array __rcu *progs) +void bpf_prog_array_free (struct bpf_prog_array *progs) { -if (!progs || - progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr) +if (!progs || progs == &empty_prog_array.hdr)return ;kfree_rcu (progs, rcu); } -int bpf_prog_array_length (struct bpf_prog_array __rcu *array) +int bpf_prog_array_length (struct bpf_prog_array *array) {struct bpf_prog_array_item *item;u32 cnt =0 ; -rcu_read_lock (); - item =rcu_dereference (array)->items; -for (; item->prog; item++) +for (item = array->items; item->prog; item++)if (item->prog != &dummy_bpf_prog.prog) cnt++; -rcu_read_unlock ();return cnt; } -static bool bpf_prog_array_copy_core (struct bpf_prog_array __rcu *array, +static bool bpf_prog_array_copy_core (struct bpf_prog_array *array,u32 *prog_ids,u32 request_cnt) {struct bpf_prog_array_item *item;int i =0 ; - item =rcu_dereference_check (array,1 )->items; -for (; item->prog; item++) { +for (item = array->items; item->prog; item++) {if (item->prog == &dummy_bpf_prog.prog)continue ; prog_ids[i] = item->prog->aux->id;
@@ -1839,7 +1834,7 @@ static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,return !!(item->prog); } -int bpf_prog_array_copy_to_user (struct bpf_prog_array __rcu *array, +int bpf_prog_array_copy_to_user (struct bpf_prog_array *array,__u32 __user *prog_ids,u32 cnt) {unsigned long err =0 ;
@@ -1850,18 +1845,12 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,* cnt = bpf_prog_array_length(); * if (cnt > 0) * bpf_prog_array_copy_to_user(..., cnt); -* so below kcalloc doesn't need extra cnt > 0 check, but -* bpf_prog_array_length() releases rcu lock and -* prog array could have been swapped with empty or larger array, -* so always copy 'cnt' prog_ids to the user. -* In a rare race the user will see zero prog_ids +* so below kcalloc doesn't need extra cnt > 0 check. */ ids =kcalloc (cnt,sizeof (u32),GFP_USER | __GFP_NOWARN);if (!ids)return -ENOMEM ; -rcu_read_lock (); nospc =bpf_prog_array_copy_core (array, ids, cnt); -rcu_read_unlock (); err =copy_to_user (prog_ids, ids, cnt *sizeof (u32));kfree (ids);if (err)
@@ -1871,19 +1860,19 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,return 0 ; } -void bpf_prog_array_delete_safe (struct bpf_prog_array __rcu *array, +void bpf_prog_array_delete_safe (struct bpf_prog_array *array,struct bpf_prog *old_prog) { -struct bpf_prog_array_item *item = array->items; +struct bpf_prog_array_item *item; -for (; item->prog; item++) +for (item = array->items; item->prog; item++)if (item->prog == old_prog) {WRITE_ONCE (item->prog, &dummy_bpf_prog.prog);break ; } } -int bpf_prog_array_copy (struct bpf_prog_array __rcu *old_array, +int bpf_prog_array_copy (struct bpf_prog_array *old_array,struct bpf_prog *exclude_prog,struct bpf_prog *include_prog,struct bpf_prog_array **new_array)
@@ -1947,7 +1936,7 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,return 0 ; } -int bpf_prog_array_copy_info (struct bpf_prog_array __rcu *array, +int bpf_prog_array_copy_info (struct bpf_prog_array *array,u32 *prog_ids,u32 request_cnt,u32 *prog_cnt) {
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
PATCH v4 2/4 bpf: media: properly use bpf_prog_array api
Now that we don't have __rcu markers on the bpf_prog_array helpers, let's use proper rcu_dereference_protected to obtain array pointer under mutex.
--- drivers/media/rc/bpf-lirc.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-)
drivers/media/rc/bpf-lirc.c +17 -13
@@ -8,6 +8,9 @@#include <linux/bpf_lirc.h> #include "rc-core-priv.h" +#define lirc_rcu_dereference (p) \ + rcu_dereference_protected(p, lockdep_is_held(&ir_raw_handler_lock)) +/* * BPF interface for raw IR */
@@ -136,7 +139,7 @@ const struct bpf_verifier_ops lirc_mode2_verifier_ops = {static int lirc_bpf_attach (struct rc_dev *rcdev,struct bpf_prog *prog) { -struct bpf_prog_array __rcu *old_array; +struct bpf_prog_array *old_array;struct bpf_prog_array *new_array;struct ir_raw_event_ctrl *raw;int ret;
@@ -154,12 +157,12 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog) goto unlock; } -if (raw->progs &&bpf_prog_array_length (raw->progs) >=BPF_MAX_PROGS ) { + old_array =lirc_rcu_dereference (raw->progs); +if (old_array &&bpf_prog_array_length (old_array) >=BPF_MAX_PROGS ) { ret = -E2BIG ; goto unlock; } - old_array = raw->progs; ret =bpf_prog_array_copy (old_array,NULL , prog, &new_array);if (ret <0 ) goto unlock;
@@ -174,7 +177,7 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)static int lirc_bpf_detach (struct rc_dev *rcdev,struct bpf_prog *prog) { -struct bpf_prog_array __rcu *old_array; +struct bpf_prog_array *old_array;struct bpf_prog_array *new_array;struct ir_raw_event_ctrl *raw;int ret;
@@ -192,7 +195,7 @@ static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog) goto unlock; } - old_array = raw->progs; + old_array =lirc_rcu_dereference (raw->progs); ret =bpf_prog_array_copy (old_array, prog,NULL , &new_array);/* * Do not use bpf_prog_array_delete_safe() as we would end up
@@ -223,21 +226,22 @@ void lirc_bpf_run(struct rc_dev *rcdev, u32 sample)/* * This should be called once the rc thread has been stopped, so there can be * no concurrent bpf execution. +* +* Should be called with the ir_raw_handler_lock held. */ void lirc_bpf_free (struct rc_dev *rcdev) {struct bpf_prog_array_item *item; +struct bpf_prog_array *array; -if (!rcdev->raw->progs) + array =lirc_rcu_dereference (rcdev->raw->progs); +if (!array)return ; - item =rcu_dereference (rcdev->raw->progs)->items; -while (item->prog) { +for (item = array->items; item->prog; item++)bpf_prog_put (item->prog); - item++; - } -bpf_prog_array_free (rcdev->raw->progs); +bpf_prog_array_free (array); }int lirc_prog_attach (const union bpf_attr *attr,struct bpf_prog *prog)
@@ -290,7 +294,7 @@ int lirc_prog_detach(const union bpf_attr *attr)int lirc_prog_query (const union bpf_attr *attr,union bpf_attr __user *uattr) {__u32 __user *prog_ids =u64_to_user_ptr (attr->query.prog_ids); -struct bpf_prog_array __rcu *progs; +struct bpf_prog_array *progs;struct rc_dev *rcdev;u32 cnt, flags =0 ;int ret;
@@ -311,7 +315,7 @@ int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)if (ret) goto put; - progs = rcdev->raw->progs; + progs =lirc_rcu_dereference (rcdev->raw->progs); cnt = progs ?bpf_prog_array_length (progs) :0 ;if (copy_to_user (&uattr->query.prog_cnt, &cnt,sizeof (cnt))) {
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
PATCH v4 3/4 bpf: cgroup: properly use bpf_prog_array api
In reply to Stanislav FomichevNow that we don't have __rcu markers on the bpf_prog_array helpers, let's use proper rcu_dereference_protected to obtain array pointer under mutex.
We also don't need __rcu annotations on cgroup_bpf.inactive since it's not read/updated concurrently.
v4:
* drop cgroup_rcu_xyz wrappers and use rcu APIs directly; presumably should be more clear to understand which mutex/refcount protects each particular place
v3:
* amend cgroup_rcu_dereference to include percpu_ref_is_dying; cgroup_bpf is now reference counted and we don't hold cgroup_mutex anymore in cgroup_bpf_release
v2:
* replace xchg with rcu_swap_protected
--- include/linux/bpf-cgroup.h | 2 +- kernel/bpf/cgroup.c | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-)
include/linux/bpf-cgroup.h +1 -1
@@ -72,7 +72,7 @@ struct cgroup_bpf {u32 flags[MAX_BPF_ATTACH_TYPE ];/* temp storage for effective prog array used by prog_attach/detach */ -struct bpf_prog_array __rcu *inactive; +struct bpf_prog_array *inactive;/* reference counter used to detach bpf programs after cgroup removal */ struct percpu_ref refcnt;
kernel/bpf/cgroup.c +17 -11
@@ -38,6 +38,7 @@ static void cgroup_bpf_release(struct work_struct *work) struct cgroup *cgrp = container_of(work, struct cgroup, bpf.release_work);enum bpf_cgroup_storage_type stype; +struct bpf_prog_array *old_array;unsigned int type;for (type =0 ; type <ARRAY_SIZE (cgrp->bpf.progs); type++) {
@@ -54,7 +55,10 @@ static void cgroup_bpf_release(struct work_struct *work)kfree (pl);static_branch_dec (&cgroup_bpf_enabled_key); } -bpf_prog_array_free (cgrp->bpf.effective[type]); + old_array =rcu_dereference_protected ( + cgrp->bpf.effective[type], +percpu_ref_is_dying (&cgrp->bpf.refcnt)); +bpf_prog_array_free (old_array); }percpu_ref_exit (&cgrp->bpf.refcnt);
@@ -126,7 +130,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp,*/ static int compute_effective_progs (struct cgroup *cgrp,enum bpf_attach_type type, -struct bpf_prog_array __rcu **array) +struct bpf_prog_array **array) {enum bpf_cgroup_storage_type stype;struct bpf_prog_array *progs;
@@ -164,17 +168,16 @@ static int compute_effective_progs(struct cgroup *cgrp, } }while ((p =cgroup_parent (p))); -rcu_assign_pointer (*array, progs); + *array = progs;return 0 ; }static void activate_effective_progs (struct cgroup *cgrp,enum bpf_attach_type type, -struct bpf_prog_array __rcu *array) +struct bpf_prog_array *old_array) { -struct bpf_prog_array __rcu *old_array; - - old_array =xchg (&cgrp->bpf.effective[type], array); +rcu_swap_protected (cgrp->bpf.effective[type], old_array, +lockdep_is_held (&cgroup_mutex));/* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */
@@ -191,7 +194,7 @@ int cgroup_bpf_inherit(struct cgroup *cgrp)* that array below is variable length */ #define NR ARRAY_SIZE(cgrp->bpf.effective) -struct bpf_prog_array __rcu *arrays[NR ] = {}; +struct bpf_prog_array *arrays[NR ] = {};int ret, i; ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
@@ -477,10 +480,14 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,enum bpf_attach_type type = attr->query.attach_type;struct list_head *progs = &cgrp->bpf.progs[type];u32 flags = cgrp->bpf.flags[type]; +struct bpf_prog_array *effective;int cnt, ret =0 , i; + effective =rcu_dereference_protected (cgrp->bpf.effective[type], +lockdep_is_held (&cgroup_mutex)); +if (attr->query.query_flags &BPF_F_QUERY_EFFECTIVE ) - cnt =bpf_prog_array_length (cgrp->bpf.effective[type]); + cnt =bpf_prog_array_length (effective);else cnt =prog_list_length (progs);
@@ -497,8 +504,7 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, }if (attr->query.query_flags &BPF_F_QUERY_EFFECTIVE ) { -return bpf_prog_array_copy_to_user (cgrp->bpf.effective[type], - prog_ids, cnt); +return bpf_prog_array_copy_to_user (effective, prog_ids, cnt); }else {struct bpf_prog_list *pl;u32 id;
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
PATCH v4 4/4 bpf: tracing: properly use bpf_prog_array api
In reply to Stanislav FomichevNow that we don't have __rcu markers on the bpf_prog_array helpers, let's use proper rcu_dereference_protected to obtain array pointer under mutex.
--- kernel/trace/bpf_trace.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-)
kernel/trace/bpf_trace.c +10 -8
@@ -19,6 +19,9 @@#include "trace_probe.h" #include "trace.h" +#define bpf_event_rcu_dereference (p) \ + rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) + #ifdef CONFIG_MODULES struct bpf_trace_module { struct module *module;
@@ -1099,7 +1102,7 @@ static DEFINE_MUTEX(bpf_event_mutex);int perf_event_attach_bpf_prog (struct perf_event *event,struct bpf_prog *prog) { -struct bpf_prog_array __rcu *old_array; +struct bpf_prog_array *old_array;struct bpf_prog_array *new_array;int ret = -EEXIST ;
@@ -1117,7 +1120,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event,if (event->prog) goto unlock; - old_array = event->tp_event->prog_array; + old_array =bpf_event_rcu_dereference (event->tp_event->prog_array);if (old_array &&bpf_prog_array_length (old_array) >=BPF_TRACE_MAX_PROGS ) { ret = -E2BIG ;
@@ -1140,7 +1143,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event,void perf_event_detach_bpf_prog (struct perf_event *event) { -struct bpf_prog_array __rcu *old_array; +struct bpf_prog_array *old_array;struct bpf_prog_array *new_array;int ret;
@@ -1149,7 +1152,7 @@ void perf_event_detach_bpf_prog(struct perf_event *event)if (!event->prog) goto unlock; - old_array = event->tp_event->prog_array; + old_array =bpf_event_rcu_dereference (event->tp_event->prog_array); ret =bpf_prog_array_copy (old_array, event->prog,NULL , &new_array);if (ret == -ENOENT ) goto unlock;
@@ -1171,6 +1174,7 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info) {struct perf_event_query_bpf __user *uquery = info;struct perf_event_query_bpf query = {}; +struct bpf_prog_array *progs;u32 *ids, prog_cnt, ids_len;int ret;
@@ -1195,10 +1199,8 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info)*/ mutex_lock (&bpf_event_mutex); - ret =bpf_prog_array_copy_info (event->tp_event->prog_array, - ids, - ids_len, - &prog_cnt); + progs =bpf_event_rcu_dereference (event->tp_event->prog_array); + ret =bpf_prog_array_copy_info (progs, ids, ids_len, &prog_cnt);mutex_unlock (&bpf_event_mutex); if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
Signature · 1 line
2.22.0.rc1.257.g3120a18244-goog
Re: [PATCH bpf-next v4 3/4] bpf: cgroup: properly use bpf_prog_array api
In reply to Stanislav FomichevQuoted 17 lines · Stanislav Fomichev
Now that we don't have
__rcumarkers on thebpf_prog_arrayhelpers, let's use properrcu_dereference_protectedto obtain array pointer under mutex.We also don't need
__rcuannotations on cgroup_bpf.inactive since it's not read/updated concurrently.v4:
* dropcgroup_rcu_xyzwrappers and use rcu APIs directly; presumably should be more clear to understand which mutex/refcount protects each particular placev3:
* amendcgroup_rcu_dereferenceto includepercpu_ref_is_dying;cgroup_bpfis now reference counted and we don't holdcgroup_mutexanymore incgroup_bpf_release
Quoted 5 lines
v2:
* replace xchg withrcu_swap_protectedCc Roman Gushchin <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>
Yeah, the looks even better.
Thanks!
Daniel Borkmann
maintainerQuoted 41 lines · Stanislav Fomichev
Drop
__rcuannotations and rcu read sections frombpf_prog_arrayhelper functions. They are not needed since all existing callers call those helpers from the rcu update side while holding a mutex. This guarantees that use-after-free could not happen.In the next patches I'll fix the callers with missing
rcu_dereference_protectedto make sparse/lockdep happy, the proper way to use these helpers is:struct bpf_prog_array __rcu *progs = ...; struct bpf_prog_array *p; mutex_lock(&mtx); p = rcu_dereference_protected(progs, lockdep_is_held(&mtx)); bpf_prog_array_length(p); bpf_prog_array_copy_to_user(p, ...); bpf_prog_array_delete_safe(p, ...); bpf_prog_array_copy_info(p, ...); bpf_prog_array_copy(p, ...); bpf_prog_array_free(p); mutex_unlock(&mtx);No functional changes!
rcu_dereference_protectedwithlockdep_is_heldshould catch any cases where we update prog array without a mutex (I've looked at existing call sites and I think we hold a mutex everywhere).Motivation is to fix sparse warnings:
kernel/bpf/core.c:1803:9: warning: incorrect type in argument 1 (different address spaces)kernel/bpf/core.c:1803:9: expected structcallback_head*headkernel/bpf/core.c:1803:9: got structcallback_head[noderef] <asn:4> *kernel/bpf/core.c:1877:44: warning: incorrect type in initializer (different address spaces)kernel/bpf/core.c:1877:44: expected structbpf_prog_array_item*itemkernel/bpf/core.c:1877:44: got structbpf_prog_array_item[noderef] <asn:4> *kernel/bpf/core.c:1901:26: warning: incorrect type in assignment (different address spaces)kernel/bpf/core.c:1901:26: expected structbpf_prog_array_item*existingkernel/bpf/core.c:1901:26: got structbpf_prog_array_item[noderef] <asn:4> *kernel/bpf/core.c:1935:26: warning: incorrect type in assignment (different address spaces)kernel/bpf/core.c:1935:26: expected structbpf_prog_array_item*[assigned] existingkernel/bpf/core.c:1935:26: got structbpf_prog_array_item[noderef] <asn:4> *
Quoted 7 lines
v2:
* remove comment about potential race; that can't happen because all callers are in rcu-update sectionCc Roman Gushchin <[email protected]>Acked-by Roman Gushchin <[email protected]>Signed-off-by Stanislav Fomichev <[email protected]>
Series applied, thanks!