bpf: per-cpu cgroup local storage
Subsystems (7)
v4 cover bpf: per-cpu cgroup local storage
This patchset implements per-cpu cgroup local storage and provides an example how per-cpu and shared cgroup local storage can be used for efficient accounting of network traffic.
v4->v3:
1) incorporated Alexei's feedback
v3->v2:
1) incorporated Song's feedback
2) rebased on top of current bpf-next
v2->v1:
1) added a selftest implementing network counters
2) added a missing free() in cgroup local storage selftest
Roman Gushchin (10):
bpf: extend cgroup bpf core to allow multiple cgroup storage types
bpf: rework cgroup storage pointer passing
bpf: introduce per-cpu cgroup local storage
bpf: don't allow create maps of per-cpu cgroup local storages
bpf: sync include/uapi/linux/bpf.h to tools/include/uapi/linux/bpf.h
bpftool: add support for PERCPU_CGROUP_STORAGE maps selftests/bpf: add verifier per-cpu cgroup storage tests selftests/bpf: extend the storage test to test per-cpu cgroup storage samples/bpf: extend test_cgrp2_attach2 test to use per-cpu cgroup
storage
selftests/bpf: cgroup local storage-based network counters
include/linux/bpf-cgroup.h | 55 ++++-- include/linux/bpf.h | 12 +- include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/cgroup.c | 74 +++++--- kernel/bpf/helpers.c | 25 ++- kernel/bpf/local_storage.c | 169 +++++++++++++++--- kernel/bpf/map_in_map.c | 3 +- kernel/bpf/syscall.c | 20 ++- kernel/bpf/verifier.c | 23 ++- net/bpf/test_run.c | 20 ++- samples/bpf/test_cgrp2_attach2.c | 19 +- tools/bpf/bpftool/map.c | 4 +- tools/include/uapi/linux/bpf.h | 1 + tools/testing/selftests/bpf/Makefile | 6 +- tools/testing/selftests/bpf/netcnt_common.h | 24 +++ tools/testing/selftests/bpf/netcnt_prog.c | 71 ++++++++ .../selftests/bpf/test_cgroup_storage.c | 60 ++++++- tools/testing/selftests/bpf/test_netcnt.c | 158 ++++++++++++++++ tools/testing/selftests/bpf/test_verifier.c | 139 +++++++++++++- 20 files changed, 786 insertions(+), 99 deletions(-) create mode 100644 tools/testing/selftests/bpf/netcnt_common.h create mode 100644 tools/testing/selftests/bpf/netcnt_prog.c create mode 100644 tools/testing/selftests/bpf/test_netcnt.c
Signature · 1 line
2.17.1
PATCH v4 1/10 bpf: extend cgroup bpf core to allow multiple cgroup storage types
In order to introduce per-cpu cgroup storage, let's generalize bpf cgroup core to support multiple cgroup storage types. Potentially, per-node cgroup storage can be added later.
This commit is mostly a formal change that replaces cgroup_storage pointer with a array of cgroup_storage pointers. It doesn't actually introduce a new storage type,
it will be done later.
Each bpf program is now able to have one cgroup storage of each type.
--- include/linux/bpf-cgroup.h | 38 ++++++++++++++------ include/linux/bpf.h | 11 ++++-- kernel/bpf/cgroup.c | 74 ++++++++++++++++++++++++++------------ kernel/bpf/helpers.c | 15 ++++---- kernel/bpf/local_storage.c | 18 ++++++---- kernel/bpf/syscall.c | 9 +++-- kernel/bpf/verifier.c | 8 +++-- net/bpf/test_run.c | 20 +++++++---- 8 files changed, 136 insertions(+), 57 deletions(-)
include/linux/bpf-cgroup.h +28 -10
@@ -2,6 +2,7 @@#ifndef _BPF_CGROUP_H#define _BPF_CGROUP_H +#include <linux/bpf.h> #include <linux/errno.h> #include <linux/jump_label.h> #include <linux/percpu.h>
@@ -22,7 +23,10 @@ struct bpf_cgroup_storage;extern struct static_key_false cgroup_bpf_enabled_key;#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) -DECLARE_PER_CPU(void*, bpf_cgroup_storage); +DECLARE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); + +#define for_each_cgroup_storage_type (stype) \ + for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)struct bpf_cgroup_storage_map ;
@@ -43,7 +47,7 @@ struct bpf_cgroup_storage {struct bpf_prog_list {struct list_head node;struct bpf_prog *prog; -struct bpf_cgroup_storage *storage; +struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE ]; };struct bpf_prog_array ;
@@ -101,18 +105,29 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,int __cgroup_bpf_check_dev_permission (short dev_type,u32 major,u32 minor,short access,enum bpf_attach_type type); -static inline void bpf_cgroup_storage_set (struct bpf_cgroup_storage *storage) +static inline enum bpf_cgroup_storage_type cgroup_storage_type ( +struct bpf_map *map) { +return BPF_CGROUP_STORAGE_SHARED ; +} + +static inline void bpf_cgroup_storage_set (struct bpf_cgroup_storage + *storage[MAX_BPF_CGROUP_STORAGE_TYPE ]) +{ +enum bpf_cgroup_storage_type stype;struct bpf_storage_buffer *buf; -if (!storage) -return ; +for_each_cgroup_storage_type (stype) { +if (!storage[stype]) +continue ; - buf =READ_ONCE (storage->buf); -this_cpu_write (bpf_cgroup_storage, &buf->data[0 ]); + buf =READ_ONCE (storage[stype]->buf); +this_cpu_write (bpf_cgroup_storage[stype], &buf->data[0 ]); + } } -struct bpf_cgroup_storage *bpf_cgroup_storage_alloc (struct bpf_prog *prog); +struct bpf_cgroup_storage *bpf_cgroup_storage_alloc (struct bpf_prog *prog, +enum bpf_cgroup_storage_type stype);void bpf_cgroup_storage_free (struct bpf_cgroup_storage *storage); void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, struct cgroup *cgroup,
@@ -265,13 +280,14 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,return -EINVAL ; } -static inline void bpf_cgroup_storage_set (struct bpf_cgroup_storage *storage) {} +static inline void bpf_cgroup_storage_set ( +struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE ]) {}static inline int bpf_cgroup_storage_assign (struct bpf_prog *prog,struct bpf_map *map) {return 0 ; }static inline void bpf_cgroup_storage_release (struct bpf_prog *prog,struct bpf_map *map) {}static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc ( -struct bpf_prog *prog) {return 0 ; } +struct bpf_prog *prog,enum bpf_cgroup_storage_type stype) {return 0 ; }static inline void bpf_cgroup_storage_free (struct bpf_cgroup_storage *storage) {}
@@ -293,6 +309,8 @@ static inline void bpf_cgroup_storage_free(#define BPF_CGROUP_RUN_PROG_SOCK_OPS (sock_ops) ({ 0; })#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP (type,major,minor,access) ({ 0; }) +#define for_each_cgroup_storage_type (stype) for (; false; ) +#endif /* CONFIG_CGROUP_BPF */ #endif /* _BPF_CGROUP_H */
include/linux/bpf.h +9 -2
@@ -272,6 +272,13 @@ struct bpf_prog_offload {u32 jited_len; }; +enum bpf_cgroup_storage_type { +BPF_CGROUP_STORAGE_SHARED , + __BPF_CGROUP_STORAGE_MAX +}; + +#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX +struct bpf_prog_aux {atomic_t refcnt;u32 used_map_cnt;
@@ -289,7 +296,7 @@ struct bpf_prog_aux {struct bpf_prog *prog;struct user_struct *user;u64 load_time;/* ns since boottime */ -struct bpf_map *cgroup_storage; +struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ];char name[BPF_OBJ_NAME_LEN ];#ifdef CONFIG_SECURITY void *security;
@@ -358,7 +365,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,*/ struct bpf_prog_array_item {struct bpf_prog *prog; -struct bpf_cgroup_storage *cgroup_storage; +struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]; };struct bpf_prog_array {
kernel/bpf/cgroup.c +52 -22
167 lines not on this page · raw message
kernel/bpf/helpers.c +9 -6
@@ -194,16 +194,18 @@ const struct bpf_func_proto bpf_get_current_cgroup_id_proto = { .ret_type = RET_INTEGER, }; -DECLARE_PER_CPU(void*, bpf_cgroup_storage); +#ifdef CONFIG_CGROUP_BPF +DECLARE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) { -/* map and flags arguments are not used now, -* but provide an ability to extend the API -* for other types of local storages. -* verifier checks that their values are correct. +/* flags argument is not used now, +* but provides an ability to extend the API. +* verifier checks that its value is correct. */ -return (unsigned long )this_cpu_read (bpf_cgroup_storage); +enum bpf_cgroup_storage_type stype =cgroup_storage_type (map); + +return (unsigned long )this_cpu_read (bpf_cgroup_storage[stype]); } const struct bpf_func_proto bpf_get_local_storage_proto = {
@@ -214,3 +216,4 @@ const struct bpf_func_proto bpf_get_local_storage_proto = { .arg2_type = ARG_ANYTHING, };#endif +#endif
kernel/bpf/local_storage.c +11 -7
@@ -7,7 +7,7 @@#include <linux/rbtree.h> #include <linux/slab.h> -DEFINE_PER_CPU(void*, bpf_cgroup_storage); +DEFINE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);#ifdef CONFIG_CGROUP_BPF
@@ -251,6 +251,7 @@ const struct bpf_map_ops cgroup_storage_map_ops = {int bpf_cgroup_storage_assign (struct bpf_prog *prog,struct bpf_map *_map) { +enum bpf_cgroup_storage_type stype =cgroup_storage_type (_map);struct bpf_cgroup_storage_map *map =map_to_storage (_map);int ret = -EBUSY ;
@@ -258,11 +259,12 @@ int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)if (map->prog && map->prog != prog) goto unlock; -if (prog->aux->cgroup_storage && prog->aux->cgroup_storage != _map) +if (prog->aux->cgroup_storage[stype] && + prog->aux->cgroup_storage[stype] != _map) goto unlock; map->prog = prog; - prog->aux->cgroup_storage = _map; + prog->aux->cgroup_storage[stype] = _map; ret =0 ; unlock:spin_unlock_bh (&map->lock);
@@ -272,24 +274,26 @@ int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)void bpf_cgroup_storage_release (struct bpf_prog *prog,struct bpf_map *_map) { +enum bpf_cgroup_storage_type stype =cgroup_storage_type (_map);struct bpf_cgroup_storage_map *map =map_to_storage (_map);spin_lock_bh (&map->lock);if (map->prog == prog) { -WARN_ON (prog->aux->cgroup_storage != _map); +WARN_ON (prog->aux->cgroup_storage[stype] != _map); map->prog =NULL ; - prog->aux->cgroup_storage =NULL ; + prog->aux->cgroup_storage[stype] =NULL ; }spin_unlock_bh (&map->lock); } -struct bpf_cgroup_storage *bpf_cgroup_storage_alloc (struct bpf_prog *prog) +struct bpf_cgroup_storage *bpf_cgroup_storage_alloc (struct bpf_prog *prog, +enum bpf_cgroup_storage_type stype) {struct bpf_cgroup_storage *storage;struct bpf_map *map;u32 pages; - map = prog->aux->cgroup_storage; + map = prog->aux->cgroup_storage[stype];if (!map)return NULL ;
kernel/bpf/syscall.c +7 -2
@@ -988,10 +988,15 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)/* drop refcnt on maps used by eBPF program and free auxilary data */ static void free_used_maps (struct bpf_prog_aux *aux) { +enum bpf_cgroup_storage_type stype;int i; -if (aux->cgroup_storage) -bpf_cgroup_storage_release (aux->prog, aux->cgroup_storage); +for_each_cgroup_storage_type (stype) { +if (!aux->cgroup_storage[stype]) +continue ; +bpf_cgroup_storage_release (aux->prog, + aux->cgroup_storage[stype]); + }for (i =0 ; i < aux->used_map_cnt; i++)bpf_map_put (aux->used_maps[i]);
kernel/bpf/verifier.c +6 -2
@@ -5171,11 +5171,15 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)/* drop refcnt of maps used by the rejected program */ static void release_maps (struct bpf_verifier_env *env) { +enum bpf_cgroup_storage_type stype;int i; -if (env->prog->aux->cgroup_storage) +for_each_cgroup_storage_type (stype) { +if (!env->prog->aux->cgroup_storage[stype]) +continue ;bpf_cgroup_storage_release (env->prog, - env->prog->aux->cgroup_storage); + env->prog->aux->cgroup_storage[stype]); + }for (i =0 ; i < env->used_map_cnt; i++)bpf_map_put (env->used_maps[i]);
net/bpf/test_run.c +14 -6
@@ -12,7 +12,7 @@#include <linux/sched/signal.h> static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx, -struct bpf_cgroup_storage *storage) +struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE ]) {u32 ret;
@@ -28,13 +28,20 @@ static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx,static u32 bpf_test_run (struct bpf_prog *prog,void *ctx,u32 repeat,u32 *time) { -struct bpf_cgroup_storage *storage =NULL ; +struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE ] = {0 }; +enum bpf_cgroup_storage_type stype;u64 time_start, time_spent =0 ;u32 ret =0 , i; - storage =bpf_cgroup_storage_alloc (prog); -if (IS_ERR (storage)) -return PTR_ERR (storage); +for_each_cgroup_storage_type (stype) { + storage[stype] =bpf_cgroup_storage_alloc (prog, stype); +if (IS_ERR (storage[stype])) { + storage[stype] =NULL ; +for_each_cgroup_storage_type (stype) +bpf_cgroup_storage_free (storage[stype]); +return -ENOMEM ; + } + }if (!repeat) repeat =1 ;
@@ -53,7 +60,8 @@ static u32 bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *time)do_div (time_spent, repeat); *time = time_spent >U32_MAX ?U32_MAX : (u32 )time_spent; -bpf_cgroup_storage_free (storage); +for_each_cgroup_storage_type (stype) +bpf_cgroup_storage_free (storage[stype]);return ret; }
Signature · 1 line
2.17.1
PATCH v4 2/10 bpf: rework cgroup storage pointer passing
In reply to Roman GushchinTo simplify the following introduction of per-cpu cgroup storage, let's rework a bit a mechanism of passing a pointer to a cgroup storage into the bpf_get_local_storage(). Let's save a pointer to the corresponding bpf_cgroup_storage structure, instead of a pointer to the actual buffer.
It will help us to handle per-cpu storage later, which has a different way of accessing to the actual data.
--- include/linux/bpf-cgroup.h | 13 ++++--------- kernel/bpf/helpers.c | 8 ++++++-- kernel/bpf/local_storage.c | 3 ++- 3 files changed, 12 insertions(+), 12 deletions(-)
include/linux/bpf-cgroup.h +4 -9
@@ -23,7 +23,8 @@ struct bpf_cgroup_storage;extern struct static_key_false cgroup_bpf_enabled_key;#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) -DECLARE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); +DECLARE_PER_CPU(struct bpf_cgroup_storage*, + bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]);#define for_each_cgroup_storage_type (stype) \ for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
@@ -115,15 +116,9 @@ static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {enum bpf_cgroup_storage_type stype; -struct bpf_storage_buffer *buf; - -for_each_cgroup_storage_type (stype) { -if (!storage[stype]) -continue ; - buf =READ_ONCE (storage[stype]->buf); -this_cpu_write (bpf_cgroup_storage[stype], &buf->data[0 ]); - } +for_each_cgroup_storage_type (stype) +this_cpu_write (bpf_cgroup_storage[stype], storage[stype]); } struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
kernel/bpf/helpers.c +6 -2
@@ -195,7 +195,8 @@ const struct bpf_func_proto bpf_get_current_cgroup_id_proto = { };#ifdef CONFIG_CGROUP_BPF -DECLARE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); +DECLARE_PER_CPU(struct bpf_cgroup_storage*, + bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]); BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) {
@@ -204,8 +205,11 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)* verifier checks that its value is correct. */ enum bpf_cgroup_storage_type stype =cgroup_storage_type (map); +struct bpf_cgroup_storage *storage; -return (unsigned long )this_cpu_read (bpf_cgroup_storage[stype]); + storage =this_cpu_read (bpf_cgroup_storage[stype]); + +return (unsigned long )&READ_ONCE (storage->buf)->data[0 ]; }const struct bpf_func_proto bpf_get_local_storage_proto = {
kernel/bpf/local_storage.c +2 -1
@@ -7,7 +7,8 @@#include <linux/rbtree.h> #include <linux/slab.h> -DEFINE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); +DEFINE_PER_CPU(struct bpf_cgroup_storage*, + bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE ]);#ifdef CONFIG_CGROUP_BPF
Signature · 1 line
2.17.1
PATCH v4 3/10 bpf: introduce per-cpu cgroup local storage
In reply to Roman GushchinThis commit introduced per-cpu cgroup local storage.
Per-cpu cgroup local storage is very similar to simple cgroup storage (let's call it shared), except all the data is per-cpu.
The main goal of per-cpu variant is to implement super fast counters (e.g. packet counters), which don't require neither lookups, neither atomic operations.
Quoted 1 line
From userspace's point of view, accessing a per-cpu cgroup storage
is similar to other per-cpu map types (e.g. per-cpu hashmaps and arrays).
Writing to a per-cpu cgroup storage is not atomic, but is performed by copying longs, so some minimal atomicity is here, exactly as with other per-cpu maps.
--- include/linux/bpf-cgroup.h | 20 ++++- include/linux/bpf.h | 1 + include/linux/bpf_types.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/helpers.c | 8 +- kernel/bpf/local_storage.c | 150 ++++++++++++++++++++++++++++++++----- kernel/bpf/syscall.c | 11 ++- kernel/bpf/verifier.c | 15 +++- 8 files changed, 179 insertions(+), 28 deletions(-)
include/linux/bpf-cgroup.h +19 -1
@@ -37,7 +37,10 @@ struct bpf_storage_buffer { };struct bpf_cgroup_storage { -struct bpf_storage_buffer *buf; +union { +struct bpf_storage_buffer *buf; +void __percpu *percpu_buf; + };struct bpf_cgroup_storage_map *map;struct bpf_cgroup_storage_key key;struct list_head list;
@@ -109,6 +112,9 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,static inline enum bpf_cgroup_storage_type cgroup_storage_type (struct bpf_map *map) { +if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) +return BPF_CGROUP_STORAGE_PERCPU ; +return BPF_CGROUP_STORAGE_SHARED ; }
@@ -131,6 +137,10 @@ void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);int bpf_cgroup_storage_assign (struct bpf_prog *prog,struct bpf_map *map);void bpf_cgroup_storage_release (struct bpf_prog *prog,struct bpf_map *map); +int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_cgroup_storage_update (struct bpf_map *map,void *key, +void *value,u64 flags); +/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */ #define BPF_CGROUP_RUN_PROG_INET_INGRESS (sk, skb) \ ({ \
@@ -285,6 +295,14 @@ static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return 0; }static inline void bpf_cgroup_storage_free (struct bpf_cgroup_storage *storage) {} +static inline int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key, +void *value) { +return 0 ; +} +static inline int bpf_percpu_cgroup_storage_update (struct bpf_map *map, +void *key,void *value,u64 flags) { +return 0 ; +}#define cgroup_bpf_enabled (0)#define BPF_CGROUP_PRE_CONNECT_ENABLED (sk) (0)
include/linux/bpf.h +1 -0
@@ -274,6 +274,7 @@ struct bpf_prog_offload {enum bpf_cgroup_storage_type {BPF_CGROUP_STORAGE_SHARED , +BPF_CGROUP_STORAGE_PERCPU , __BPF_CGROUP_STORAGE_MAX };
include/linux/bpf_types.h +1 -0
@@ -43,6 +43,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_ARRAY, cgroup_array_map_ops)#endif #ifdef CONFIG_CGROUP_BPF BPF_MAP_TYPE(BPF_MAP_TYPE_CGROUP_STORAGE, cgroup_storage_map_ops) +BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, cgroup_storage_map_ops)#endif BPF_MAP_TYPE(BPF_MAP_TYPE_HASH, htab_map_ops)BPF_MAP_TYPE (BPF_MAP_TYPE_PERCPU_HASH , htab_percpu_map_ops)
include/uapi/linux/bpf.h +1 -0
@@ -127,6 +127,7 @@ enum bpf_map_type {BPF_MAP_TYPE_SOCKHASH ,BPF_MAP_TYPE_CGROUP_STORAGE ,BPF_MAP_TYPE_REUSEPORT_SOCKARRAY , + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, }; enum bpf_prog_type {
kernel/bpf/helpers.c +7 -1
@@ -206,10 +206,16 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)*/ enum bpf_cgroup_storage_type stype =cgroup_storage_type (map);struct bpf_cgroup_storage *storage; +void *ptr; storage =this_cpu_read (bpf_cgroup_storage[stype]); -return (unsigned long )&READ_ONCE (storage->buf)->data[0 ]; +if (stype ==BPF_CGROUP_STORAGE_SHARED ) + ptr = &READ_ONCE (storage->buf)->data[0 ]; +else + ptr =this_cpu_ptr (storage->percpu_buf); + +return (unsigned long )ptr; }const struct bpf_func_proto bpf_get_local_storage_proto = {
kernel/bpf/local_storage.c +130 -20
196 lines not on this page · raw message
kernel/bpf/syscall.c +9 -2
@@ -686,7 +686,8 @@ static int map_lookup_elem(union bpf_attr *attr)if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) + map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || + map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) value_size =round_up (map->value_size,8 ) *num_possible_cpus ();else if (IS_FD_MAP (map)) value_size =sizeof (u32);
@@ -705,6 +706,8 @@ static int map_lookup_elem(union bpf_attr *attr) err =bpf_percpu_hash_copy (map, key, value); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) { err =bpf_percpu_array_copy (map, key, value); + }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) { + err =bpf_percpu_cgroup_storage_copy (map, key, value); }else if (map->map_type ==BPF_MAP_TYPE_STACK_TRACE ) { err =bpf_stackmap_copy (map, key, value); }else if (IS_FD_ARRAY (map)) {
@@ -774,7 +777,8 @@ static int map_update_elem(union bpf_attr *attr)if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) + map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || + map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) value_size =round_up (map->value_size,8 ) *num_possible_cpus ();else value_size = map->value_size;
@@ -809,6 +813,9 @@ static int map_update_elem(union bpf_attr *attr) err =bpf_percpu_hash_update (map, key, value, attr->flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) { err =bpf_percpu_array_update (map, key, value, attr->flags); + }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) { + err =bpf_percpu_cgroup_storage_update (map, key, value, + attr->flags); }else if (IS_FD_ARRAY (map)) {rcu_read_lock (); err = bpf_fd_array_map_update_elem(map, f.file, key, value,
kernel/bpf/verifier.c +11 -4
@@ -2074,6 +2074,7 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, goto error;break ;case BPF_MAP_TYPE_CGROUP_STORAGE : +case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE :if (func_id != BPF_FUNC_get_local_storage) goto error;break ;
@@ -2164,7 +2165,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, goto error;break ;case BPF_FUNC_get_local_storage: -if (map->map_type !=BPF_MAP_TYPE_CGROUP_STORAGE ) +if (map->map_type !=BPF_MAP_TYPE_CGROUP_STORAGE && + map->map_type !=BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) goto error;break ;case BPF_FUNC_sk_select_reuseport:
@@ -5049,6 +5051,12 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,return 0 ; } +static bool bpf_map_is_cgroup_storage (struct bpf_map *map) +{ +return (map->map_type ==BPF_MAP_TYPE_CGROUP_STORAGE || + map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ); +} +/* look for pseudo eBPF instructions that access map FDs and * replace them with actual map pointers */
@@ -5139,10 +5147,9 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) } env->used_maps[env->used_map_cnt++] = map; -if (map->map_type ==BPF_MAP_TYPE_CGROUP_STORAGE && +if (bpf_map_is_cgroup_storage (map) &&bpf_cgroup_storage_assign (env->prog, map)) { -verbose (env, -"only one cgroup storage is allowed\n" ); +verbose (env,"only one cgroup storage of each type is allowed\n" );fdput (f);return -EBUSY ; }
Signature · 1 line
2.17.1
PATCH v4 4/10 bpf: don't allow create maps of per-cpu cgroup local storages
In reply to Roman GushchinExplicitly forbid creating map of per-cpu cgroup local storages. This behavior matches the behavior of shared cgroup storages.
--- kernel/bpf/map_in_map.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/bpf/map_in_map.c +2 -1
@@ -24,7 +24,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)* in the verifier is not enough. */ if (inner_map->map_type ==BPF_MAP_TYPE_PROG_ARRAY || - inner_map->map_type ==BPF_MAP_TYPE_CGROUP_STORAGE ) { + inner_map->map_type ==BPF_MAP_TYPE_CGROUP_STORAGE || + inner_map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) {fdput (f);return ERR_PTR (-ENOTSUPP ); }
Signature · 1 line
2.17.1
PATCH v4 5/10 bpf: sync include/uapi/linux/bpf.h to tools/include/uapi/linux/bpf.h
In reply to Roman GushchinThe sync is required due to the appearance of a new map type:BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, which implements per-cpu cgroup local storage.
--- tools/include/uapi/linux/bpf.h | 1 + 1 file changed, 1 insertion(+)
tools/include/uapi/linux/bpf.h +1 -0
@@ -127,6 +127,7 @@ enum bpf_map_type {BPF_MAP_TYPE_SOCKHASH ,BPF_MAP_TYPE_CGROUP_STORAGE ,BPF_MAP_TYPE_REUSEPORT_SOCKARRAY , + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, }; enum bpf_prog_type {
Signature · 1 line
2.17.1
PATCH v4 6/10 bpftool: add support for PERCPU_CGROUP_STORAGE maps
In reply to Roman GushchinThis commit adds support for BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE map type.
--- tools/bpf/bpftool/map.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
tools/bpf/bpftool/map.c +3 -1
@@ -72,13 +72,15 @@ static const char * const map_type_name[] = { [BPF_MAP_TYPE_SOCKHASH] = "sockhash", [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", + [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", };static bool map_is_per_cpu (__u32 type) {return type ==BPF_MAP_TYPE_PERCPU_HASH || type ==BPF_MAP_TYPE_PERCPU_ARRAY || - type ==BPF_MAP_TYPE_LRU_PERCPU_HASH ; + type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || + type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ; }static bool map_is_map_of_maps (__u32 type)
Signature · 1 line
2.17.1
PATCH v4 7/10 selftests/bpf: add verifier per-cpu cgroup storage tests
In reply to Roman GushchinThis commits adds verifier tests covering per-cpu cgroup storage functionality. There are 6 new tests, which are exactly the same as for shared cgroup storage, but do use per-cpu cgroup storage map.
Expected output:
$ ./test_verifier #0/u add+sub+mul OK #0/p add+sub+mul OK ... #286/p invalid cgroup storage access 6 OK #287/p valid per-cpu cgroup storage access OK #288/p invalid per-cpu cgroup storage access 1 OK #289/p invalid per-cpu cgroup storage access 2 OK #290/p invalid per-cpu cgroup storage access 3 OK #291/p invalid per-cpu cgroup storage access 4 OK #292/p invalid per-cpu cgroup storage access 5 OK #293/p invalid per-cpu cgroup storage access 6 OK #294/p multiple registers share map_lookup_elem result OK ... #662/p mov64 src == dst OK #663/p mov64 src != dst OK Summary: 914 PASSED, 0 SKIPPED, 0 FAILED
--- tools/testing/selftests/bpf/test_verifier.c | 139 +++++++++++++++++++- 1 file changed, 133 insertions(+), 6 deletions(-)
tools/testing/selftests/bpf/test_verifier.c +133 -6
@@ -68,6 +68,7 @@ struct bpf_test {int fixup_prog2[MAX_FIXUPS ];int fixup_map_in_map[MAX_FIXUPS ];int fixup_cgroup_storage[MAX_FIXUPS ]; +int fixup_percpu_cgroup_storage[MAX_FIXUPS ];const char *errstr;const char *errstr_unpriv;uint32_t retval;
@@ -4676,7 +4677,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { - "invalid per-cgroup storage access 3", + "invalid cgroup storage access 3", .insns = {BPF_MOV64_IMM (BPF_REG_2 ,0 ), BPF_LD_MAP_FD(BPF_REG_1, 0),
@@ -4743,6 +4744,121 @@ static struct bpf_test tests[] = { .errstr = "get_local_storage() doesn't support non-zero flags", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, + { + "valid per-cpu cgroup storage access", + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,0 ), +BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ), +BPF_ALU64_IMM (BPF_AND ,BPF_REG_0 ,1 ), + BPF_EXIT_INSN(), + }, + .fixup_percpu_cgroup_storage = { 1 }, + .result = ACCEPT, + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 1" , + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,0 ), +BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ), +BPF_ALU64_IMM (BPF_AND ,BPF_REG_0 ,1 ), + BPF_EXIT_INSN(), + }, + .fixup_map1 = { 1 }, + .result = REJECT, + .errstr = "cannot pass map_type 1 into func bpf_get_local_storage", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 2" , + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_LD_MAP_FD (BPF_REG_1 ,1 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_ALU64_IMM (BPF_AND ,BPF_REG_0 ,1 ), + BPF_EXIT_INSN(), + }, + .result =REJECT , + .errstr = "fd 1 is not pointing to valid bpf_map", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 3" , + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,256 ), +BPF_ALU64_IMM (BPF_ADD ,BPF_REG_1 ,1 ), +BPF_MOV64_IMM (BPF_REG_0 ,0 ), + BPF_EXIT_INSN(), + }, + .fixup_percpu_cgroup_storage = { 1 }, + .result = REJECT, + .errstr = "invalid access to map value, value_size=64 off=256 size=4", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 4" , + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,-2 ), +BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ), +BPF_ALU64_IMM (BPF_ADD ,BPF_REG_1 ,1 ), + BPF_EXIT_INSN(), + }, + .fixup_cgroup_storage = { 1 }, + .result = REJECT, + .errstr = "invalid access to map value, value_size=64 off=-2 size=4", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 5" , + .insns = { +BPF_MOV64_IMM (BPF_REG_2 ,7 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,0 ), +BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ), +BPF_ALU64_IMM (BPF_AND ,BPF_REG_0 ,1 ), + BPF_EXIT_INSN(), + }, + .fixup_percpu_cgroup_storage = { 1 }, + .result = REJECT, + .errstr = "get_local_storage() doesn't support non-zero flags", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { +"invalid per-cpu cgroup storage access 6" , + .insns = { +BPF_MOV64_REG (BPF_REG_2 ,BPF_REG_1 ), +BPF_LD_MAP_FD (BPF_REG_1 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , + BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_1 ,BPF_REG_0 ,0 ), +BPF_MOV64_REG (BPF_REG_0 ,BPF_REG_1 ), +BPF_ALU64_IMM (BPF_AND ,BPF_REG_0 ,1 ), + BPF_EXIT_INSN(), + }, + .fixup_percpu_cgroup_storage = { 1 }, + .result = REJECT, + .errstr = "get_local_storage() doesn't support non-zero flags", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, {"multiple registers share map_lookup_elem result" , .insns = {
@@ -12615,15 +12731,17 @@ static int create_map_in_map(void)return outer_map_fd; } -static int create_cgroup_storage (void ) +static int create_cgroup_storage (bool percpu) { +enum bpf_map_type type = percpu ?BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE : +BPF_MAP_TYPE_CGROUP_STORAGE ;int fd; - fd =bpf_create_map (BPF_MAP_TYPE_CGROUP_STORAGE , -sizeof (struct bpf_cgroup_storage_key ), + fd =bpf_create_map (type,sizeof (struct bpf_cgroup_storage_key ),TEST_DATA_LEN ,0 ,0 );if (fd <0 ) -printf ("Failed to create array '%s'!\n" ,strerror (errno)); +printf ("Failed to create cgroup storage '%s'!\n" , +strerror (errno));return fd; }
@@ -12641,6 +12759,7 @@ static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog,int *fixup_prog2 = test->fixup_prog2;int *fixup_map_in_map = test->fixup_map_in_map;int *fixup_cgroup_storage = test->fixup_cgroup_storage; +int *fixup_percpu_cgroup_storage = test->fixup_percpu_cgroup_storage;if (test->fill_helper) test->fill_helper (test);
@@ -12710,12 +12829,20 @@ static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog, }if (*fixup_cgroup_storage) { - map_fds[7 ] =create_cgroup_storage (); + map_fds[7 ] =create_cgroup_storage (false);do { prog[*fixup_cgroup_storage].imm = map_fds[7 ]; fixup_cgroup_storage++; }while (*fixup_cgroup_storage); } + +if (*fixup_percpu_cgroup_storage) { + map_fds[8 ] =create_cgroup_storage (true); +do { + prog[*fixup_percpu_cgroup_storage].imm = map_fds[8 ]; + fixup_percpu_cgroup_storage++; + }while (*fixup_percpu_cgroup_storage); + } } static void do_test_single(struct bpf_test *test, bool unpriv,
Signature · 1 line
2.17.1
PATCH v4 8/10 selftests/bpf: extend the storage test to test per-cpu cgroup storage
In reply to Roman GushchinThis test extends the cgroup storage test to use per-cpu flavor of the cgroup storage as well.
The test initializes a per-cpu cgroup storage to some non-zero initial value (1000), and then simple bumps a per-cpu counter each time the shared counter is atomically incremented. Then it reads all per-cpu areas from the userspace side, and checks that the sum of values adds to the expected sum.
Expected output:
$ ./test_cgroup_storage test_cgroup_storage:PASS
--- .../selftests/bpf/test_cgroup_storage.c | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-)
tools/testing/selftests/bpf/test_cgroup_storage.c +57 -3
@@ -4,6 +4,7 @@#include <linux/filter.h> #include <stdio.h> #include <stdlib.h> +#include <sys/sysinfo.h> #include "bpf_rlimit.h" #include "cgroup_helpers.h"
@@ -15,6 +16,14 @@ char bpf_log_buf[BPF_LOG_BUF_SIZE]; int main(int argc, char **argv) { struct bpf_insn prog[] = { + BPF_LD_MAP_FD(BPF_REG_1, 0), /* percpu map fd */ + BPF_MOV64_IMM(BPF_REG_2, 0), /* flags, not used */ + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, + BPF_FUNC_get_local_storage), + BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_0, 0), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 0x1), + BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_3, 0), + BPF_LD_MAP_FD(BPF_REG_1, 0), /* map fd */ BPF_MOV64_IMM(BPF_REG_2, 0), /* flags, not used */ BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
@@ -28,9 +37,18 @@ int main(int argc, char **argv) };size_t insns_cnt =sizeof (prog) /sizeof (struct bpf_insn );int error =EXIT_FAILURE ; -int map_fd, prog_fd, cgroup_fd; +int map_fd, percpu_map_fd, prog_fd, cgroup_fd;struct bpf_cgroup_storage_key key;unsigned long long value; +unsigned long long *percpu_value; +int cpu, nproc; + + nproc =get_nprocs_conf (); + percpu_value =malloc (sizeof (*percpu_value) * nproc); +if (!percpu_value) { +printf ("Not enough memory for per-cpu area (%d cpus)\n" , nproc); + goto err; + } map_fd =bpf_create_map (BPF_MAP_TYPE_CGROUP_STORAGE ,sizeof (key),sizeof (value),0 ,0 );
@@ -39,7 +57,15 @@ int main(int argc, char **argv) goto out; } - prog[0 ].imm = map_fd; + percpu_map_fd =bpf_create_map (BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE , +sizeof (key),sizeof (value),0 ,0 ); +if (percpu_map_fd <0 ) { +printf ("Failed to create map: %s\n" ,strerror (errno)); + goto out; + } + + prog[0 ].imm = percpu_map_fd; + prog[7 ].imm = map_fd; prog_fd =bpf_load_program (BPF_PROG_TYPE_CGROUP_SKB , prog, insns_cnt,"GPL" ,0 , bpf_log_buf,BPF_LOG_BUF_SIZE );
@@ -77,7 +103,15 @@ int main(int argc, char **argv) }if (bpf_map_lookup_elem (map_fd, &key, &value)) { -printf ("Failed to lookup cgroup storage\n" ); +printf ("Failed to lookup cgroup storage 0\n" ); + goto err; + } + +for (cpu =0 ; cpu < nproc; cpu++) + percpu_value[cpu] =1000 ; + +if (bpf_map_update_elem (percpu_map_fd, &key, percpu_value,0 )) { +printf ("Failed to update the data in the cgroup storage\n" ); goto err; }
@@ -120,11 +154,31 @@ int main(int argc, char **argv) goto err; } +/* Check the final value of the counter in the percpu local storage */ + +for (cpu =0 ; cpu < nproc; cpu++) + percpu_value[cpu] =0 ; + +if (bpf_map_lookup_elem (percpu_map_fd, &key, percpu_value)) { +printf ("Failed to lookup the per-cpu cgroup storage\n" ); + goto err; + } + + value =0 ; +for (cpu =0 ; cpu < nproc; cpu++) + value += percpu_value[cpu]; + +if (value != nproc *1000 +6 ) { +printf ("Unexpected data in the per-cpu cgroup storage\n" ); + goto err; + } + error =0 ;printf ("test_cgroup_storage:PASS\n" ); err:cleanup_cgroup_environment (); +free (percpu_value); out:return error;
Signature · 1 line
2.17.1
PATCH v4 9/10 samples/bpf: extend test_cgrp2_attach2 test to use per-cpu cgroup storage
In reply to Roman GushchinThis commit extends the test_cgrp2_attach2 test to cover per-cpu cgroup storage. Bpf program will use shared and per-cpu cgroup storages simultaneously, so a better coverage of corresponding core code will be achieved.
Expected output:
$ ./test_cgrp2_attach2 Attached DROP prog. This ping in cgroup /foo should fail... ping: sendmsg: Operation not permitted Attached DROP prog. This ping in cgroup /foo/bar should fail... ping: sendmsg: Operation not permitted Attached PASS prog. This ping in cgroup /foo/bar should pass... Detached PASS from /foo/bar while DROP is attached to /foo. This ping in cgroup /foo/bar should fail... ping: sendmsg: Operation not permitted Attached PASS from /foo/bar and detached DROP from /foo. This ping in cgroup /foo/bar should pass... ### override:PASS ### multi:PASS
--- samples/bpf/test_cgrp2_attach2.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-)
samples/bpf/test_cgrp2_attach2.c +18 -1
@@ -209,7 +209,7 @@ static int map_fd = -1;static int prog_load_cnt (int verdict,int val) { -int cgroup_storage_fd; +int cgroup_storage_fd, percpu_cgroup_storage_fd;if (map_fd <0 ) map_fd =bpf_create_map (BPF_MAP_TYPE_ARRAY ,4 ,8 ,1 ,0 );
@@ -225,6 +225,14 @@ static int prog_load_cnt(int verdict, int val)return -1 ; } + percpu_cgroup_storage_fd =bpf_create_map ( +BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE , +sizeof (struct bpf_cgroup_storage_key ),8 ,0 ,0 ); +if (percpu_cgroup_storage_fd <0 ) { +printf ("failed to create map '%s'\n" ,strerror (errno)); +return -1 ; + } +struct bpf_insn prog[] = {BPF_MOV32_IMM (BPF_REG_0 ,0 ),BPF_STX_MEM (BPF_W ,BPF_REG_10 ,BPF_REG_0 ,-4 ),/* *(u32 *)(fp - 4) = r0 */
@@ -235,11 +243,20 @@ static int prog_load_cnt(int verdict, int val)BPF_JMP_IMM (BPF_JEQ ,BPF_REG_0 ,0 ,2 ),BPF_MOV64_IMM (BPF_REG_1 , val),/* r1 = 1 */ BPF_RAW_INSN (BPF_STX |BPF_XADD |BPF_DW ,BPF_REG_0 ,BPF_REG_1 ,0 ,0 ),/* xadd r0 += r1 */ +BPF_LD_MAP_FD (BPF_REG_1 , cgroup_storage_fd),BPF_MOV64_IMM (BPF_REG_2 ,0 ),BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , BPF_FUNC_get_local_storage),BPF_MOV64_IMM (BPF_REG_1 , val),BPF_RAW_INSN (BPF_STX |BPF_XADD |BPF_W ,BPF_REG_0 ,BPF_REG_1 ,0 ,0 ), + +BPF_LD_MAP_FD (BPF_REG_1 , percpu_cgroup_storage_fd), +BPF_MOV64_IMM (BPF_REG_2 ,0 ), +BPF_RAW_INSN (BPF_JMP |BPF_CALL ,0 ,0 ,0 , BPF_FUNC_get_local_storage), +BPF_LDX_MEM (BPF_W ,BPF_REG_3 ,BPF_REG_0 ,0 ), +BPF_ALU64_IMM (BPF_ADD ,BPF_REG_3 ,0x1 ), +BPF_STX_MEM (BPF_W ,BPF_REG_0 ,BPF_REG_3 ,0 ), +BPF_MOV64_IMM (BPF_REG_0 , verdict),/* r0 = verdict */ BPF_EXIT_INSN(), };
Signature · 1 line
2.17.1
PATCH v4 10/10 selftests/bpf: cgroup local storage-based network counters
In reply to Roman GushchinThis commit adds a bpf kselftest, which demonstrates how percpu and shared cgroup local storage can be used for efficient lookup-free network accounting.
Cgroup local storage provides generic memory area with a very efficient lookup free access. To avoid expensive atomic operations for each packet, per-cpu cgroup local storage is used. Each packet is initially charged to a per-cpu counter, and only if the counter reaches certain value (32 in this case), the charge is moved into the global atomic counter. This allows to amortize atomic operations, keeping reasonable accuracy.
The test also implements a naive network traffic throttling, mostly to demonstrate the possibility of bpf cgroup--based network bandwidth control.
Expected output:
./test_netcnt
test_netcnt:PASS
--- tools/testing/selftests/bpf/Makefile | 6 +- tools/testing/selftests/bpf/netcnt_common.h | 24 +++ tools/testing/selftests/bpf/netcnt_prog.c | 71 +++++++++ tools/testing/selftests/bpf/test_netcnt.c | 158 ++++++++++++++++++++ 4 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/netcnt_common.h create mode 100644 tools/testing/selftests/bpf/netcnt_prog.c create mode 100644 tools/testing/selftests/bpf/test_netcnt.c
tools/testing/selftests/bpf/Makefile +4 -2
@@ -23,7 +23,8 @@ $(TEST_CUSTOM_PROGS): $(OUTPUT)/%: %.c TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \ test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \ test_sock test_btf test_sockmap test_lirc_mode2_user get_cgroup_id_user \ - test_socket_cookie test_cgroup_storage test_select_reuseport test_section_names + test_socket_cookie test_cgroup_storage test_select_reuseport test_section_names \ + test_netcnt TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \ test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \
@@ -35,7 +36,7 @@ TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test test_get_stack_rawtp.o test_sockmap_kern.o test_sockhash_kern.o \ test_lwt_seg6local.o sendmsg4_prog.o sendmsg6_prog.o test_lirc_mode2_kern.o \ get_cgroup_id_kern.o socket_cookie_prog.o test_select_reuseport_kern.o \ - test_skb_cgroup_id_kern.o bpf_flow.o + test_skb_cgroup_id_kern.o bpf_flow.o netcnt_prog.o # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \
@@ -72,6 +73,7 @@ $(OUTPUT)/test_tcpbpf_user: cgroup_helpers.c $(OUTPUT)/test_progs: trace_helpers.c $(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c $(OUTPUT)/test_cgroup_storage: cgroup_helpers.c +$(OUTPUT)/test_netcnt: cgroup_helpers.c .PHONY: force
tools/testing/selftests/bpf/netcnt_common.h new file +24 -0
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __NETCNT_COMMON_H +#define __NETCNT_COMMON_H + +#include <linux/types.h> + +#define MAX_PERCPU_PACKETS 32 + +struct percpu_net_cnt { +__u64 packets; +__u64 bytes; + +__u64 prev_ts; + +__u64 prev_packets; +__u64 prev_bytes; +}; + +struct net_cnt { +__u64 packets; +__u64 bytes; +}; + +#endif
tools/testing/selftests/bpf/netcnt_prog.c new file +71 -0
71 lines not on this page · raw message
tools/testing/selftests/bpf/test_netcnt.c new file +158 -0
158 lines not on this page · raw message
Signature · 1 line
2.17.1
Quoted 7 lines · Roman Gushchin
This patchset implements per-cpu cgroup local storage and provides an example how per-cpu and shared cgroup local storage can be used for efficient accounting of network traffic.
v4->v3:
1) incorporated Alexei's feedback
Quoted 7 lines
v3->v2:
1) incorporated Song's feedback
2) rebased on top of current bpf-next
v2->v1:
1) added a selftest implementing network counters
2) added a missingfree()in cgroup local storage selftest
Applied to bpf-next, thanks!