bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps
Subsystems (5)
v6 cover bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps
This patch set introduces the BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps, as the requirement of BPF_F_ALL_CPUS flag for percpu_array maps was discussed in the thread of
"[PATCH bpf-next v3 0/4] bpf: Introduce global percpu data"[1].
The goal of BPF_F_ALL_CPUS flag is to reduce data caching overhead in light skeletons by allowing a single value to be reused to update values across all CPUs. This avoids the M:N problem where M cached values are used to update a map on N CPUs kernel.
The BPF_F_CPU flag is accompanied by *flags*-embedded cpu info, which specifies the target CPU for the operation:
* For lookup operations: the flag field alongside cpu info enable querying a value on the specified CPU.
* For update operations: the flag field alongside cpu info enable updating value for specified CPU.
Links:
[1] https://lore.kernel.org/bpf/[email protected]/
Changes:
v5 -> v6:
* Move bpf_map_check_op_flags() from 'bpf.h' to 'syscall.c'.
* Address comments from Alexei:
* Drop the refactoring code of data copying logic for percpu maps.
* Drop bpf_map_check_op_flags() wrappers.
v4 -> v5:
* Address comments from Andrii:
* Refactor data copying logic for all percpu maps.
* Drop this_cpu_ptr() micro-optimization.
* Drop cpu check in libbpf's validate_map_op().
* Enhance bpf_map_check_op_flags() using *allowed flags* instead of
'extra_flags_mask'.
v3 -> v4:
* Address comments from Andrii:
* Remove unnecessary map_type check in bpf_map_value_size().
* Reduce code churn.
* Remove unnecessary do_delete check in
__htab_map_lookup_and_delete_batch().
* Introduce bpf_percpu_copy_to_user() and bpf_percpu_copy_from_user().
* Rename check_map_flags() to bpf_map_check_op_flags() with
extra_flags_mask.
* Add human-readable pr_warn() explanations in validate_map_op().
* Use flags in bpf_map__delete_elem() and
bpf_map__lookup_and_delete_elem(). * Drop "for alignment reasons".
v3 link: https://lore.kernel.org/bpf/[email protected]/
v2 -> v3:
* Address comments from Alexei:
* Use BPF_F_ALL_CPUS instead of BPF_ALL_CPUS magic.
* Introduce these two cpu flags for all percpu maps.
* Address comments from Jiri:
* Reduce some unnecessary u32 cast.
* Refactor more generic map flags check function.
* A code style issue.
v2 link: https://lore.kernel.org/bpf/[email protected]/
v1 -> v2:
* Address comments from Andrii:
* Embed cpu info as high 32 bits of *flags* totally.
* Use ERANGE instead of E2BIG.
* Few format issues.
Leon Hwang (7):
bpf: Introduce internal bpf_map_check_op_flags helper function
bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array
maps
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash
and lru_percpu_hash maps
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for
percpu_cgroup_storage maps
libbpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps selftests/bpf: Add cases to test BPF_F_CPU and BPF_F_ALL_CPUS flags
include/linux/bpf-cgroup.h | 4 +- include/linux/bpf.h | 26 +- include/uapi/linux/bpf.h | 2 + kernel/bpf/arraymap.c | 36 ++- kernel/bpf/hashtab.c | 105 +++++--- kernel/bpf/local_storage.c | 46 +++- kernel/bpf/syscall.c | 93 ++++--- tools/include/uapi/linux/bpf.h | 2 + tools/lib/bpf/bpf.h | 8 + tools/lib/bpf/libbpf.c | 26 +- tools/lib/bpf/libbpf.h | 21 +- .../selftests/bpf/prog_tests/percpu_alloc.c | 233 ++++++++++++++++++ .../selftests/bpf/progs/percpu_alloc_array.c | 32 +++ 13 files changed, 528 insertions(+), 106 deletions(-)
--
2.50.1
PATCH v6 1/7 bpf: Introduce internal bpf_map_check_op_flags helper function
It is to unify map flags checking for lookup_elem, update_elem, lookup_batch and update_batch APIs.
Therefore, it will be convenient to check BPF_F_CPU and BPF_F_ALL_CPUS flags in it for these APIs in next patch.
--- kernel/bpf/syscall.c | 45 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 23 deletions(-)
kernel/bpf/syscall.c +22 -23
@@ -131,6 +131,17 @@ bool bpf_map_write_active(const struct bpf_map *map)return atomic64_read (&map->writecnt) !=0 ; } +static int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) +{ +if (flags & ~allowed_flags) +return -EINVAL ; + +if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK )) +return -EINVAL ; + +return 0 ; +} +static u32 bpf_map_value_size (const struct bpf_map *map) { if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
@@ -1669,9 +1680,6 @@ static int map_lookup_elem(union bpf_attr *attr)if (CHECK_ATTR (BPF_MAP_LOOKUP_ELEM ))return -EINVAL ; -if (attr->flags & ~BPF_F_LOCK ) -return -EINVAL ; -CLASS (fd, f)(attr->map_fd); map =__bpf_map_get (f);if (IS_ERR (map))
@@ -1679,9 +1687,9 @@ static int map_lookup_elem(union bpf_attr *attr)if (!(map_get_sys_perms (map, f) &FMODE_CAN_READ ))return -EPERM ; -if ((attr->flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) -return -EINVAL ; + err =bpf_map_check_op_flags (map, attr->flags,BPF_F_LOCK ); +if (err) +return err; key =__bpf_copy_key (ukey, map->key_size);if (IS_ERR (key))
@@ -1744,11 +1752,9 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) goto err_put; } -if ((attr->flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) { - err = -EINVAL ; + err =bpf_map_check_op_flags (map, attr->flags, ~0 ); +if (err) goto err_put; - } key =___bpf_copy_key (ukey, map->key_size);if (IS_ERR (key)) {
@@ -1952,13 +1958,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,void *key, *value;int err =0 ; -if (attr->batch.elem_flags & ~BPF_F_LOCK ) -return -EINVAL ; - -if ((attr->batch.elem_flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) { -return -EINVAL ; - } + err =bpf_map_check_op_flags (map, attr->batch.elem_flags,BPF_F_LOCK ); +if (err) +return err; value_size =bpf_map_value_size (map);
@@ -2015,12 +2017,9 @@ int generic_map_lookup_batch(struct bpf_map *map,u32 value_size, cp, max_count;int err; -if (attr->batch.elem_flags & ~BPF_F_LOCK ) -return -EINVAL ; - -if ((attr->batch.elem_flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) -return -EINVAL ; + err =bpf_map_check_op_flags (map, attr->batch.elem_flags,BPF_F_LOCK ); +if (err) +return err; value_size =bpf_map_value_size (map);
Signature · 1 line
2.50.1
PATCH v6 2/7 bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
In reply to Leon HwangIntroduce BPF_F_CPU and BPF_F_ALL_CPUS flags and check them for following APIs:
* 'map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'
And, get the correct value size for these APIs.
--- include/linux/bpf.h | 22 ++++++++++++++++++ include/uapi/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 42 ++++++++++++++++++++++------------ tools/include/uapi/linux/bpf.h | 2 ++ 4 files changed, 54 insertions(+), 14 deletions(-)
include/linux/bpf.h +22 -0
@@ -3709,4 +3709,26 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline int bpf_map_check_cpu_flags (u64 flags,bool check_all_cpus_flag) +{ +const u64 cpu_flags =BPF_F_CPU |BPF_F_ALL_CPUS ; +u32 cpu; + +if (check_all_cpus_flag) { +if (unlikely ((u32 )flags >BPF_F_ALL_CPUS )) +return -EINVAL ; +if (unlikely ((flags & cpu_flags) == cpu_flags)) +return -EINVAL ; + }else { +if (unlikely ((u32 )flags & ~BPF_F_CPU )) +return -EINVAL ; + } + + cpu = flags >>32 ; +if (unlikely ((flags &BPF_F_CPU ) && cpu >=num_possible_cpus ())) +return -ERANGE ; + +return 0 ; +} +#endif /* _LINUX_BPF_H */
include/uapi/linux/bpf.h +2 -0
@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */
kernel/bpf/syscall.c +28 -14
101 lines not on this page · raw message
tools/include/uapi/linux/bpf.h +2 -0
@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */
Signature · 1 line
2.50.1
Re: [PATCH bpf-next v6 2/7] bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
Quoted 24 lines · Leon Hwang
Introduce
BPF_F_CPUandBPF_F_ALL_CPUSflags and check them for following APIs:* '
map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'And, get the correct value size for these APIs.
Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 22 ++++++++++++++++++ include/uapi/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 42 ++++++++++++++++++++++------------ tools/include/uapi/linux/bpf.h | 2 ++ 4 files changed, 54 insertions(+), 14 deletions(-)include/linux/bpf.h +0
-0
include/linux/bpf.h int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * · line 3713
Quoted 5 lines
include/linux/bpf.h +2
-0@@ -3709,4 +3709,26 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline int bpf_map_check_cpu_flags (u64 flags,bool check_all_cpus_flag) +{
This function is not used in this patch. Don't add it without users.
Also I really don't like 'bool' arguments.
They make callsite hard to read.
Instead of bool usebpf_map_check_flags(u64 flags, u64 allowed_flags)
so the callsites will look like:bpf_map_check_flags(flags, BPF_F_CPU);
andbpf_map_check_flags(flags, BPF_F_CPU | BPF_F_ALL_CPUS);
Also two functions that do very similar things look redundant. This bpf_map_check_flags() vs bpf_map_check_op_flags()... I think one should do it.
pw-bot: cr
Quoted 162 lines
include/linux/bpf.h +20
-0+const u64 cpu_flags =BPF_F_CPU |BPF_F_ALL_CPUS ; +u32 cpu; + +if (check_all_cpus_flag) { +if (unlikely ((u32 )flags >BPF_F_ALL_CPUS )) +return -EINVAL ; +if (unlikely ((flags & cpu_flags) == cpu_flags)) +return -EINVAL ; + }else { +if (unlikely ((u32 )flags & ~BPF_F_CPU )) +return -EINVAL ; + } + + cpu = flags >>32 ; +if (unlikely ((flags &BPF_F_CPU ) && cpu >=num_possible_cpus ())) +return -ERANGE ; + + return 0; +} +#endif /* _LINUX_BPF_H */ include/uapi/linux/bpf.h +2
-0@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */ kernel/bpf/syscall.c +28
-14@@ -131,23 +131,36 @@ bool bpf_map_write_active(const struct bpf_map *map)return atomic64_read (&map->writecnt) !=0 ; } +static bool bpf_map_supports_cpu_flags (enum bpf_map_type map_type) +{ +return false; +} +static int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) { -if (flags & ~allowed_flags) +if ((u32 )flags & ~allowed_flags)return -EINVAL ;if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK ))return -EINVAL ; +if (!(flags &BPF_F_CPU ) && flags >>32 ) +return -EINVAL ; + +if ((flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) && !bpf_map_supports_cpu_flags (map->map_type)) +return -EINVAL ; +return 0 ; } -static u32 bpf_map_value_size (const struct bpf_map *map) +static u32 bpf_map_value_size (const struct bpf_map *map,u64 flags) { -if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || - map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) +if (flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) +return round_up (map->value_size,8 ); +else if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || + map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || + map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE )return round_up (map->value_size,8 ) *num_possible_cpus ();else if (IS_FD_MAP (map))return sizeof (u32);@@ -1687,7 +1700,7 @@ static int map_lookup_elem(union bpf_attr *attr)if (!(map_get_sys_perms (map, f) &FMODE_CAN_READ ))return -EPERM ; - err =bpf_map_check_op_flags (map, attr->flags,BPF_F_LOCK ); + err =bpf_map_check_op_flags (map, attr->flags,BPF_F_LOCK |BPF_F_CPU );if (err)return err;@@ -1695,7 +1708,7 @@ static int map_lookup_elem(union bpf_attr *attr)if (IS_ERR (key))return PTR_ERR (key); - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->flags); err = -ENOMEM ; value =kvmalloc (value_size,GFP_USER | __GFP_NOWARN);@@ -1762,7 +1775,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) goto err_put; } - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->flags); value =kvmemdup_bpfptr (uvalue, value_size);if (IS_ERR (value)) { err =PTR_ERR (value);@@ -1958,11 +1971,12 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,void *key, *value;int err =0 ; - err =bpf_map_check_op_flags (map, attr->batch.elem_flags,BPF_F_LOCK ); + err =bpf_map_check_op_flags (map, attr->batch.elem_flags, +BPF_F_LOCK |BPF_F_CPU |BPF_F_ALL_CPUS );if (err)return err; - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->batch.elem_flags); max_count = attr->batch.count;if (!max_count)@@ -2017,11 +2031,11 @@ int generic_map_lookup_batch(struct bpf_map *map,u32 value_size, cp, max_count;int err; - err =bpf_map_check_op_flags (map, attr->batch.elem_flags,BPF_F_LOCK ); + err =bpf_map_check_op_flags (map, attr->batch.elem_flags,BPF_F_LOCK |BPF_F_CPU );if (err)return err; - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->batch.elem_flags); max_count = attr->batch.count;if (!max_count)@@ -2143,7 +2157,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) goto err_put; } - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map,0 ); err = -ENOMEM ; value =kvmalloc (value_size,GFP_USER | __GFP_NOWARN);tools/include/uapi/linux/bpf.h +2
-1@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */ --2.50.1
Re: [PATCH bpf-next v6 2/7] bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
Quoted 28 lines · Alexei Starovoitov
Quoted 27 lines · Leon Hwang
Introduce
BPF_F_CPUandBPF_F_ALL_CPUSflags and check them for following APIs:* '
map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'And, get the correct value size for these APIs.
Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 22 ++++++++++++++++++ include/uapi/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 42 ++++++++++++++++++++++------------ tools/include/uapi/linux/bpf.h | 2 ++ 4 files changed, 54 insertions(+), 14 deletions(-)include/linux/bpf.h +0
-0@@ -3709,4 +3709,26 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void );
Quoted 7 lines
include/linux/bpf.h int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * · line 3713
Quoted 2 lines
include/linux/bpf.h +2
-0+static inline int bpf_map_check_cpu_flags (u64 flags,bool check_all_cpus_flag) +{This function is not used in this patch. Don't add it without users.
Also I really don't like 'bool' arguments.
They make callsite hard to read.
Agreed. Using a bool argument here makes the call sites harder to understand.
Quoted 12 lines
Instead of bool use
bpf_map_check_flags(u64 flags, u64 allowed_flags)so the callsites will look like:
bpf_map_check_flags(flags, BPF_F_CPU);
andbpf_map_check_flags(flags, BPF_F_CPU | BPF_F_ALL_CPUS);Also two functions that do very similar things look redundant. This
bpf_map_check_flags()vsbpf_map_check_op_flags()... I think one should do it.
Yes. It would be better to consolidate this logic into bpf_map_check_op_flags(), rather than introducing a separate but overlapping helper. That should keep the code simpler and avoid redundancy.
Thanks,
Leon
[...]
PATCH v6 3/7 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps
In reply to Leon HwangIntroduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs.
Introduce support for the BPF_F_CPU flag in percpu_array maps to allow:
* update value for specified CPU for both update_elem and update_batch APIs.
* lookup value for specified CPU for both lookup_elem and lookup_batch APIs.
The BPF_F_CPU flag is passed via:
* map_flags of lookup_elem and update_elem APIs along with embedded cpu info.
* elem_flags of lookup_batch and update_batch APIs along with embedded cpu info.
--- include/linux/bpf.h | 2 +- kernel/bpf/arraymap.c | 36 +++++++++++++++++++++++++++++------- kernel/bpf/syscall.c | 9 +++++++-- 3 files changed, 37 insertions(+), 10 deletions(-)
include/linux/bpf.h +1 -1
@@ -2697,7 +2697,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *callee);int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value); -int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_hash_update (struct bpf_map *map,void *key,void *value,u64 flags); int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
kernel/bpf/arraymap.c +29 -7
@@ -295,14 +295,18 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key,return per_cpu_ptr (array->pptrs[index & array->index_mask], cpu); } -int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value) +int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value,u64 map_flags) { struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key; +int err, cpu, off =0 ;void __percpu *pptr; -int cpu, off =0 ;u32 size; + err =bpf_map_check_cpu_flags (map_flags, false); +if (unlikely (err)) +return err; +if (unlikely (index >= array->map.max_entries))return -ENOENT ;
@@ -313,11 +317,18 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; +if (map_flags &BPF_F_CPU ) { + cpu = map_flags >>32 ; +copy_map_value_long (map, value,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value); + goto unlock; + }for_each_possible_cpu (cpu) {copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu));check_and_init_map_value (map, value + off); off += size; } +unlock:rcu_read_unlock ();return 0 ; }
@@ -386,13 +397,13 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, { struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key; +int err, cpu, off =0 ;void __percpu *pptr; -int cpu, off =0 ;u32 size; -if (unlikely (map_flags >BPF_EXIST )) -/* unknown flags */ -return -EINVAL ; + err =bpf_map_check_cpu_flags (map_flags, true); +if (unlikely (err)) +return err;if (unlikely (index >= array->map.max_entries))/* all elements were pre-allocated, cannot insert a new one */
@@ -411,11 +422,22 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; +if (map_flags &BPF_F_CPU ) { + cpu = map_flags >>32 ; +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value); +bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); + goto unlock; + }for_each_possible_cpu (cpu) {copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off);bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; +/* same user-provided value is used if BPF_F_ALL_CPUS is +* specified, otherwise value is an array of per-CPU values. +*/ +if (!(map_flags &BPF_F_ALL_CPUS )) + off += size; } +unlock:rcu_read_unlock ();return 0 ; }
kernel/bpf/syscall.c +7 -2
@@ -133,7 +133,12 @@ bool bpf_map_write_active(const struct bpf_map *map)static bool bpf_map_supports_cpu_flags (enum bpf_map_type map_type) { -return false; +switch (map_type) { +case BPF_MAP_TYPE_PERCPU_ARRAY : +return true; +default : +return false; + } }static int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags)
@@ -338,7 +343,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { err =bpf_percpu_hash_copy (map, key, value); } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { - err =bpf_percpu_array_copy (map, key, value); + err =bpf_percpu_array_copy (map, key, value, flags); } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { err =bpf_percpu_cgroup_storage_copy (map, key, value); } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
Signature · 1 line
2.50.1
PATCH v6 4/7 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash and lru_percpu_hash maps
In reply to Leon HwangIntroduce BPF_F_ALL_CPUS flag support for percpu_hash and lru_percpu_hash maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs.
Introduce BPF_F_CPU flag support for percpu_hash and lru_percpu_hash maps to allow:
* update value for specified CPU for both update_elem and update_batch APIs.
* lookup value for specified CPU for both lookup_elem and lookup_batch APIs.
The BPF_F_CPU flag is passed via:
* map_flags along with embedded cpu info.
* elem_flags along with embedded cpu info.
--- include/linux/bpf.h | 2 +- kernel/bpf/hashtab.c | 105 +++++++++++++++++++++++++++++++------------ kernel/bpf/syscall.c | 4 +- 3 files changed, 80 insertions(+), 31 deletions(-)
include/linux/bpf.h +1 -1
@@ -2696,7 +2696,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,struct bpf_func_state *caller, struct bpf_func_state *callee); -int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_hash_update (struct bpf_map *map,void *key,void *value,u64 flags);
kernel/bpf/hashtab.c +76 -29
219 lines not on this page · raw message
kernel/bpf/syscall.c +3 -1
@@ -135,6 +135,8 @@ static bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) {switch (map_type) {case BPF_MAP_TYPE_PERCPU_ARRAY : +case BPF_MAP_TYPE_PERCPU_HASH : +case BPF_MAP_TYPE_LRU_PERCPU_HASH :return true;default :return false;
@@ -341,7 +343,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,bpf_disable_instrumentation ();if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH ) { - err =bpf_percpu_hash_copy (map, key, value); + err =bpf_percpu_hash_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) { err =bpf_percpu_array_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) {
Signature · 1 line
2.50.1
PATCH v6 5/7 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_cgroup_storage maps
In reply to Leon HwangIntroduce BPF_F_ALL_CPUS flag support for percpu_cgroup_storage maps to allow updating values for all CPUs with a single value for update_elem API.
Introduce BPF_F_CPU flag support for percpu_cgroup_storage maps to allow:
* update value for specified CPU for update_elem API.
* lookup value for specified CPU for lookup_elem API.
The BPF_F_CPU flag is passed via map_flags along with embedded cpu info.
--- include/linux/bpf-cgroup.h | 4 ++-- kernel/bpf/local_storage.c | 46 +++++++++++++++++++++++++++++--------- kernel/bpf/syscall.c | 3 ++- 3 files changed, 39 insertions(+), 14 deletions(-)
include/linux/bpf-cgroup.h +2 -2
@@ -172,7 +172,7 @@ void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,void bpf_cgroup_storage_unlink (struct bpf_cgroup_storage *storage);int bpf_cgroup_storage_assign (struct bpf_prog_aux *aux,struct bpf_map *map); -int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_cgroup_storage_update (struct bpf_map *map,void *key,void *value,u64 flags);
@@ -467,7 +467,7 @@ static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(static inline void bpf_cgroup_storage_free (struct bpf_cgroup_storage *storage) {}static inline int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key, -void *value) { +void *value,u64 flags) {return 0 ; } static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
kernel/bpf/local_storage.c +35 -11
96 lines not on this page · raw message
kernel/bpf/syscall.c +2 -1
@@ -137,6 +137,7 @@ static bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)case BPF_MAP_TYPE_PERCPU_ARRAY :case BPF_MAP_TYPE_PERCPU_HASH :case BPF_MAP_TYPE_LRU_PERCPU_HASH : +case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE :return true;default :return false;
@@ -347,7 +348,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) { err =bpf_percpu_array_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) { - err =bpf_percpu_cgroup_storage_copy (map, key, value); + err =bpf_percpu_cgroup_storage_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_STACK_TRACE ) { err =bpf_stackmap_copy (map, key, value); }else if (IS_FD_ARRAY (map) ||IS_FD_PROG_ARRAY (map)) {
Signature · 1 line
2.50.1
PATCH v6 6/7 libbpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps
In reply to Leon HwangAdd libbpf support for the BPF_F_CPU flag for percpu maps by embedding the cpu info into the high 32 bits of:
1. **flags**: bpf_map_lookup_elem_flags(), bpf_map__lookup_elem(), bpf_map_update_elem() and bpf_map__update_elem()
2. **opts->elem_flags**: bpf_map_lookup_batch() and bpf_map_update_batch()
And the flag can be BPF_F_ALL_CPUS, but cannot be
'BPF_F_CPU | BPF_F_ALL_CPUS'.
Behavior:
* If the flag is BPF_F_ALL_CPUS, the update is applied across all CPUs.
* If the flag is BPF_F_CPU, it updates value only to the specified CPU.
* If the flag is BPF_F_CPU, lookup value only from the specified CPU.
* lookup does not support BPF_F_ALL_CPUS.
--- tools/lib/bpf/bpf.h | 8 ++++++++ tools/lib/bpf/libbpf.c | 26 ++++++++++++++++++++------ tools/lib/bpf/libbpf.h | 21 ++++++++------------- 3 files changed, 36 insertions(+), 19 deletions(-)
tools/lib/bpf/bpf.h +8 -0
@@ -286,6 +286,14 @@ LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch,* Update spin_lock-ed map elements. This must be * specified if the map value contains a spinlock. * +* **BPF_F_CPU** +* As for percpu maps, update value on the specified CPU. And the cpu +* info is embedded into the high 32 bits of **opts->elem_flags**. +* +* **BPF_F_ALL_CPUS** +* As for percpu maps, update value across all CPUs. This flag cannot +* be used with BPF_F_CPU at the same time. +* * @param fd BPF map file descriptor * @param keys pointer to an array of *count* keys * @param values pointer to an array of *count* values
tools/lib/bpf/libbpf.c +20 -6
@@ -10603,7 +10603,7 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) }static int validate_map_op (const struct bpf_map *map,size_t key_sz, -size_t value_sz,bool check_value_sz) +size_t value_sz,bool check_value_sz,__u64 flags) {if (!map_is_created (map))/* map is not yet created */ return -ENOENT ;
@@ -10630,6 +10630,20 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,int num_cpu =libbpf_num_possible_cpus ();size_t elem_sz =roundup (map->def.value_size,8 ); +if (flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) { +if ((flags &BPF_F_CPU ) && (flags &BPF_F_ALL_CPUS )) { +pr_warn ("map '%s': can't use BPF_F_CPU and BPF_F_ALL_CPUS at the same time\n" , + map->name); +return -EINVAL ; + } +if (value_sz != elem_sz) { +pr_warn ("map '%s': unexpected value size %zu provided for per-CPU map, expected %zu\n" , + map->name, value_sz, elem_sz); +return -EINVAL ; + } +break ; + } +if (value_sz != num_cpu * elem_sz) {pr_warn ("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n" , map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
@@ -10654,7 +10668,7 @@ int bpf_map__lookup_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10667,7 +10681,7 @@ int bpf_map__update_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10679,7 +10693,7 @@ int bpf_map__delete_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ); + err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ , flags);if (err)return libbpf_err (err);
@@ -10692,7 +10706,7 @@ int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10704,7 +10718,7 @@ int bpf_map__get_next_key(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ); + err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ,0 );if (err)return libbpf_err (err);
tools/lib/bpf/libbpf.h +8 -13
@@ -1196,12 +1196,13 @@ LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map);* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** * @param value pointer to memory in which looked up value will be stored * @param value_sz size in byte of value data memory; it has to match BPF map -* definition's **value_size**. For per-CPU BPF maps value size has to be -* a product of BPF map value size and number of possible CPUs in the system -* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for -* per-CPU values value size has to be aligned up to closest 8 bytes for -* alignment reasons, so expected size is: `round_up(value_size, 8) -* * libbpf_num_possible_cpus()`. +* definition's **value_size**. For per-CPU BPF maps, value size can be +* `round_up(value_size, 8)` if **BPF_F_CPU** or **BPF_F_ALL_CPUS** is +* specified in **flags**, otherwise a product of BPF map value size and number +* of possible CPUs in the system (could be fetched with +* **libbpf_num_possible_cpus()**). Note else that for per-CPU values value +* size has to be aligned up to closest 8 bytes, so expected size is: +* `round_up(value_size, 8) * libbpf_num_possible_cpus()`. * @flags extra flags passed to kernel for this operation * @return 0, on success; negative error, otherwise *
@@ -1219,13 +1220,7 @@ LIBBPF_API int bpf_map__lookup_elem(const struct bpf_map *map,* @param key pointer to memory containing bytes of the key * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** * @param value pointer to memory containing bytes of the value -* @param value_sz size in byte of value data memory; it has to match BPF map -* definition's **value_size**. For per-CPU BPF maps value size has to be -* a product of BPF map value size and number of possible CPUs in the system -* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for -* per-CPU values value size has to be aligned up to closest 8 bytes for -* alignment reasons, so expected size is: `round_up(value_size, 8) -* * libbpf_num_possible_cpus()`. +* @param value_sz refer to **bpf_map__lookup_elem**'s description.' * @flags extra flags passed to kernel for this operation * @return 0, on success; negative error, otherwise *
Signature · 1 line
2.50.1
PATCH v6 7/7 selftests/bpf: Add cases to test BPF_F_CPU and BPF_F_ALL_CPUS flags
In reply to Leon HwangAdd test coverage for the new BPF_F_CPU and BPF_F_ALL_CPUS flags support in percpu maps. The following APIs are exercised:
* bpf_map_update_batch()
* bpf_map_lookup_batch()
* bpf_map_update_elem()
* bpf_map__update_elem()
* bpf_map_lookup_elem_flags()
* bpf_map__lookup_elem()
cd tools/testing/selftests/bpf/
./test_progs -t percpu_alloc
253/13 percpu_alloc/cpu_flag_percpu_array:OK 253/14 percpu_alloc/cpu_flag_percpu_hash:OK 253/15 percpu_alloc/cpu_flag_lru_percpu_hash:OK 253/16 percpu_alloc/cpu_flag_percpu_cgroup_storage:OK 253 percpu_alloc:OK
Summary: 1/16 PASSED, 0 SKIPPED, 0 FAILED
--- .../selftests/bpf/prog_tests/percpu_alloc.c | 233 ++++++++++++++++++ .../selftests/bpf/progs/percpu_alloc_array.c | 32 +++ 2 files changed, 265 insertions(+)
tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +233 -0
@@ -1,5 +1,6 @@// SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "cgroup_helpers.h" #include "percpu_alloc_array.skel.h" #include "percpu_alloc_cgrp_local_storage.skel.h" #include "percpu_alloc_fail.skel.h"
@@ -115,6 +116,230 @@ static void test_failure(void) {RUN_TESTS (percpu_alloc_fail); } +static void test_percpu_map_op_cpu_flag (struct bpf_map *map,void *keys,size_t key_sz, +u32 max_entries,bool test_batch) +{ +int i, j, cpu, map_fd, value_size, nr_cpus, err; +u64 *values =NULL , batch =0 , flags; +const u64 value =0xDEADC0DE ; +size_t value_sz =sizeof (u64); +u32 count; +LIBBPF_OPTS (bpf_map_batch_opts, batch_opts); + + nr_cpus =libbpf_num_possible_cpus (); +if (!ASSERT_GT (nr_cpus,0 ,"libbpf_num_possible_cpus" )) +return ; + + value_size = value_sz * nr_cpus; + values =calloc (max_entries, value_size); +if (!ASSERT_OK_PTR (values,"calloc values" )) + goto out; +memset (values,0 , value_size * max_entries); + + map_fd =bpf_map__fd (map); + flags =BPF_F_CPU |BPF_F_ALL_CPUS ; + err =bpf_map_lookup_elem_flags (map_fd, keys, values, flags); +if (!ASSERT_ERR (err,"bpf_map_lookup_elem_flags err" )) + goto out; + + err =bpf_map_update_elem (map_fd, keys, values, flags); +if (!ASSERT_ERR (err,"bpf_map_update_elem err" )) + goto out; + + flags = (u64 )nr_cpus <<32 |BPF_F_CPU ; + err =bpf_map_update_elem (map_fd, keys, values, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_update_elem -ERANGE" )) + goto out; + + err =bpf_map__update_elem (map, keys, key_sz, values, value_sz, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map__update_elem -ERANGE" )) + goto out; + + err =bpf_map_lookup_elem_flags (map_fd, keys, values, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_lookup_elem_flags -ERANGE" )) + goto out; + + err =bpf_map__lookup_elem (map, keys, key_sz, values, value_sz, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map__lookup_elem -ERANGE" )) + goto out; + +for (cpu =0 ; cpu < nr_cpus; cpu++) { +/* clear value on all cpus */ + values[0 ] =0 ; + flags =BPF_F_ALL_CPUS ; +for (i =0 ; i < max_entries; i++) { + err =bpf_map__update_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__update_elem all_cpus" )) + goto out; + } + +/* update value on specified cpu */ +for (i =0 ; i < max_entries; i++) { + values[0 ] = value; + flags = (u64 )cpu <<32 |BPF_F_CPU ; + err =bpf_map__update_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__update_elem specified cpu" )) + goto out; + +/* lookup then check value on CPUs */ +for (j =0 ; j < nr_cpus; j++) { + flags = (u64 )j <<32 |BPF_F_CPU ; + err =bpf_map__lookup_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__lookup_elem specified cpu" )) + goto out; +if (!ASSERT_EQ (values[0 ], j != cpu ?0 : value, +"bpf_map__lookup_elem value on specified cpu" )) + goto out; + } + } + } + +if (!test_batch) + goto out; + + batch_opts.elem_flags = (u64 )nr_cpus <<32 |BPF_F_CPU ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_update_batch -ERANGE" )) + goto out; + +for (cpu =0 ; cpu < nr_cpus; cpu++) { +memset (values,0 , max_entries * value_size); + +/* clear values across all CPUs */ + batch_opts.elem_flags =BPF_F_ALL_CPUS ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_OK (err,"bpf_map_update_batch all_cpus" )) + goto out; + +/* update values on specified CPU */ +for (i =0 ; i < max_entries; i++) + values[i] = value; + + batch_opts.elem_flags = (u64 )cpu <<32 |BPF_F_CPU ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_OK (err,"bpf_map_update_batch specified cpu" )) + goto out; + +/* lookup values on specified CPU */ +memset (values,0 , max_entries * value_sz); + err =bpf_map_lookup_batch (map_fd,NULL , &batch, keys, values, &count, &batch_opts); +if (!ASSERT_TRUE (!err || err == -ENOENT ,"bpf_map_lookup_batch specified cpu" )) + goto out; + +for (i =0 ; i < max_entries; i++) +if (!ASSERT_EQ (values[i], value,"value on specified cpu" )) + goto out; + +/* lookup values from all CPUs */ + batch_opts.elem_flags =0 ; +memset (values,0 , max_entries * value_size); + err =bpf_map_lookup_batch (map_fd,NULL , &batch, keys, values, &count, &batch_opts); +if (!ASSERT_TRUE (!err || err == -ENOENT ,"bpf_map_lookup_batch all_cpus" )) + goto out; + +for (i =0 ; i < max_entries; i++) { +for (j =0 ; j < nr_cpus; j++) { +if (!ASSERT_EQ (values[i*nr_cpus + j], j != cpu ?0 : value, +"value on specified cpu" )) + goto out; + } + } + } + +out: +if (values) +free (values); +} + +static void test_percpu_map_cpu_flag (enum bpf_map_type map_type) +{ +struct percpu_alloc_array *skel; +size_t key_sz =sizeof (int ); +int *keys =NULL , i, err; +struct bpf_map *map; +u32 max_entries; + + skel =percpu_alloc_array__open (); +if (!ASSERT_OK_PTR (skel,"percpu_alloc_array__open" )) +return ; + + map = skel->maps.percpu; +bpf_map__set_type (map, map_type); + + err =percpu_alloc_array__load (skel); +if (!ASSERT_OK (err,"test_percpu_alloc__load" )) + goto out; + + max_entries =bpf_map__max_entries (map); + keys =calloc (max_entries, key_sz); +if (!ASSERT_OK_PTR (keys,"calloc keys" )) + goto out; + +for (i =0 ; i < max_entries; i++) + keys[i] = i; + +test_percpu_map_op_cpu_flag (map, keys, key_sz, max_entries, true); +out: +if (keys) +free (keys); +percpu_alloc_array__destroy (skel); +} + +static void test_percpu_array_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_PERCPU_ARRAY ); +} + +static void test_percpu_hash_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_PERCPU_HASH ); +} + +static void test_lru_percpu_hash_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_LRU_PERCPU_HASH ); +} + +static void test_percpu_cgroup_storage_cpu_flag (void ) +{ +struct bpf_cgroup_storage_key key; +struct percpu_alloc_array *skel; +int cgroup =-1 , prog_fd, err; +struct bpf_map *map; + + skel =percpu_alloc_array__open_and_load (); +if (!ASSERT_OK_PTR (skel,"percpu_alloc_array__open_and_load" )) +return ; + + cgroup =create_and_get_cgroup ("/cg_percpu" ); +if (!ASSERT_GE (cgroup,0 ,"create_and_get_cgroup" )) + goto out; + + err =join_cgroup ("/cg_percpu" ); +if (!ASSERT_OK (err,"join_cgroup" )) + goto out; + + prog_fd =bpf_program__fd (skel->progs.cgroup_egress); + err =bpf_prog_attach (prog_fd, cgroup,BPF_CGROUP_INET_EGRESS ,0 ); +if (!ASSERT_OK (err,"bpf_prog_attach" )) + goto out; + + map = skel->maps.percpu_cgroup_storage; + err =bpf_map_get_next_key (bpf_map__fd (map),NULL , &key); +if (!ASSERT_OK (err,"bpf_map_get_next_key" )) + goto out; + +test_percpu_map_op_cpu_flag (map, &key,sizeof (key),1 , false); +out: +bpf_prog_detach2 (-1 , cgroup,BPF_CGROUP_INET_EGRESS ); +close (cgroup); +cleanup_cgroup_environment (); +percpu_alloc_array__destroy (skel); +} +void test_percpu_alloc (void ) {if (test__start_subtest ("array" ))
@@ -125,4 +350,12 @@ void test_percpu_alloc(void)test_cgrp_local_storage ();if (test__start_subtest ("failure_tests" ))test_failure (); +if (test__start_subtest ("cpu_flag_percpu_array" )) +test_percpu_array_cpu_flag (); +if (test__start_subtest ("cpu_flag_percpu_hash" )) +test_percpu_hash_cpu_flag (); +if (test__start_subtest ("cpu_flag_lru_percpu_hash" )) +test_lru_percpu_hash_cpu_flag (); +if (test__start_subtest ("cpu_flag_percpu_cgroup_storage" )) +test_percpu_cgroup_storage_cpu_flag (); }
tools/testing/selftests/bpf/progs/percpu_alloc_array.c +32 -0
@@ -187,4 +187,36 @@ int BPF_PROG(test_array_map_10)return 0 ; } +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, u64); +} percpu SEC(".maps"); + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG (test_percpu_array ,int x) +{ +u64 value =0xDEADC0DE ; +int key =0 ; + +bpf_map_update_elem (&percpu, &key, &value,BPF_ANY ); +return 0 ; +} + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, u64); +} percpu_cgroup_storage SEC(".maps"); + +SEC("cgroup_skb/egress") +int cgroup_egress (struct __sk_buff *skb) +{ +u64 *val =bpf_get_local_storage (&percpu_cgroup_storage,0 ); + +__sync_fetch_and_add (val,1 ); +return 1 ; +} + char _license[] SEC("license") = "GPL";
Signature · 1 line
2.50.1