bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps
Subsystems (5)
v5 cover bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps
This patch set introduces the BPF_F_CPU and BPF_F_ALL_CPUS flags for percpu maps, as the requirement of BPF_F_ALL_CPUS flag for percpu_array maps was discussed in the thread of
"[PATCH bpf-next v3 0/4] bpf: Introduce global percpu data"[1].
The goal of BPF_F_ALL_CPUS flag is to reduce data caching overhead in light skeletons by allowing a single value to be reused to update values across all CPUs. This avoids the M:N problem where M cached values are used to update a map on N CPUs kernel.
The BPF_F_CPU flag is accompanied by *flags*-embedded cpu info, which specifies the target CPU for the operation:
* For lookup operations: the flag field alongside cpu info enable querying a value on the specified CPU.
* For update operations: the flag field alongside cpu info enable updating value for specified CPU.
Links:
[1] https://lore.kernel.org/bpf/[email protected]/
Changes:
v4 -> v5:
* Address comments from Andrii:
* Refactor data copying logic for all percpu maps.
* Drop this_cpu_ptr() micro-optimization.
* Drop cpu check in libbpf's validate_map_op().
* Enhance bpf_map_check_op_flags() using *allowed flags* instead of
'extra_flags_mask'.
v3 -> v4:
* Address comments from Andrii:
* Remove unnecessary map_type check in bpf_map_value_size().
* Reduce code churn.
* Remove unnecessary do_delete check in
__htab_map_lookup_and_delete_batch().
* Introduce bpf_percpu_copy_to_user() and bpf_percpu_copy_from_user().
* Rename check_map_flags() to bpf_map_check_op_flags() with
extra_flags_mask.
* Add human-readable pr_warn() explanations in validate_map_op().
* Use flags in bpf_map__delete_elem() and
bpf_map__lookup_and_delete_elem(). * Drop "for alignment reasons".
v3 link: https://lore.kernel.org/bpf/[email protected]/
v2 -> v3:
* Address comments from Alexei:
* Use BPF_F_ALL_CPUS instead of BPF_ALL_CPUS magic.
* Introduce these two cpu flags for all percpu maps.
* Address comments from Jiri:
* Reduce some unnecessary u32 cast.
* Refactor more generic map flags check function.
* A code style issue.
v2 link: https://lore.kernel.org/bpf/[email protected]/
v1 -> v2:
* Address comments from Andrii:
* Embed cpu info as high 32 bits of *flags* totally.
* Use ERANGE instead of E2BIG.
* Few format issues.
Leon Hwang (9):
bpf: Generalize data copying for percpu maps
bpf: Introduce internal bpf_map_check_op_flags helper function
bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps
data copying
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array
maps
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash
and lru_percpu_hash maps
bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for
percpu_cgroup_storage maps
libbpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps selftests/bpf: Add cases to test BPF_F_CPU and BPF_F_ALL_CPUS flags
include/linux/bpf-cgroup.h | 4 +- include/linux/bpf.h | 121 ++++++++- include/uapi/linux/bpf.h | 2 + kernel/bpf/arraymap.c | 28 +-- kernel/bpf/hashtab.c | 90 +++---- kernel/bpf/local_storage.c | 32 +-- kernel/bpf/syscall.c | 64 ++--- tools/include/uapi/linux/bpf.h | 2 + tools/lib/bpf/bpf.h | 8 + tools/lib/bpf/libbpf.c | 26 +- tools/lib/bpf/libbpf.h | 21 +- .../selftests/bpf/prog_tests/percpu_alloc.c | 233 ++++++++++++++++++ .../selftests/bpf/progs/percpu_alloc_array.c | 32 +++ 13 files changed, 528 insertions(+), 135 deletions(-)
--
2.50.1
PATCH v5 1/9 bpf: Generalize data copying for percpu maps
Refactor the data copying logic of the following percpu map types:
* percpu_array
* percpu_hash
* lru_percpu_hash
* percpu_cgroup_storage
by introducing two helpers:
* 'bpf_percpu_copy_data()'
* 'bpf_percpu_update_data()'
It is to introduce BPF_F_CPU and BPF_F_ALL_CPUS flags for these percpu maps with less code churn.
--- include/linux/bpf.h | 28 +++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 27 ++++----------------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 48 deletions(-)
include/linux/bpf.h +27 -1
@@ -547,6 +547,33 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); } +#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_data (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); + off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj); + +static inline void bpf_percpu_update_data (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); +bpf_long_memcpy (per_cpu_ptr (pptr, cpu), value + off, size); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;
@@ -2417,7 +2444,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);
kernel/bpf/arraymap.c +2 -12
@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))
@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_data (map, pptr, value, size);rcu_read_unlock ();return 0 ; }
@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))
@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_update_data (map, pptr, value, size);rcu_read_unlock ();return 0 ; }
kernel/bpf/hashtab.c +4 -23
@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu; -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_update_data (&htab->map, pptr, value, size); } }
@@ -1610,14 +1606,9 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,if (is_percpu) {u32 roundup_value_size =round_up (map->value_size,8 );void __percpu *pptr; -int off =0 , cpu; pptr =htab_elem_get_ptr (l, key_size); -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (&htab->map, value + off); - off += roundup_value_size; - } +bpf_percpu_copy_data (&htab->map, pptr, value, roundup_value_size); }else {void *src =htab_elem_value (l, map->key_size);
@@ -1802,15 +1793,10 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,memcpy (dst_key, l->key, key_size);if (is_percpu) { -int off =0 , cpu;void __percpu *pptr; pptr =htab_elem_get_ptr (l, map->key_size); -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map, dst_val + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (&htab->map, dst_val + off); - off += size; - } +bpf_percpu_copy_data (&htab->map, pptr, dst_val, size); }else { value =htab_elem_value (l, key_size);if (is_fd_htab (htab)) {
@@ -2370,7 +2356,6 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)struct htab_elem *l;void __percpu *pptr;int ret = -ENOENT ; -int cpu, off =0 ;u32 size;/* per_cpu areas are zero-filled and bpf programs can only
@@ -2386,11 +2371,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)* eviction heuristics when user space does a map walk. */ pptr =htab_elem_get_ptr (l, map->key_size); -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_data (map, pptr, value, size); ret =0 ; out:rcu_read_unlock ();
kernel/bpf/local_storage.c +6 -12
@@ -184,7 +184,7 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key, {struct bpf_cgroup_storage_map *map =map_to_storage (_map);struct bpf_cgroup_storage *storage; -int cpu, off =0 ; +void __percpu *pptr;u32 size;rcu_read_lock ();
@@ -199,11 +199,8 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,* will not leak any kernel data */ size =round_up (_map->value_size,8 ); -for_each_possible_cpu (cpu) { -bpf_long_memcpy (value + off, -per_cpu_ptr (storage->percpu_buf, cpu), size); - off += size; - } + pptr = storage->percpu_buf; +bpf_percpu_copy_data (_map, pptr, value, size);rcu_read_unlock ();return 0 ; }
@@ -213,7 +210,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, {struct bpf_cgroup_storage_map *map =map_to_storage (_map);struct bpf_cgroup_storage *storage; -int cpu, off =0 ; +void __percpu *pptr;u32 size;if (map_flags !=BPF_ANY && map_flags !=BPF_EXIST )
@@ -233,11 +230,8 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,* so no kernel data leaks possible */ size =round_up (_map->value_size,8 ); -for_each_possible_cpu (cpu) { -bpf_long_memcpy (per_cpu_ptr (storage->percpu_buf, cpu), - value + off, size); - off += size; - } + pptr = storage->percpu_buf; +bpf_percpu_update_data (_map, pptr, value, size);rcu_read_unlock ();return 0 ; }
Signature · 1 line
2.50.1
Re: [PATCH bpf-next v5 1/9] bpf: Generalize data copying for percpu maps
Quoted 43 lines · Leon Hwang
Refactor the data copying logic of the following percpu map types:
*
percpu_array
*percpu_hash
*lru_percpu_hash
*percpu_cgroup_storageby introducing two helpers:
* '
bpf_percpu_copy_data()'
* 'bpf_percpu_update_data()'It is to introduce
BPF_F_CPUandBPF_F_ALL_CPUSflags for these percpu maps with less code churn.Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 28 +++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 27 ++++----------------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 48 deletions(-)include/linux/bpf.h +11
-0@@ -547,6 +547,33 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); } +#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_data (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); + off += size; + } +}
..
include/linux/bpf.h int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) · line 315
Quoted 10 lines
include/linux/bpf.h +1
-5@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_data (map, pptr, value, size);
Same issue as before. This is not equivalent.
Stop this "refactoring".
pw-bot: cr
Re: [PATCH bpf-next v5 1/9] bpf: Generalize data copying for percpu maps
Quoted 2 lines · Alexei Starovoitov
Quoted 1 line · Leon Hwang
[...]
Quoted 5 lines
Quoted 5 lines
kernel/bpf/arraymap.c +0
-1@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) {
Quoted 9 lines
kernel/bpf/arraymap.c int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) · line 315
Quoted 5 lines
kernel/bpf/arraymap.c +1
-4-copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_data (map, pptr, value, size);Same issue as before. This is not equivalent.
Stop this "refactoring".
Got it.
I’ll drop this refactoring and keep the original per-CPU copy logic as is, and focus only on adding the BPF_F_CPU / BPF_F_ALL_CPUS support.
Thanks,
Leon
PATCH v5 2/9 bpf: Introduce internal bpf_map_check_op_flags helper function
In reply to Leon HwangIt is to unify map flags checking for lookup_elem, update_elem, lookup_batch and update_batch APIs.
Therefore, it will be convenient to check BPF_F_CPU and BPF_F_ALL_CPUS flags in it for these APIs in next patch.
--- include/linux/bpf.h | 31 +++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 34 +++++++++++----------------------- 2 files changed, 42 insertions(+), 23 deletions(-)
include/linux/bpf.h +31 -0
@@ -3735,4 +3735,35 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) +{ +if (flags & ~allowed_flags) +return -EINVAL ; + +if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK )) +return -EINVAL ; + +return 0 ; +} + +static inline int bpf_map_check_lookup_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +} + +static inline int bpf_map_check_update_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags, ~0 ); +} + +static inline int bpf_map_check_lookup_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_lookup_flags (map, flags); +} + +static inline int bpf_map_check_update_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +} +#endif /* _LINUX_BPF_H */
kernel/bpf/syscall.c +11 -23
@@ -1669,9 +1669,6 @@ static int map_lookup_elem(union bpf_attr *attr)if (CHECK_ATTR (BPF_MAP_LOOKUP_ELEM ))return -EINVAL ; -if (attr->flags & ~BPF_F_LOCK ) -return -EINVAL ; -CLASS (fd, f)(attr->map_fd); map =__bpf_map_get (f);if (IS_ERR (map))
@@ -1679,9 +1676,9 @@ static int map_lookup_elem(union bpf_attr *attr)if (!(map_get_sys_perms (map, f) &FMODE_CAN_READ ))return -EPERM ; -if ((attr->flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) -return -EINVAL ; + err =bpf_map_check_lookup_flags (map, attr->flags); +if (err) +return err; key =__bpf_copy_key (ukey, map->key_size);if (IS_ERR (key))
@@ -1744,11 +1741,9 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) goto err_put; } -if ((attr->flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) { - err = -EINVAL ; + err =bpf_map_check_update_flags (map, attr->flags); +if (err) goto err_put; - } key =___bpf_copy_key (ukey, map->key_size);if (IS_ERR (key)) {
@@ -1952,13 +1947,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,void *key, *value;int err =0 ; -if (attr->batch.elem_flags & ~BPF_F_LOCK ) -return -EINVAL ; - -if ((attr->batch.elem_flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) { -return -EINVAL ; - } + err =bpf_map_check_update_batch_flags (map, attr->batch.elem_flags); +if (err) +return err; value_size =bpf_map_value_size (map);
@@ -2015,12 +2006,9 @@ int generic_map_lookup_batch(struct bpf_map *map,u32 value_size, cp, max_count;int err; -if (attr->batch.elem_flags & ~BPF_F_LOCK ) -return -EINVAL ; - -if ((attr->batch.elem_flags &BPF_F_LOCK ) && - !btf_record_has_field (map->record,BPF_SPIN_LOCK )) -return -EINVAL ; + err =bpf_map_check_lookup_batch_flags (map, attr->batch.elem_flags); +if (err) +return err; value_size =bpf_map_value_size (map);
Signature · 1 line
2.50.1
Re: [PATCH bpf-next v5 2/9] bpf: Introduce internal bpf_map_check_op_flags helper function
Quoted 41 lines · Leon Hwang
It is to unify map flags checking for
lookup_elem,update_elem,lookup_batchandupdate_batchAPIs.Therefore, it will be convenient to check
BPF_F_CPUandBPF_F_ALL_CPUSflags in it for these APIs in next patch.Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 31 +++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 34 +++++++++++----------------------- 2 files changed, 42 insertions(+), 23 deletions(-)include/linux/bpf.h +20
-0@@ -3735,4 +3735,35 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) +{ +if (flags & ~allowed_flags) +return -EINVAL ; + +if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK )) +return -EINVAL ; + +return 0 ; +} + +static inline int bpf_map_check_lookup_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +} + +static inline int bpf_map_check_update_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags, ~0 ); +}
include/linux/bpf.h int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * · line 3767
Quoted 10 lines
include/linux/bpf.h +10
-0+ +static inline int bpf_map_check_lookup_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_lookup_flags (map, flags); +} + +static inline int bpf_map_check_update_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +}
I don't like these pointless wrappers.
They make the code less readable.
Re: [PATCH bpf-next v5 2/9] bpf: Introduce internal bpf_map_check_op_flags helper function
Quoted 47 lines · Alexei Starovoitov
Quoted 46 lines · Leon Hwang
It is to unify map flags checking for
lookup_elem,update_elem,lookup_batchandupdate_batchAPIs.Therefore, it will be convenient to check
BPF_F_CPUandBPF_F_ALL_CPUSflags in it for these APIs in next patch.Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 31 +++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 34 +++++++++++----------------------- 2 files changed, 42 insertions(+), 23 deletions(-)include/linux/bpf.h +25
-0@@ -3735,4 +3735,35 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) +{ +if (flags & ~allowed_flags) +return -EINVAL ; + +if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK )) +return -EINVAL ; + +return 0 ; +} + +static inline int bpf_map_check_lookup_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +} + +static inline int bpf_map_check_update_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags, ~0 ); +} + +static inline int bpf_map_check_lookup_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_lookup_flags (map, flags); +}
Quoted 8 lines
include/linux/bpf.h int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * · line 3767
Quoted 5 lines
include/linux/bpf.h +5
-0+ +static inline int bpf_map_check_update_batch_flags (struct bpf_map *map,u64 flags) +{ +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +}I don't like these pointless wrappers.
They make the code less readable.
Thanks for the feedback.
My intent was to keep the helpers close in style to bpf_map_check_op_flags(), so that lookup/update (single or batch) would follow a consistent pattern. This way it’s easier to see the relation between map ops and their corresponding flag checks.
That said, I understand your point about readability. I will drop these wrappers and just call bpf_map_check_op_flags() directly at each site.
Thanks,
Leon
PATCH v5 3/9 bpf: Introduce BPF_F_CPU and BPF_F_ALL_CPUS flags
In reply to Leon HwangIntroduce BPF_F_CPU and BPF_F_ALL_CPUS flags and check them for following APIs:
* 'map_lookup_elem()'
* 'map_update_elem()'
* 'generic_map_lookup_batch()'
* 'generic_map_update_batch()'
And, get the correct value size for these APIs.
--- include/linux/bpf.h | 39 +++++++++++++++++++++++++++++++--- include/uapi/linux/bpf.h | 2 ++ kernel/bpf/syscall.c | 24 +++++++++++---------- tools/include/uapi/linux/bpf.h | 2 ++ 4 files changed, 53 insertions(+), 14 deletions(-)
include/linux/bpf.h +36 -3
@@ -3735,20 +3735,31 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * const char **linep, int *nump);struct bpf_prog *bpf_prog_find_from_stack (void ); +static inline bool bpf_map_supports_cpu_flags (enum bpf_map_type map_type) +{ +return false; +} +static inline int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags) { -if (flags & ~allowed_flags) +if ((u32 )flags & ~allowed_flags)return -EINVAL ;if ((flags &BPF_F_LOCK ) && !btf_record_has_field (map->record,BPF_SPIN_LOCK ))return -EINVAL ; +if (!(flags &BPF_F_CPU ) && flags >>32 ) +return -EINVAL ; + +if ((flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) && !bpf_map_supports_cpu_flags (map->map_type)) +return -EINVAL ; +return 0 ; }static inline int bpf_map_check_lookup_flags (struct bpf_map *map,u64 flags) { -return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK |BPF_F_CPU ); }static inline int bpf_map_check_update_flags (struct bpf_map *map,u64 flags)
@@ -3763,7 +3774,29 @@ static inline int bpf_map_check_lookup_batch_flags(struct bpf_map *map, u64 flagstatic inline int bpf_map_check_update_batch_flags (struct bpf_map *map,u64 flags) { -return bpf_map_check_op_flags (map, flags,BPF_F_LOCK ); +return bpf_map_check_op_flags (map, flags,BPF_F_LOCK |BPF_F_CPU |BPF_F_ALL_CPUS ); +} + +static inline int bpf_map_check_cpu_flags (u64 flags,bool check_all_cpus_flag) +{ +const u64 cpu_flags =BPF_F_CPU |BPF_F_ALL_CPUS ; +u32 cpu; + +if (check_all_cpus_flag) { +if (unlikely ((u32 )flags >BPF_F_ALL_CPUS )) +return -EINVAL ; +if (unlikely ((flags & cpu_flags) == cpu_flags)) +return -EINVAL ; + }else { +if (unlikely ((u32 )flags & ~BPF_F_CPU )) +return -EINVAL ; + } + + cpu = flags >>32 ; +if (unlikely ((flags &BPF_F_CPU ) && cpu >=num_possible_cpus ())) +return -ERANGE ; + +return 0 ; }#endif /* _LINUX_BPF_H */
include/uapi/linux/bpf.h +2 -0
@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */
kernel/bpf/syscall.c +13 -11
@@ -131,12 +131,14 @@ bool bpf_map_write_active(const struct bpf_map *map)return atomic64_read (&map->writecnt) !=0 ; } -static u32 bpf_map_value_size (const struct bpf_map *map) -{ -if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || - map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) +static u32 bpf_map_value_size (const struct bpf_map *map,u64 flags) +{ +if (flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) +return round_up (map->value_size,8 ); +else if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || + map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY || + map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE )return round_up (map->value_size,8 ) *num_possible_cpus ();else if (IS_FD_MAP (map))return sizeof (u32);
@@ -1684,7 +1686,7 @@ static int map_lookup_elem(union bpf_attr *attr)if (IS_ERR (key))return PTR_ERR (key); - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->flags); err = -ENOMEM ; value =kvmalloc (value_size,GFP_USER | __GFP_NOWARN);
@@ -1751,7 +1753,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) goto err_put; } - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->flags); value =kvmemdup_bpfptr (uvalue, value_size);if (IS_ERR (value)) { err =PTR_ERR (value);
@@ -1951,7 +1953,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,if (err)return err; - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->batch.elem_flags); max_count = attr->batch.count;if (!max_count)
@@ -2010,7 +2012,7 @@ int generic_map_lookup_batch(struct bpf_map *map,if (err)return err; - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map, attr->batch.elem_flags); max_count = attr->batch.count;if (!max_count)
@@ -2132,7 +2134,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) goto err_put; } - value_size =bpf_map_value_size (map); + value_size =bpf_map_value_size (map,0 ); err = -ENOMEM ; value =kvmalloc (value_size,GFP_USER | __GFP_NOWARN);
tools/include/uapi/linux/bpf.h +2 -0
@@ -1372,6 +1372,8 @@ enum {BPF_NOEXIST =1 ,/* create new element if it didn't exist */ BPF_EXIST =2 ,/* update existing element */ BPF_F_LOCK =4 ,/* spin_lock-ed map_lookup/map_update */ +BPF_F_CPU =8 ,/* cpu flag for percpu maps, upper 32-bit of flags is a cpu number */ + BPF_F_ALL_CPUS = 16, /* update value across all CPUs for percpu maps */ };/* flags for BPF_MAP_CREATE command */
Signature · 1 line
2.50.1
PATCH v5 4/9 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps data copying
In reply to Leon HwangAdd BPF_F_CPU and BPF_F_ALL_CPUS flags support in the following functions:
* 'bpf_percpu_copy_data()'
* 'bpf_percpu_update_data()'
As a result, all percpu maps have to provide map flags for them.
--- include/linux/bpf-cgroup.h | 4 ++-- include/linux/bpf.h | 27 ++++++++++++++++++++++----- kernel/bpf/arraymap.c | 6 +++--- kernel/bpf/hashtab.c | 30 +++++++++++++++--------------- kernel/bpf/local_storage.c | 6 +++--- kernel/bpf/syscall.c | 6 +++--- 6 files changed, 48 insertions(+), 31 deletions(-)
include/linux/bpf-cgroup.h +2 -2
@@ -172,7 +172,7 @@ void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,void bpf_cgroup_storage_unlink (struct bpf_cgroup_storage *storage);int bpf_cgroup_storage_assign (struct bpf_prog_aux *aux,struct bpf_map *map); -int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_cgroup_storage_update (struct bpf_map *map,void *key,void *value,u64 flags);
@@ -467,7 +467,7 @@ static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(static inline void bpf_cgroup_storage_free (struct bpf_cgroup_storage *storage) {}static inline int bpf_percpu_cgroup_storage_copy (struct bpf_map *map,void *key, -void *value) { +void *value,u64 flags) {return 0 ; } static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
include/linux/bpf.h +22 -5
@@ -549,10 +549,16 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src#ifdef CONFIG_BPF_SYSCALL static inline void bpf_percpu_copy_data (struct bpf_map *map,void __percpu *pptr,void *value, -u32 size) +u32 size,u64 flags) {int cpu, off =0 ; +if (flags &BPF_F_CPU ) { + cpu = flags >>32 ; +copy_map_value_long (map, value,per_cpu_ptr (pptr, cpu)); +return ; + } +for_each_possible_cpu (cpu) {copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); off += size;
@@ -562,14 +568,25 @@ static inline void bpf_percpu_copy_data(struct bpf_map *map, void __percpu *pptrvoid bpf_obj_free_fields (const struct btf_record *rec,void *obj);static inline void bpf_percpu_update_data (struct bpf_map *map,void __percpu *pptr,void *value, -u32 size) +u32 size,u64 flags) {int cpu, off =0 ; +if (flags &BPF_F_CPU ) { + cpu = flags >>32 ; +bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); +bpf_long_memcpy (per_cpu_ptr (pptr, cpu), value, size); +return ; + } +for_each_possible_cpu (cpu) {bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu));bpf_long_memcpy (per_cpu_ptr (pptr, cpu), value + off, size); - off += size; +/* same user-provided value is used if BPF_F_ALL_CPUS is +* specified, otherwise value is an array of per-cpu values. +*/ +if (!(flags &BPF_F_ALL_CPUS )) + off += size; } }#endif
@@ -2722,8 +2739,8 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,struct bpf_func_state *caller, struct bpf_func_state *callee); -int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value); -int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value); +int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value,u64 flags); +int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value,u64 flags);int bpf_percpu_hash_update (struct bpf_map *map,void *key,void *value,u64 flags); int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
kernel/bpf/arraymap.c +3 -3
@@ -295,7 +295,7 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key,return per_cpu_ptr (array->pptrs[index & array->index_mask], cpu); } -int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value) +int bpf_percpu_array_copy (struct bpf_map *map,void *key,void *value,u64 map_flags) { struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;
@@ -312,7 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -bpf_percpu_copy_data (map, pptr, value, size); +bpf_percpu_copy_data (map, pptr, value, size, map_flags);rcu_read_unlock ();return 0 ; }
@@ -405,7 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -bpf_percpu_update_data (map, pptr, value, size); +bpf_percpu_update_data (map, pptr, value, size, map_flags);rcu_read_unlock ();return 0 ; }
kernel/bpf/hashtab.c +15 -15
@@ -937,7 +937,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) }static void pcpu_copy_value (struct bpf_htab *htab,void __percpu *pptr, -void *value,bool onallcpus) +void *value,bool onallcpus,u64 map_flags) {if (!onallcpus) {/* copy true value_size bytes */
@@ -945,12 +945,12 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, }else {u32 size =round_up (htab->map.value_size,8 ); -bpf_percpu_update_data (&htab->map, pptr, value, size); +bpf_percpu_update_data (&htab->map, pptr, value, size, map_flags); } }static void pcpu_init_value (struct bpf_htab *htab,void __percpu *pptr, -void *value,bool onallcpus) +void *value,bool onallcpus,u64 map_flags) {/* When not setting the initial value on all cpus, zero-fill element * values for other cpus. Otherwise, bpf program has no way to ensure
@@ -968,7 +968,7 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,zero_map_value (&htab->map,per_cpu_ptr (pptr, cpu)); } }else { -pcpu_copy_value (htab, pptr, value, onallcpus); +pcpu_copy_value (htab, pptr, value, onallcpus, map_flags); } }
@@ -980,7 +980,7 @@ static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)static struct htab_elem *alloc_htab_elem (struct bpf_htab *htab,void *key,void *value,u32 key_size,u32 hash,bool percpu,bool onallcpus, -struct htab_elem *old_elem) +struct htab_elem *old_elem,u64 map_flags) {u32 size = htab->map.value_size;bool prealloc =htab_is_prealloc (htab);
@@ -1038,7 +1038,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, pptr = *(void __percpu **)ptr; } -pcpu_init_value (htab, pptr, value, onallcpus); +pcpu_init_value (htab, pptr, value, onallcpus, map_flags);if (!prealloc)htab_elem_set_ptr (l_new, key_size, pptr);
@@ -1143,7 +1143,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, } l_new =alloc_htab_elem (htab, key, value, key_size, hash, false, false, - l_old); + l_old, map_flags);if (IS_ERR (l_new)) {/* all pre-allocated elements are in use or memory exhausted */ ret =PTR_ERR (l_new);
@@ -1287,7 +1287,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,/* Update value in-place */ if (percpu) {pcpu_copy_value (htab,htab_elem_get_ptr (l_old, key_size), - value, onallcpus); + value, onallcpus, map_flags); }else {void **inner_map_pptr =htab_elem_value (l_old, key_size);
@@ -1296,7 +1296,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, } }else { l_new =alloc_htab_elem (htab, key, value, key_size, - hash, percpu, onallcpus,NULL ); + hash, percpu, onallcpus,NULL , map_flags);if (IS_ERR (l_new)) { ret =PTR_ERR (l_new); goto err;
@@ -1362,10 +1362,10 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,/* per-cpu hash map can update value in-place */ pcpu_copy_value (htab,htab_elem_get_ptr (l_old, key_size), - value, onallcpus); + value, onallcpus, map_flags); }else {pcpu_init_value (htab,htab_elem_get_ptr (l_new, key_size), - value, onallcpus); + value, onallcpus, map_flags);hlist_nulls_add_head_rcu (&l_new->hash_node, head); l_new =NULL ; }
@@ -1608,7 +1608,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,void __percpu *pptr; pptr =htab_elem_get_ptr (l, key_size); -bpf_percpu_copy_data (&htab->map, pptr, value, roundup_value_size); +bpf_percpu_copy_data (&htab->map, pptr, value, roundup_value_size,0 /* map_flags */ ); }else {void *src =htab_elem_value (l, map->key_size);
@@ -1796,7 +1796,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,void __percpu *pptr; pptr =htab_elem_get_ptr (l, map->key_size); -bpf_percpu_copy_data (&htab->map, pptr, dst_val, size); +bpf_percpu_copy_data (&htab->map, pptr, dst_val, size, elem_map_flags); }else { value =htab_elem_value (l, key_size);if (is_fd_htab (htab)) {
@@ -2351,7 +2351,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *kreturn NULL ; } -int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value) +int bpf_percpu_hash_copy (struct bpf_map *map,void *key,void *value,u64 map_flags) {struct htab_elem *l;void __percpu *pptr;
@@ -2371,7 +2371,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)* eviction heuristics when user space does a map walk. */ pptr =htab_elem_get_ptr (l, map->key_size); -bpf_percpu_copy_data (map, pptr, value, size); +bpf_percpu_copy_data (map, pptr, value, size, map_flags); ret =0 ; out:rcu_read_unlock ();
kernel/bpf/local_storage.c +3 -3
@@ -180,7 +180,7 @@ static long cgroup_storage_update_elem(struct bpf_map *map, void *key, }int bpf_percpu_cgroup_storage_copy (struct bpf_map *_map,void *key, -void *value) +void *value,u64 map_flags) {struct bpf_cgroup_storage_map *map =map_to_storage (_map);struct bpf_cgroup_storage *storage;
@@ -200,7 +200,7 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,*/ size =round_up (_map->value_size,8 ); pptr = storage->percpu_buf; -bpf_percpu_copy_data (_map, pptr, value, size); +bpf_percpu_copy_data (_map, pptr, value, size, map_flags);rcu_read_unlock ();return 0 ; }
@@ -231,7 +231,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,*/ size =round_up (_map->value_size,8 ); pptr = storage->percpu_buf; -bpf_percpu_update_data (_map, pptr, value, size); +bpf_percpu_update_data (_map, pptr, value, size, map_flags);rcu_read_unlock ();return 0 ; }
kernel/bpf/syscall.c +3 -3
@@ -314,11 +314,11 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,bpf_disable_instrumentation ();if (map->map_type ==BPF_MAP_TYPE_PERCPU_HASH || map->map_type ==BPF_MAP_TYPE_LRU_PERCPU_HASH ) { - err =bpf_percpu_hash_copy (map, key, value); + err =bpf_percpu_hash_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_ARRAY ) { - err =bpf_percpu_array_copy (map, key, value); + err =bpf_percpu_array_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ) { - err =bpf_percpu_cgroup_storage_copy (map, key, value); + err =bpf_percpu_cgroup_storage_copy (map, key, value, flags); }else if (map->map_type ==BPF_MAP_TYPE_STACK_TRACE ) { err =bpf_stackmap_copy (map, key, value); }else if (IS_FD_ARRAY (map) ||IS_FD_PROG_ARRAY (map)) {
Signature · 1 line
2.50.1
PATCH v5 5/9 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps
In reply to Leon HwangIntroduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs.
Introduce support for the BPF_F_CPU flag in percpu_array maps to allow:
* update value for specified CPU for both update_elem and update_batch APIs.
* lookup value for specified CPU for both lookup_elem and lookup_batch APIs.
The BPF_F_CPU flag is passed via:
* map_flags of lookup_elem and update_elem APIs along with embedded cpu info.
* elem_flags of lookup_batch and update_batch APIs along with embedded cpu info.
--- include/linux/bpf.h | 7 ++++++- kernel/bpf/arraymap.c | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-)
include/linux/bpf.h +6 -1
@@ -3754,7 +3754,12 @@ struct bpf_prog *bpf_prog_find_from_stack(void);static inline bool bpf_map_supports_cpu_flags (enum bpf_map_type map_type) { -return false; +switch (map_type) { +case BPF_MAP_TYPE_PERCPU_ARRAY : +return true; +default : +return false; + } }static inline int bpf_map_check_op_flags (struct bpf_map *map,u64 flags,u64 allowed_flags)
kernel/bpf/arraymap.c +9 -3
@@ -301,6 +301,11 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_fu32 index = *(u32 *)key;void __percpu *pptr;u32 size; +int err; + + err =bpf_map_check_cpu_flags (map_flags, false); +if (unlikely (err)) +return err;if (unlikely (index >= array->map.max_entries))return -ENOENT ;
@@ -383,10 +388,11 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,u32 index = *(u32 *)key;void __percpu *pptr;u32 size; +int err; -if (unlikely (map_flags >BPF_EXIST )) -/* unknown flags */ -return -EINVAL ; + err =bpf_map_check_cpu_flags (map_flags, true); +if (unlikely (err)) +return err;if (unlikely (index >= array->map.max_entries))/* all elements were pre-allocated, cannot insert a new one */
Signature · 1 line
2.50.1
PATCH v5 6/9 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_hash and lru_percpu_hash maps
In reply to Leon HwangIntroduce BPF_F_ALL_CPUS flag support for percpu_hash and lru_percpu_hash maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs.
Introduce BPF_F_CPU flag support for percpu_hash and lru_percpu_hash maps to allow:
* update value for specified CPU for both update_elem and update_batch APIs.
* lookup value for specified CPU for both lookup_elem and lookup_batch APIs.
The BPF_F_CPU flag is passed via:
* map_flags along with embedded cpu info.
* elem_flags along with embedded cpu info.
--- include/linux/bpf.h | 2 ++ kernel/bpf/hashtab.c | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 10 deletions(-)
include/linux/bpf.h +2 -0
@@ -3756,6 +3756,8 @@ static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) {switch (map_type) {case BPF_MAP_TYPE_PERCPU_ARRAY : +case BPF_MAP_TYPE_PERCPU_HASH : +case BPF_MAP_TYPE_LRU_PERCPU_HASH :return true;default :return false;
kernel/bpf/hashtab.c +31 -10
66 lines not on this page · raw message
Signature · 1 line
2.50.1
PATCH v5 7/9 bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_cgroup_storage maps
In reply to Leon HwangIntroduce BPF_F_ALL_CPUS flag support for percpu_cgroup_storage maps to allow updating values for all CPUs with a single value for update_elem API.
Introduce BPF_F_CPU flag support for percpu_cgroup_storage maps to allow:
* update value for specified CPU for update_elem API.
* lookup value for specified CPU for lookup_elem API.
The BPF_F_CPU flag is passed via map_flags along with embedded cpu info.
--- include/linux/bpf.h | 1 + kernel/bpf/local_storage.c | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-)
include/linux/bpf.h +1 -0
@@ -3758,6 +3758,7 @@ static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)case BPF_MAP_TYPE_PERCPU_ARRAY :case BPF_MAP_TYPE_PERCPU_HASH :case BPF_MAP_TYPE_LRU_PERCPU_HASH : +case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE :return true;default :return false;
kernel/bpf/local_storage.c +11 -1
@@ -186,6 +186,11 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,struct bpf_cgroup_storage *storage;void __percpu *pptr;u32 size; +int err; + + err =bpf_map_check_cpu_flags (map_flags, false); +if (err) +return err;rcu_read_lock (); storage =cgroup_storage_lookup (map, key, false);
@@ -212,10 +217,15 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,struct bpf_cgroup_storage *storage;void __percpu *pptr;u32 size; +int err; -if (map_flags !=BPF_ANY && map_flags !=BPF_EXIST ) +if ((u32 )map_flags & ~(BPF_ANY |BPF_EXIST |BPF_F_CPU |BPF_F_ALL_CPUS ))return -EINVAL ; + err =bpf_map_check_cpu_flags (map_flags, true); +if (err) +return err; +rcu_read_lock (); storage =cgroup_storage_lookup (map, key, false);if (!storage) {
Signature · 1 line
2.50.1
PATCH v5 8/9 libbpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu maps
In reply to Leon HwangAdd libbpf support for the BPF_F_CPU flag for percpu maps by embedding the cpu info into the high 32 bits of:
1. **flags**: bpf_map_lookup_elem_flags(), bpf_map__lookup_elem(), bpf_map_update_elem() and bpf_map__update_elem()
2. **opts->elem_flags**: bpf_map_lookup_batch() and bpf_map_update_batch()
And the flag can be BPF_F_ALL_CPUS, but cannot be
'BPF_F_CPU | BPF_F_ALL_CPUS'.
Behavior:
* If the flag is BPF_F_ALL_CPUS, the update is applied across all CPUs.
* If the flag is BPF_F_CPU, it updates value only to the specified CPU.
* If the flag is BPF_F_CPU, lookup value only from the specified CPU.
* lookup does not support BPF_F_ALL_CPUS.
--- tools/lib/bpf/bpf.h | 8 ++++++++ tools/lib/bpf/libbpf.c | 26 ++++++++++++++++++++------ tools/lib/bpf/libbpf.h | 21 ++++++++------------- 3 files changed, 36 insertions(+), 19 deletions(-)
tools/lib/bpf/bpf.h +8 -0
@@ -286,6 +286,14 @@ LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch,* Update spin_lock-ed map elements. This must be * specified if the map value contains a spinlock. * +* **BPF_F_CPU** +* As for percpu maps, update value on the specified CPU. And the cpu +* info is embedded into the high 32 bits of **opts->elem_flags**. +* +* **BPF_F_ALL_CPUS** +* As for percpu maps, update value across all CPUs. This flag cannot +* be used with BPF_F_CPU at the same time. +* * @param fd BPF map file descriptor * @param keys pointer to an array of *count* keys * @param values pointer to an array of *count* values
tools/lib/bpf/libbpf.c +20 -6
@@ -10603,7 +10603,7 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) }static int validate_map_op (const struct bpf_map *map,size_t key_sz, -size_t value_sz,bool check_value_sz) +size_t value_sz,bool check_value_sz,__u64 flags) {if (!map_is_created (map))/* map is not yet created */ return -ENOENT ;
@@ -10630,6 +10630,20 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,int num_cpu =libbpf_num_possible_cpus ();size_t elem_sz =roundup (map->def.value_size,8 ); +if (flags & (BPF_F_CPU |BPF_F_ALL_CPUS )) { +if ((flags &BPF_F_CPU ) && (flags &BPF_F_ALL_CPUS )) { +pr_warn ("map '%s': can't use BPF_F_CPU and BPF_F_ALL_CPUS at the same time\n" , + map->name); +return -EINVAL ; + } +if (value_sz != elem_sz) { +pr_warn ("map '%s': unexpected value size %zu provided for per-CPU map, expected %zu\n" , + map->name, value_sz, elem_sz); +return -EINVAL ; + } +break ; + } +if (value_sz != num_cpu * elem_sz) {pr_warn ("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n" , map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
@@ -10654,7 +10668,7 @@ int bpf_map__lookup_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10667,7 +10681,7 @@ int bpf_map__update_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10679,7 +10693,7 @@ int bpf_map__delete_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ); + err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ , flags);if (err)return libbpf_err (err);
@@ -10692,7 +10706,7 @@ int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz, value_sz, true); + err =validate_map_op (map, key_sz, value_sz, true, flags);if (err)return libbpf_err (err);
@@ -10704,7 +10718,7 @@ int bpf_map__get_next_key(const struct bpf_map *map, {int err; - err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ); + err =validate_map_op (map, key_sz,0 , false/* check_value_sz */ ,0 );if (err)return libbpf_err (err);
tools/lib/bpf/libbpf.h +8 -13
@@ -1196,12 +1196,13 @@ LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map);* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** * @param value pointer to memory in which looked up value will be stored * @param value_sz size in byte of value data memory; it has to match BPF map -* definition's **value_size**. For per-CPU BPF maps value size has to be -* a product of BPF map value size and number of possible CPUs in the system -* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for -* per-CPU values value size has to be aligned up to closest 8 bytes for -* alignment reasons, so expected size is: `round_up(value_size, 8) -* * libbpf_num_possible_cpus()`. +* definition's **value_size**. For per-CPU BPF maps, value size can be +* `round_up(value_size, 8)` if **BPF_F_CPU** or **BPF_F_ALL_CPUS** is +* specified in **flags**, otherwise a product of BPF map value size and number +* of possible CPUs in the system (could be fetched with +* **libbpf_num_possible_cpus()**). Note else that for per-CPU values value +* size has to be aligned up to closest 8 bytes, so expected size is: +* `round_up(value_size, 8) * libbpf_num_possible_cpus()`. * @flags extra flags passed to kernel for this operation * @return 0, on success; negative error, otherwise *
@@ -1219,13 +1220,7 @@ LIBBPF_API int bpf_map__lookup_elem(const struct bpf_map *map,* @param key pointer to memory containing bytes of the key * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** * @param value pointer to memory containing bytes of the value -* @param value_sz size in byte of value data memory; it has to match BPF map -* definition's **value_size**. For per-CPU BPF maps value size has to be -* a product of BPF map value size and number of possible CPUs in the system -* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for -* per-CPU values value size has to be aligned up to closest 8 bytes for -* alignment reasons, so expected size is: `round_up(value_size, 8) -* * libbpf_num_possible_cpus()`. +* @param value_sz refer to **bpf_map__lookup_elem**'s description.' * @flags extra flags passed to kernel for this operation * @return 0, on success; negative error, otherwise *
Signature · 1 line
2.50.1
PATCH v5 9/9 selftests/bpf: Add cases to test BPF_F_CPU and BPF_F_ALL_CPUS flags
In reply to Leon HwangAdd test coverage for the new BPF_F_CPU and BPF_F_ALL_CPUS flags support in percpu maps. The following APIs are exercised:
* bpf_map_update_batch()
* bpf_map_lookup_batch()
* bpf_map_update_elem()
* bpf_map__update_elem()
* bpf_map_lookup_elem_flags()
* bpf_map__lookup_elem()
cd tools/testing/selftests/bpf/
./test_progs -t percpu_alloc
253/13 percpu_alloc/cpu_flag_percpu_array:OK 253/14 percpu_alloc/cpu_flag_percpu_hash:OK 253/15 percpu_alloc/cpu_flag_lru_percpu_hash:OK 253/16 percpu_alloc/cpu_flag_percpu_cgroup_storage:OK 253 percpu_alloc:OK
Summary: 1/16 PASSED, 0 SKIPPED, 0 FAILED
--- .../selftests/bpf/prog_tests/percpu_alloc.c | 233 ++++++++++++++++++ .../selftests/bpf/progs/percpu_alloc_array.c | 32 +++ 2 files changed, 265 insertions(+)
tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +233 -0
@@ -1,5 +1,6 @@// SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "cgroup_helpers.h" #include "percpu_alloc_array.skel.h" #include "percpu_alloc_cgrp_local_storage.skel.h" #include "percpu_alloc_fail.skel.h"
@@ -115,6 +116,230 @@ static void test_failure(void) {RUN_TESTS (percpu_alloc_fail); } +static void test_percpu_map_op_cpu_flag (struct bpf_map *map,void *keys,size_t key_sz, +u32 max_entries,bool test_batch) +{ +int i, j, cpu, map_fd, value_size, nr_cpus, err; +u64 *values =NULL , batch =0 , flags; +const u64 value =0xDEADC0DE ; +size_t value_sz =sizeof (u64); +u32 count; +LIBBPF_OPTS (bpf_map_batch_opts, batch_opts); + + nr_cpus =libbpf_num_possible_cpus (); +if (!ASSERT_GT (nr_cpus,0 ,"libbpf_num_possible_cpus" )) +return ; + + value_size = value_sz * nr_cpus; + values =calloc (max_entries, value_size); +if (!ASSERT_OK_PTR (values,"calloc values" )) + goto out; +memset (values,0 , value_size * max_entries); + + map_fd =bpf_map__fd (map); + flags =BPF_F_CPU |BPF_F_ALL_CPUS ; + err =bpf_map_lookup_elem_flags (map_fd, keys, values, flags); +if (!ASSERT_ERR (err,"bpf_map_lookup_elem_flags err" )) + goto out; + + err =bpf_map_update_elem (map_fd, keys, values, flags); +if (!ASSERT_ERR (err,"bpf_map_update_elem err" )) + goto out; + + flags = (u64 )nr_cpus <<32 |BPF_F_CPU ; + err =bpf_map_update_elem (map_fd, keys, values, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_update_elem -ERANGE" )) + goto out; + + err =bpf_map__update_elem (map, keys, key_sz, values, value_sz, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map__update_elem -ERANGE" )) + goto out; + + err =bpf_map_lookup_elem_flags (map_fd, keys, values, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_lookup_elem_flags -ERANGE" )) + goto out; + + err =bpf_map__lookup_elem (map, keys, key_sz, values, value_sz, flags); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map__lookup_elem -ERANGE" )) + goto out; + +for (cpu =0 ; cpu < nr_cpus; cpu++) { +/* clear value on all cpus */ + values[0 ] =0 ; + flags =BPF_F_ALL_CPUS ; +for (i =0 ; i < max_entries; i++) { + err =bpf_map__update_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__update_elem all_cpus" )) + goto out; + } + +/* update value on specified cpu */ +for (i =0 ; i < max_entries; i++) { + values[0 ] = value; + flags = (u64 )cpu <<32 |BPF_F_CPU ; + err =bpf_map__update_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__update_elem specified cpu" )) + goto out; + +/* lookup then check value on CPUs */ +for (j =0 ; j < nr_cpus; j++) { + flags = (u64 )j <<32 |BPF_F_CPU ; + err =bpf_map__lookup_elem (map, keys + i * key_sz, key_sz, values, + value_sz, flags); +if (!ASSERT_OK (err,"bpf_map__lookup_elem specified cpu" )) + goto out; +if (!ASSERT_EQ (values[0 ], j != cpu ?0 : value, +"bpf_map__lookup_elem value on specified cpu" )) + goto out; + } + } + } + +if (!test_batch) + goto out; + + batch_opts.elem_flags = (u64 )nr_cpus <<32 |BPF_F_CPU ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_EQ (err, -ERANGE ,"bpf_map_update_batch -ERANGE" )) + goto out; + +for (cpu =0 ; cpu < nr_cpus; cpu++) { +memset (values,0 , max_entries * value_size); + +/* clear values across all CPUs */ + batch_opts.elem_flags =BPF_F_ALL_CPUS ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_OK (err,"bpf_map_update_batch all_cpus" )) + goto out; + +/* update values on specified CPU */ +for (i =0 ; i < max_entries; i++) + values[i] = value; + + batch_opts.elem_flags = (u64 )cpu <<32 |BPF_F_CPU ; + err =bpf_map_update_batch (map_fd, keys, values, &max_entries, &batch_opts); +if (!ASSERT_OK (err,"bpf_map_update_batch specified cpu" )) + goto out; + +/* lookup values on specified CPU */ +memset (values,0 , max_entries * value_sz); + err =bpf_map_lookup_batch (map_fd,NULL , &batch, keys, values, &count, &batch_opts); +if (!ASSERT_TRUE (!err || err == -ENOENT ,"bpf_map_lookup_batch specified cpu" )) + goto out; + +for (i =0 ; i < max_entries; i++) +if (!ASSERT_EQ (values[i], value,"value on specified cpu" )) + goto out; + +/* lookup values from all CPUs */ + batch_opts.elem_flags =0 ; +memset (values,0 , max_entries * value_size); + err =bpf_map_lookup_batch (map_fd,NULL , &batch, keys, values, &count, &batch_opts); +if (!ASSERT_TRUE (!err || err == -ENOENT ,"bpf_map_lookup_batch all_cpus" )) + goto out; + +for (i =0 ; i < max_entries; i++) { +for (j =0 ; j < nr_cpus; j++) { +if (!ASSERT_EQ (values[i*nr_cpus + j], j != cpu ?0 : value, +"value on specified cpu" )) + goto out; + } + } + } + +out: +if (values) +free (values); +} + +static void test_percpu_map_cpu_flag (enum bpf_map_type map_type) +{ +struct percpu_alloc_array *skel; +size_t key_sz =sizeof (int ); +int *keys =NULL , i, err; +struct bpf_map *map; +u32 max_entries; + + skel =percpu_alloc_array__open (); +if (!ASSERT_OK_PTR (skel,"percpu_alloc_array__open" )) +return ; + + map = skel->maps.percpu; +bpf_map__set_type (map, map_type); + + err =percpu_alloc_array__load (skel); +if (!ASSERT_OK (err,"test_percpu_alloc__load" )) + goto out; + + max_entries =bpf_map__max_entries (map); + keys =calloc (max_entries, key_sz); +if (!ASSERT_OK_PTR (keys,"calloc keys" )) + goto out; + +for (i =0 ; i < max_entries; i++) + keys[i] = i; + +test_percpu_map_op_cpu_flag (map, keys, key_sz, max_entries, true); +out: +if (keys) +free (keys); +percpu_alloc_array__destroy (skel); +} + +static void test_percpu_array_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_PERCPU_ARRAY ); +} + +static void test_percpu_hash_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_PERCPU_HASH ); +} + +static void test_lru_percpu_hash_cpu_flag (void ) +{ +test_percpu_map_cpu_flag (BPF_MAP_TYPE_LRU_PERCPU_HASH ); +} + +static void test_percpu_cgroup_storage_cpu_flag (void ) +{ +struct bpf_cgroup_storage_key key; +struct percpu_alloc_array *skel; +int cgroup =-1 , prog_fd, err; +struct bpf_map *map; + + skel =percpu_alloc_array__open_and_load (); +if (!ASSERT_OK_PTR (skel,"percpu_alloc_array__open_and_load" )) +return ; + + cgroup =create_and_get_cgroup ("/cg_percpu" ); +if (!ASSERT_GE (cgroup,0 ,"create_and_get_cgroup" )) + goto out; + + err =join_cgroup ("/cg_percpu" ); +if (!ASSERT_OK (err,"join_cgroup" )) + goto out; + + prog_fd =bpf_program__fd (skel->progs.cgroup_egress); + err =bpf_prog_attach (prog_fd, cgroup,BPF_CGROUP_INET_EGRESS ,0 ); +if (!ASSERT_OK (err,"bpf_prog_attach" )) + goto out; + + map = skel->maps.percpu_cgroup_storage; + err =bpf_map_get_next_key (bpf_map__fd (map),NULL , &key); +if (!ASSERT_OK (err,"bpf_map_get_next_key" )) + goto out; + +test_percpu_map_op_cpu_flag (map, &key,sizeof (key),1 , false); +out: +bpf_prog_detach2 (-1 , cgroup,BPF_CGROUP_INET_EGRESS ); +close (cgroup); +cleanup_cgroup_environment (); +percpu_alloc_array__destroy (skel); +} +void test_percpu_alloc (void ) {if (test__start_subtest ("array" ))
@@ -125,4 +350,12 @@ void test_percpu_alloc(void)test_cgrp_local_storage ();if (test__start_subtest ("failure_tests" ))test_failure (); +if (test__start_subtest ("cpu_flag_percpu_array" )) +test_percpu_array_cpu_flag (); +if (test__start_subtest ("cpu_flag_percpu_hash" )) +test_percpu_hash_cpu_flag (); +if (test__start_subtest ("cpu_flag_lru_percpu_hash" )) +test_lru_percpu_hash_cpu_flag (); +if (test__start_subtest ("cpu_flag_percpu_cgroup_storage" )) +test_percpu_cgroup_storage_cpu_flag (); }
tools/testing/selftests/bpf/progs/percpu_alloc_array.c +32 -0
@@ -187,4 +187,36 @@ int BPF_PROG(test_array_map_10)return 0 ; } +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, u64); +} percpu SEC(".maps"); + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG (test_percpu_array ,int x) +{ +u64 value =0xDEADC0DE ; +int key =0 ; + +bpf_map_update_elem (&percpu, &key, &value,BPF_ANY ); +return 0 ; +} + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); + __type(key, struct bpf_cgroup_storage_key); + __type(value, u64); +} percpu_cgroup_storage SEC(".maps"); + +SEC("cgroup_skb/egress") +int cgroup_egress (struct __sk_buff *skb) +{ +u64 *val =bpf_get_local_storage (&percpu_cgroup_storage,0 ); + +__sync_fetch_and_add (val,1 ); +return 1 ; +} + char _license[] SEC("license") = "GPL";
Signature · 1 line
2.50.1