bpf: Generalize data copying for percpu maps
PATCH 1/1 bpf: Generalize data copying for percpu maps
While adding support for the BPF_F_CPU and BPF_F_ALL_CPUS flags, the data copying logic of the following percpu map types needs to be updated:
* percpu_array
* percpu_hash
* lru_percpu_hash
* percpu_cgroup_storage
Following Andrii’s suggestion[0], this patch refactors the data copying logic by introducing two helpers:
* bpf_percpu_copy_to_user()
* bpf_percpu_copy_from_user()
This prepares the codebase for the upcoming CPU flag support.
[0] https://lore.kernel.org/bpf/[email protected]/
--- include/linux/bpf.h | 29 ++++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 20 +++----------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 42 deletions(-)
include/linux/bpf.h +28 -1
@@ -547,6 +547,34 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); } +#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_to_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value + off); + off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj); + +static inline void bpf_percpu_copy_from_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); +bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;
@@ -2417,7 +2445,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);
kernel/bpf/arraymap.c +2 -12
@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))
@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }
@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))
@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_copy_from_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }
kernel/bpf/hashtab.c +3 -17
@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu; -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_copy_from_user (&htab->map, pptr, value, size); } }
@@ -1802,15 +1798,10 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,memcpy (dst_key, l->key, key_size);if (is_percpu) { -int off =0 , cpu;void __percpu *pptr; pptr =htab_elem_get_ptr (l, map->key_size); -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map, dst_val + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (&htab->map, dst_val + off); - off += size; - } +bpf_percpu_copy_to_user (&htab->map, pptr, dst_val, size); }else { value =htab_elem_value (l, key_size);if (is_fd_htab (htab)) {
@@ -2370,7 +2361,6 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)struct htab_elem *l;void __percpu *pptr;int ret = -ENOENT ; -int cpu, off =0 ;u32 size;/* per_cpu areas are zero-filled and bpf programs can only
@@ -2386,11 +2376,7 @@ int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value)* eviction heuristics when user space does a map walk. */ pptr =htab_elem_get_ptr (l, map->key_size); -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size); ret =0 ; out:rcu_read_unlock ();
kernel/bpf/local_storage.c +6 -12
@@ -184,7 +184,7 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key, {struct bpf_cgroup_storage_map *map =map_to_storage (_map);struct bpf_cgroup_storage *storage; -int cpu, off =0 ; +void __percpu *pptr;u32 size;rcu_read_lock ();
@@ -199,11 +199,8 @@ int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,* will not leak any kernel data */ size =round_up (_map->value_size,8 ); -for_each_possible_cpu (cpu) { -bpf_long_memcpy (value + off, -per_cpu_ptr (storage->percpu_buf, cpu), size); - off += size; - } + pptr = storage->percpu_buf; +bpf_percpu_copy_to_user (_map, pptr, value, size);rcu_read_unlock ();return 0 ; }
@@ -213,7 +210,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key, {struct bpf_cgroup_storage_map *map =map_to_storage (_map);struct bpf_cgroup_storage *storage; -int cpu, off =0 ; +void __percpu *pptr;u32 size;if (map_flags !=BPF_ANY && map_flags !=BPF_EXIST )
@@ -233,11 +230,8 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,* so no kernel data leaks possible */ size =round_up (_map->value_size,8 ); -for_each_possible_cpu (cpu) { -bpf_long_memcpy (per_cpu_ptr (storage->percpu_buf, cpu), - value + off, size); - off += size; - } + pptr = storage->percpu_buf; +bpf_percpu_copy_from_user (_map, pptr, value, size);rcu_read_unlock ();return 0 ; }
--
2.50.1
Quoted 126 lines · Leon Hwang
While adding support for the
BPF_F_CPUandBPF_F_ALL_CPUSflags, the data copying logic of the following percpu map types needs to be updated:*
percpu_array
*percpu_hash
*lru_percpu_hash
*percpu_cgroup_storageFollowing Andrii’s suggestion[0], this patch refactors the data copying logic by introducing two helpers:
*
bpf_percpu_copy_to_user()
*bpf_percpu_copy_from_user()This prepares the codebase for the upcoming CPU flag support.
[0] https://lore.kernel.org/bpf/[email protected]/
Suggested-by Andrii Nakryiko <[email protected]>Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 29 ++++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 20 +++----------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 42 deletions(-)include/linux/bpf.h +28
-1@@ -547,6 +547,34 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); } +#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_to_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value + off); + off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj); + +static inline void bpf_percpu_copy_from_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); +bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;@@ -2417,7 +2445,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);kernel/bpf/arraymap.c +2
-12@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_copy_from_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }kernel/bpf/hashtab.c +0
-0
kernel/bpf/hashtab.c static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, · line 948
Quoted 10 lines
kernel/bpf/hashtab.c +1
-5@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu; -for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_copy_from_user (&htab->map, pptr, value, size);
This is not a refactor. There is a significant change in the logic. Why is it needed? Bug fix or introducing a bug?
The names to_user and from_user are wrong.
There is no user space memory involved.
pw-bot: cr
Quoted 6 lines · Alexei Starovoitov
Quoted 4 lines · Leon Hwang
While adding support for the
BPF_F_CPUandBPF_F_ALL_CPUSflags, the data copying logic of the following percpu map types needs to be updated:
Quoted 6 lines
Quoted 6 lines
*
percpu_array
*percpu_hash
*lru_percpu_hash
*percpu_cgroup_storageFollowing Andrii’s suggestion[0], this patch refactors the data copying
as flattering as that is, "Andrii's suggestion" is no justification why the patch is correct :)
Quoted 26 lines
Quoted 26 lines
logic by introducing two helpers:
*
bpf_percpu_copy_to_user()
*bpf_percpu_copy_from_user()This prepares the codebase for the upcoming CPU flag support.
[0] https://lore.kernel.org/bpf/[email protected]/
Suggested-by Andrii Nakryiko <[email protected]>Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 29 ++++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 20 +++----------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 42 deletions(-)include/linux/bpf.h +0
-0@@ -547,6 +547,34 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); }
Quoted 9 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 558
Quoted 9 lines
include/linux/bpf.h +9
-0+#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_to_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value + off);
I still maintain that this makes zero sense... value+off is memory that we'll copy_to_user, why are we setting refcount to 1, or rb_node/list_node to "proper empty node" is absolutely not clear... it feels like we can drop check_and_init_map_value() altogether and be absolutely no worse. If anything, memset(0) would be nicer, but I guess we didn't have it to begin with, so no need to add it now.
Quoted 5 lines
Quoted 5 lines
include/linux/bpf.h +5
-0+ off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj);
Quoted 8 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 571
Quoted 8 lines
include/linux/bpf.h +8
-0+ +static inline void bpf_percpu_copy_from_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off);
copy_map_value_long is generalization of bpf_long_memcpy, and so it would be good to call this out to explain why your refactoring is correct
Quoted 73 lines
Quoted 73 lines
include/linux/bpf.h +6
-1+bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;@@ -2417,7 +2445,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);kernel/bpf/arraymap.c +2
-12@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_copy_from_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }kernel/bpf/hashtab.c +0
-1@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu;
Quoted 8 lines
kernel/bpf/hashtab.c static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, · line 948
Quoted 5 lines
kernel/bpf/hashtab.c +1
-4-for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_copy_from_user (&htab->map, pptr, value, size);This is not a refactor. There is a significant change in the logic. Why is it needed? Bug fix or introducing a bug?
this is preparation for that BPF_F_CPU/BPF_F_ALLCPUS, but I agree that it would be better to include as preparatory patch in the actual patch set
Quoted 3 lines
The names
to_userandfrom_userare wrong.
There is no user space memory involved.
This was my suggestion because we either are copying user-supplied data or copying data back to user. Strictly speaking it's all kernel memory (copy_from_user/copy_to_user is done afterwards by the caller), but that's the intent.
Maybe "copy_in" and "copy_out" would be better, I don't know. But there is certainly a direction here w.r.t. user space provided data (note, this is not BPF program-side logic).
Quoted 2 lines
pw-bot: cr
Quoted 13 lines · Andrii Nakryiko
Quoted 11 lines · Alexei Starovoitov
Quoted 9 lines · Leon Hwang
While adding support for the
BPF_F_CPUandBPF_F_ALL_CPUSflags, the data copying logic of the following percpu map types needs to be updated:*
percpu_array
*percpu_hash
*lru_percpu_hash
*percpu_cgroup_storage
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Following Andrii’s suggestion[0], this patch refactors the data copying
as flattering as that is, "Andrii's suggestion" is no justification why the patch is correct :)
:)
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
logic by introducing two helpers:
*
bpf_percpu_copy_to_user()
*bpf_percpu_copy_from_user()This prepares the codebase for the upcoming CPU flag support.
[0] https://lore.kernel.org/bpf/[email protected]/
Suggested-by Andrii Nakryiko <[email protected]>Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 29 ++++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 20 +++----------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 42 deletions(-)include/linux/bpf.h +0
-0@@ -547,6 +547,34 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); }
Quoted 17 lines
Quoted 9 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 558
Quoted 9 lines
include/linux/bpf.h +9
-0+#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_to_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value + off);I still maintain that this makes zero sense... value+off is memory that we'll
copy_to_user, why are we setting refcount to 1, or rb_node/list_node to "proper empty node" is absolutely not clear... it feels like we can dropcheck_and_init_map_value()altogether and be absolutely no worse. If anything, memset(0) would be nicer, but I guess we didn't have it to begin with, so no need to add it now.
Agreed.
As 'copy_map_value_long()' won't copy those fields, 'check_and_init_map_value()' is unnecessary here.
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
include/linux/bpf.h +9
-0+ off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj); + +static inline void bpf_percpu_copy_from_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{
Quoted 9 lines
Quoted 4 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 571
Quoted 4 lines
include/linux/bpf.h +4
-0+int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off);
copy_map_value_longis generalization ofbpf_long_memcpy, and so it would be good to call this out to explain why your refactoring is correct
No.
It shouldn't call bpf_long_memcpy() before bpf_obj_free_fields(), or it will overwrite those fields data used for bpf_obj_free_fields().
It would be better to call bpf_obj_free_fields() then bpf_long_memcpy().
Quoted 82 lines
Quoted 73 lines
Quoted 73 lines
include/linux/bpf.h +6
-1+bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;@@ -2417,7 +2445,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);kernel/bpf/arraymap.c +2
-12@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_copy_from_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }kernel/bpf/hashtab.c +0
-1@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu;Quoted 8 lines
kernel/bpf/hashtab.c static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, · line 948
Quoted 5 lines
kernel/bpf/hashtab.c +1
-4-for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_copy_from_user (&htab->map, pptr, value, size);This is not a refactor. There is a significant change in the logic. Why is it needed? Bug fix or introducing a bug?
Quoted 4 lines
this is preparation for that BPF_F_CPU/BPF_F_ALLCPUS, but I agree that it would be better to include as preparatory patch in the actual patch set
Ack.
I'll move this patch into the patch set of BPF_F_CPU/BPF_F_ALLCPUS flags.
Quoted 4 lines
Quoted 3 lines
The names
to_userandfrom_userare wrong.
There is no user space memory involved.
Quoted 9 lines
This was my suggestion because we either are copying user-supplied data or copying data back to user. Strictly speaking it's all kernel memory (copy_from_user/copy_to_user is done afterwards by the caller), but that's the intent.
Maybe "
copy_in" and "copy_out" would be better, I don't know. But there is certainly a direction here w.r.t. user space provided data (note, this is not BPF program-side logic).
'bpf_percpu_copy_data()' and 'bpf_percpu_update_data()' would be better, as "copy_data" is used in those 'bpf_percpu_*_copy()' functions and "update_data" is used in those 'bpf_percpu_*_update()' functions.
Thanks,
Leon
Quoted 93 lines · Leon Hwang
Quoted 13 lines · Andrii Nakryiko
Quoted 11 lines · Alexei Starovoitov
Quoted 9 lines · Leon Hwang
While adding support for the
BPF_F_CPUandBPF_F_ALL_CPUSflags, the data copying logic of the following percpu map types needs to be updated:*
percpu_array
*percpu_hash
*lru_percpu_hash
*percpu_cgroup_storageQuoted 5 lines
Quoted 1 line
Quoted 1 line
Following Andrii’s suggestion[0], this patch refactors the data copying
as flattering as that is, "Andrii's suggestion" is no justification why the patch is correct :)
:)
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
logic by introducing two helpers:
*
bpf_percpu_copy_to_user()
*bpf_percpu_copy_from_user()This prepares the codebase for the upcoming CPU flag support.
[0] https://lore.kernel.org/bpf/[email protected]/
Suggested-by Andrii Nakryiko <[email protected]>Signed-off-by Leon Hwang <[email protected]>--- include/linux/bpf.h | 29 ++++++++++++++++++++++++++++- kernel/bpf/arraymap.c | 14 ++------------ kernel/bpf/hashtab.c | 20 +++----------------- kernel/bpf/local_storage.c | 18 ++++++------------ 4 files changed, 39 insertions(+), 42 deletions(-)include/linux/bpf.h +0
-0@@ -547,6 +547,34 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *srcbpf_obj_memcpy (map->record, dst, src, map->value_size, true); }Quoted 17 lines
Quoted 9 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 558
Quoted 9 lines
include/linux/bpf.h +9
-0+#ifdef CONFIG_BPF_SYSCALL +static inline void bpf_percpu_copy_to_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{ +int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); +check_and_init_map_value (map, value + off);I still maintain that this makes zero sense... value+off is memory that we'll
copy_to_user, why are we setting refcount to 1, or rb_node/list_node to "proper empty node" is absolutely not clear... it feels like we can dropcheck_and_init_map_value()altogether and be absolutely no worse. If anything, memset(0) would be nicer, but I guess we didn't have it to begin with, so no need to add it now.Agreed.
As '
copy_map_value_long()' won't copy those fields, 'check_and_init_map_value()' is unnecessary here.Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
include/linux/bpf.h +9
-0+ off += size; + } +} + +void bpf_obj_free_fields (const struct btf_record *rec,void *obj); + +static inline void bpf_percpu_copy_from_user (struct bpf_map *map,void __percpu *pptr,void *value, +u32 size) +{Quoted 9 lines
Quoted 4 lines
include/linux/bpf.h static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · line 571
Quoted 4 lines
include/linux/bpf.h +4
-0+int cpu, off =0 ; + +for_each_possible_cpu (cpu) { +copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off);
copy_map_value_longis generalization ofbpf_long_memcpy, and so it would be good to call this out to explain why your refactoring is correct
Quoted 4 lines
No.
It shouldn't call
bpf_long_memcpy()beforebpf_obj_free_fields(), or it will overwrite those fields data used forbpf_obj_free_fields().
I'm not exactly following what you are trying to say here. I didn't propose to replace copy_map_value_long() with bpf_long_memcpy(). bpf_long_memcpy() works only for those maps where we don't enable those special bpf_spinlock and other fields. copy_map_value_long() is a generalization of copying user-provided data into the map, skipping all those special fields.
Again, confused a bit here, sorry.
Quoted 2 lines
It would be better to call
bpf_obj_free_fields()thenbpf_long_memcpy().
I agree, but not sure it makes any practical difference.
Quoted 101 lines
Quoted 83 lines
Quoted 74 lines
Quoted 74 lines
include/linux/bpf.h +6
-1+bpf_obj_free_fields (map->record,per_cpu_ptr (pptr, cpu)); + off += size; + } +} +#endif +static inline void bpf_obj_swap_uptrs (const struct btf_record *rec,void *dst,void *src) {unsigned long *src_uptr, *dst_uptr;@@ -2417,7 +2445,6 @@ struct btf_record *btf_record_dup(const struct btf_record *rec);bool btf_record_equal (const struct btf_record *rec_a,const struct btf_record *rec_b);void bpf_obj_free_timer (const struct btf_record *rec,void *obj);void bpf_obj_free_workqueue (const struct btf_record *rec,void *obj); -void bpf_obj_free_fields (const struct btf_record *rec,void *obj);void __bpf_obj_drop_impl (void *p,const struct btf_record *rec,bool percpu);struct bpf_map *bpf_map_get (u32 ufd);kernel/bpf/arraymap.c +2
-12@@ -300,7 +300,6 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (index >= array->map.max_entries))@@ -313,11 +312,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map, value + off,per_cpu_ptr (pptr, cpu)); -check_and_init_map_value (map, value + off); - off += size; - } +bpf_percpu_copy_to_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }@@ -387,7 +382,6 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, struct bpf_array *array = container_of(map, struct bpf_array, map);u32 index = *(u32 *)key;void __percpu *pptr; -int cpu, off =0 ;u32 size;if (unlikely (map_flags >BPF_EXIST ))@@ -411,11 +405,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, size = array->elem_size;rcu_read_lock (); pptr = array->pptrs[index & array->index_mask]; -for_each_possible_cpu (cpu) { -copy_map_value_long (map,per_cpu_ptr (pptr, cpu), value + off); -bpf_obj_free_fields (array->map.record,per_cpu_ptr (pptr, cpu)); - off += size; - } +bpf_percpu_copy_from_user (map, pptr, value, size);rcu_read_unlock ();return 0 ; }kernel/bpf/hashtab.c +0
-1@@ -944,12 +944,8 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,copy_map_value (&htab->map,this_cpu_ptr (pptr), value); }else {u32 size =round_up (htab->map.value_size,8 ); -int off =0 , cpu;Quoted 8 lines
kernel/bpf/hashtab.c static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, · line 948
Quoted 5 lines
kernel/bpf/hashtab.c +1
-4-for_each_possible_cpu (cpu) { -copy_map_value_long (&htab->map,per_cpu_ptr (pptr, cpu), value + off); - off += size; - } +bpf_percpu_copy_from_user (&htab->map, pptr, value, size);This is not a refactor. There is a significant change in the logic. Why is it needed? Bug fix or introducing a bug?
Quoted 4 lines
this is preparation for that BPF_F_CPU/BPF_F_ALLCPUS, but I agree that it would be better to include as preparatory patch in the actual patch set
Ack.
I'll move this patch into the patch set of BPF_F_CPU/BPF_F_ALLCPUS flags.
Quoted 9 lines
Quoted 3 lines
The names
to_userandfrom_userare wrong.
There is no user space memory involved.This was my suggestion because we either are copying user-supplied data or copying data back to user. Strictly speaking it's all kernel memory (copy_from_user/copy_to_user is done afterwards by the caller), but that's the intent.
Quoted 8 lines
Quoted 4 lines
Maybe "
copy_in" and "copy_out" would be better, I don't know. But there is certainly a direction here w.r.t. user space provided data (note, this is not BPF program-side logic).'
bpf_percpu_copy_data()' and 'bpf_percpu_update_data()' would be better, as "copy_data" is used in those 'bpf_percpu_*_copy()' functions and "update_data" is used in those 'bpf_percpu_*_update()' functions.
And I have to keep looking up how "copy_data" is actually used to remind myself that it's for "lookup" to copy data *to user*, not copy *from user*. Which is why "copy_in" and "copy_out". But that's ok. My point is that I don't see copy vs update as a meaningful and clear distinction.
Quoted 4 lines
Thanks,
Leon