Skip to contentSkip to conversation

bpf: Fix percpu address space issues

PATCH Applied Uros Bizjak 3 participants 4 messages Aug 11, 2024 – Aug 22, 2024 BPF

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 bpf: Fix percpu address space issues

In arraymap.c:

In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1].

Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1].

In hashtab.c:

Assign the return value from bpf_mem_cache_alloc() to void pointer and cast the value to void __percpu ** (void pointer to percpu void pointer) before dereferencing.

In memalloc.c:

Explicitly declare __percpu variables.

Cast obj to void __percpu **.

In helpers.c:

Cast ptr in BPF_CALL_1 and BPF_CALL_2 from generic address space to __percpu address space via const uintptr_t [1].

Found by GCC's named address space checks.

There were no changes in the resulting object files.

[1] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name

Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>
Cc Andrii Nakryiko <[email protected]>
Cc Martin KaFai Lau <[email protected]>
Cc Eduard Zingerman <[email protected]>
Cc Song Liu <[email protected]>
Cc Yonghong Song <[email protected]>
Cc John Fastabend <[email protected]>
Cc KP Singh <[email protected]>
Cc Stanislav Fomichev <[email protected]>
Cc Hao Luo <[email protected]>
Cc Jiri Olsa <[email protected]>

---
v2: - cast return values from the __percpu address space to
the generic address space in bpf_array_map_seq_{start,next}().
- correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu *

---
 kernel/bpf/arraymap.c |  8 ++++----
 kernel/bpf/hashtab.c  |  9 +++++----
 kernel/bpf/helpers.c  |  4 ++--
 kernel/bpf/memalloc.c | 12 ++++++------
 4 files changed, 17 insertions(+), 16 deletions(-)
kernel/bpf/arraymap.c +4 -4
index 188e3c2effb2..a43e62e2a8bb 100644
@@ -600,7 +600,7 @@ static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
 	array = container_of(map, struct bpf_array, map);
 	index = info->index & array->index_mask;
 	if (info->percpu_value_buf)
-	       return array->pptrs[index];
+		return (void *)(uintptr_t)array->pptrs[index];
 	return array_map_elem_ptr(array, index);
 }
 
@@ -619,7 +619,7 @@ static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 	array = container_of(map, struct bpf_array, map);
 	index = info->index & array->index_mask;
 	if (info->percpu_value_buf)
-	       return array->pptrs[index];
+		return (void *)(uintptr_t)array->pptrs[index];
 	return array_map_elem_ptr(array, index);
 }
 
@@ -632,7 +632,7 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
 	struct bpf_iter_meta meta;
 	struct bpf_prog *prog;
 	int off = 0, cpu = 0;
-	void __percpu **pptr;
+	void __percpu *pptr;
 	u32 size;
 
 	meta.seq = seq;
@@ -648,7 +648,7 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
 		if (!info->percpu_value_buf) {
 			ctx.value = v;
 		} else {
-			pptr = v;
+			pptr = (void __percpu *)(uintptr_t)v;
 			size = array->elem_size;
 			for_each_possible_cpu(cpu) {
 				copy_map_value_long(map, info->percpu_value_buf + off,
kernel/bpf/hashtab.c +5 -4
index be1f64c20125..45c7195b65ba 100644
@@ -1049,14 +1049,15 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 			pptr = htab_elem_get_ptr(l_new, key_size);
 		} else {
 			/* alloc_percpu zero-fills */
-			pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
-			if (!pptr) {
+			void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
+
+			if (!ptr) {
 				bpf_mem_cache_free(&htab->ma, l_new);
 				l_new = ERR_PTR(-ENOMEM);
 				goto dec_count;
 			}
-			l_new->ptr_to_pptr = pptr;
-			pptr = *(void **)pptr;
+			l_new->ptr_to_pptr = ptr;
+			pptr = *(void __percpu **)ptr;
 		}
 
 		pcpu_init_value(htab, pptr, value, onallcpus);
kernel/bpf/helpers.c +2 -2
index d02ae323996b..dd7529153146 100644
@@ -715,7 +715,7 @@ BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 	if (cpu >= nr_cpu_ids)
 		return (unsigned long)NULL;
 
-	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
+	return (unsigned long)per_cpu_ptr((const void __percpu *)(const uintptr_t)ptr, cpu);
 }
 
 const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
@@ -728,7 +728,7 @@ const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
 
 BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
 {
-	return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
+	return (unsigned long)this_cpu_ptr((const void __percpu *)(const uintptr_t)percpu_ptr);
 }
 
 const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
kernel/bpf/memalloc.c +6 -6
index dec892ded031..b3858a76e0b3 100644
@@ -138,8 +138,8 @@ static struct llist_node notrace *__llist_del_first(struct llist_head *head)
 static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags)
 {
 	if (c->percpu_size) {
-		void **obj = kmalloc_node(c->percpu_size, flags, node);
-		void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
+		void __percpu **obj = kmalloc_node(c->percpu_size, flags, node);
+		void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
 
 		if (!obj || !pptr) {
 			free_percpu(pptr);
@@ -253,7 +253,7 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic)
 static void free_one(void *obj, bool percpu)
 {
 	if (percpu) {
-		free_percpu(((void **)obj)[1]);
+		free_percpu(((void __percpu **)obj)[1]);
 		kfree(obj);
 		return;
 	}
@@ -509,8 +509,8 @@ static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)
  */
 int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
 {
-	struct bpf_mem_caches *cc, __percpu *pcc;
-	struct bpf_mem_cache *c, __percpu *pc;
+	struct bpf_mem_caches *cc; struct bpf_mem_caches __percpu *pcc;
+	struct bpf_mem_cache *c; struct bpf_mem_cache __percpu *pc;
 	struct obj_cgroup *objcg = NULL;
 	int cpu, i, unit_size, percpu_size = 0;
 
@@ -591,7 +591,7 @@ int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg
 
 int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)
 {
-	struct bpf_mem_caches *cc, __percpu *pcc;
+	struct bpf_mem_caches *cc; struct bpf_mem_caches __percpu *pcc;
 	int cpu, i, unit_size, percpu_size;
 	struct obj_cgroup *objcg;
 	struct bpf_mem_cache *c;
Signature · 1 line
2.46.0
maintainer Acked-by on v2 1/1
#1 lore raw
Quoted 46 lines · Uros Bizjak

In arraymap.c:

In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1].

Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1].

In hashtab.c:

Assign the return value from bpf_mem_cache_alloc() to void pointer and cast the value to void __percpu ** (void pointer to percpu void pointer) before dereferencing.

In memalloc.c:

Explicitly declare __percpu variables.

Cast obj to void __percpu **.

In helpers.c:

Cast ptr in BPF_CALL_1 and BPF_CALL_2 from generic address space to __percpu address space via const uintptr_t [1].

Found by GCC's named address space checks.

There were no changes in the resulting object files.

[1] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name

Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>
Cc Andrii Nakryiko <[email protected]>
Cc Martin KaFai Lau <[email protected]>
Cc Eduard Zingerman <[email protected]>
Cc Song Liu <[email protected]>
Cc Yonghong Song <[email protected]>
Cc John Fastabend <[email protected]>
Cc KP Singh <[email protected]>
Cc Stanislav Fomichev <[email protected]>
Cc Hao Luo <[email protected]>
Cc Jiri Olsa <[email protected]>
Quoted 6 lines

---
v2: - cast return values from the __percpu address space to
the generic address space in bpf_array_map_seq_{start,next}().
- correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu *
---

Looks ok, thank you for applying suggested changes.
The only nit I have is that '(void *)(uintptr_t)p' (and it's inverse) looks quite bulky, hiding it behind some macro might make some sense.

[...]

#2 lore raw
Quoted 55 lines · Eduard Zingerman
Quoted 46 lines · Uros Bizjak

In arraymap.c:

In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1].

Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1].

In hashtab.c:

Assign the return value from bpf_mem_cache_alloc() to void pointer and cast the value to void __percpu ** (void pointer to percpu void pointer) before dereferencing.

In memalloc.c:

Explicitly declare __percpu variables.

Cast obj to void __percpu **.

In helpers.c:

Cast ptr in BPF_CALL_1 and BPF_CALL_2 from generic address space to __percpu address space via const uintptr_t [1].

Found by GCC's named address space checks.

There were no changes in the resulting object files.

[1] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name

Cc Alexei Starovoitov <[email protected]>
Cc Daniel Borkmann <[email protected]>
Cc Andrii Nakryiko <[email protected]>
Cc Martin KaFai Lau <[email protected]>
Cc Eduard Zingerman <[email protected]>
Cc Song Liu <[email protected]>
Cc Yonghong Song <[email protected]>
Cc John Fastabend <[email protected]>
Cc KP Singh <[email protected]>
Cc Stanislav Fomichev <[email protected]>
Cc Hao Luo <[email protected]>
Cc Jiri Olsa <[email protected]>
Quoted 6 lines

---
v2: - cast return values from the __percpu address space to
the generic address space in bpf_array_map_seq_{start,next}().
- correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu *
---

Quoted 3 lines

Looks ok, thank you for applying suggested changes.
The only nit I have is that '(void *)(uintptr_t)p' (and it's inverse) looks quite bulky, hiding it behind some macro might make some sense.

True, but as argued in [1], the macro wouldn't be flexible enough to cover all possible variants of the casts. Fortunately, this casting pattern appears only a few times, and perhaps its bulky appearance would motivate developers to rethink the cast.

[1] https://lore.kernel.org/lkml/CAFULd4YOf0Mz-JbR6LEWxM2M=4GTxqC9m-q_QAZJw8Ws16yrTA@mail.gmail.com/

Quoted 1 line

Thanks!

BR,
Uros.

#3 lore raw
In reply to Uros Bizjak
This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Hello:

This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Quoted 11 lines · On Sun, 11 Aug 2024 18:13:33 +0200 you wrote:

In arraymap.c:

In bpf_array_map_seq_start() and bpf_array_map_seq_next() cast return values from the __percpu address space to the generic address space via uintptr_t [1].

Correct the declaration of pptr pointer in __bpf_array_map_seq_show() to void __percpu * and cast the value from the generic address space to the __percpu address space via uintptr_t [1].

[...]

Here is the summary with links:
- [v2] bpf: Fix percpu address space issues

    https://git.kernel.org/bpf/bpf-next/c/6d641ca50d7e

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html