Skip to contentSkip to conversation

Add support for memory-mapping BPF array maps

PATCH Superseded Andrii Nakryiko 4 participants 15 messages Nov 9, 2019 – Nov 13, 2019

Subsystems (7)

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v2 cover Add support for memory-mapping BPF array maps

This patch set adds ability to memory-map BPF array maps (single- and multi-element). The primary use case is memory-mapping BPF array maps, created to back global data variables, created by libbpf implicitly. This allows for much better usability, along with avoiding syscalls to read or update data completely.

Due to memory-mapping requirements, BPF array map that is supposed to be memory-mapped, has to be created with special BPF_F_MMAPABLE attribute, which triggers slightly different memory allocation strategy internally. See patch 1 for details.

Libbpf is extended to detect kernel support for this flag, and if supported, will specify it for all global data maps automatically.

v1->v2:
- fix map lookup code generation for BPF_F_MMAPABLE case;
- prevent BPF_F_MMAPABLE flag for all but plain array map type;
- centralize ref-counting in generic bpf_map_mmap();
- don't use uref counting (Alexei);
- use vfree() directly;
- print flags with %x (Song);
- extend tests to verify bpf_map_{lookup,update}_elem() logic as well.

Andrii Nakryiko (3):
bpf: add mmap() support for BPF_MAP_TYPE_ARRAY
libbpf: make global data internal arrays mmap()-able, if possible selftests/bpf: add BPF_TYPE_MAP_ARRAY mmap() tests

 include/linux/bpf.h                           |   9 +-
 include/uapi/linux/bpf.h                      |   3 +
 kernel/bpf/arraymap.c                         | 111 ++++++++++--
 kernel/bpf/syscall.c                          |  47 +++++
 tools/include/uapi/linux/bpf.h                |   3 +
 tools/lib/bpf/libbpf.c                        |  31 +++-
 .../selftests/bpf/prog_tests/core_reloc.c     |  45 +++--
 tools/testing/selftests/bpf/prog_tests/mmap.c | 170 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_mmap.c |  41 +++++
 9 files changed, 426 insertions(+), 34 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/mmap.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_mmap.c
Signature · 1 line
2.17.1
#1 lore raw

PATCH v2 1/3 bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

    map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

With BPF_F_MMAPABLE array allocating data in separate chunk of memory, array_map_gen_lookup has to accomodate these changes. For non-memory-mapped there are no changes and no extra instructions. For BPF_F_MMAPABLE case, pointer to where array data is stored has to be dereferenced first.

Generated code for non-memory-mapped array:

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:19]
24: (07) r1 += 408 /* array->inline_data offset */
25: (61) r0 = *(u32 *)(r2 +0)
26: (35) if r0 >= 0x3 goto pc+3
27: (67) r0 <<= 3
28: (0f) r0 += r1
29: (05) goto pc+1
30: (b7) r0 = 0

Generated code for memory-mapped array:

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:27]
24: (07) r1 += 400 /* array->data offset */
25: (79) r1 = *(u64 *)(r1 +0) /* extra dereference */
26: (61) r0 = *(u32 *)(r2 +0)
27: (35) if r0 >= 0x3 goto pc+3
28: (67) r0 <<= 3
29: (0f) r0 += r1
30: (05) goto pc+1
31: (b7) r0 = 0

Cc Rik van Riel <[email protected]>
Cc Johannes Weiner <[email protected]>
---
 include/linux/bpf.h            |   9 ++-
 include/uapi/linux/bpf.h       |   3 +
 kernel/bpf/arraymap.c          | 111 +++++++++++++++++++++++++++++----
 kernel/bpf/syscall.c           |  47 ++++++++++++++
 tools/include/uapi/linux/bpf.h |   3 +
 5 files changed, 159 insertions(+), 14 deletions(-)
include/linux/bpf.h +7 -2
index 7c7f518811a6..296332227959 100644
@@ -12,6 +12,7 @@
 #include <linux/err.h>
 #include <linux/rbtree_latch.h>
 #include <linux/numa.h>
+#include <linux/mm_types.h>
 #include <linux/wait.h>
 #include <linux/u64_stats_sync.h>
 
@@ -66,6 +67,7 @@ struct bpf_map_ops {
 				     u64 *imm, u32 off);
 	int (*map_direct_value_meta)(const struct bpf_map *map,
 				     u64 imm, u32 *off);
+	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
 };
 
 struct bpf_map_memory {
@@ -95,7 +97,7 @@ struct bpf_map {
 	struct btf *btf;
 	struct bpf_map_memory memory;
 	bool unpriv_array;
-	bool frozen; /* write-once */
+	bool frozen; /* write-once; write-protected by freeze_mutex */
 	/* 48 bytes hole */
 
 	/* The 3rd and 4th cacheline with misc members to avoid false sharing
@@ -105,6 +107,8 @@ struct bpf_map {
 	atomic_t usercnt;
 	struct work_struct work;
 	char name[BPF_OBJ_NAME_LEN];
+	struct mutex freeze_mutex;
+	int writecnt; /* writable mmap cnt; protected by freeze_mutex */
 };
 
 static inline bool map_value_has_spin_lock(const struct bpf_map *map)
@@ -461,8 +465,9 @@ struct bpf_array {
 	 */
 	enum bpf_prog_type owner_prog_type;
 	bool owner_jited;
+	void *data;
 	union {
-		char value[0] __aligned(8);
+		char inline_data[0] __aligned(8);
 		void *ptrs[0] __aligned(8);
 		void __percpu *pptrs[0] __aligned(8);
 	};
include/uapi/linux/bpf.h +3 -0
index df6809a76404..bb39b53622d9 100644
@@ -346,6 +346,9 @@ enum bpf_attach_type {
 /* Clone map from listener for newly accepted socket */
 #define BPF_F_CLONE		(1U << 9)
 
+/* Enable memory-mapping BPF map */
+#define BPF_F_MMAPABLE		(1U << 10)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
kernel/bpf/arraymap.c +99 -12

198 lines not on this page · raw message

kernel/bpf/syscall.c +47 -0

77 lines not on this page · raw message

tools/include/uapi/linux/bpf.h +3 -0
index df6809a76404..bb39b53622d9 100644
@@ -346,6 +346,9 @@ enum bpf_attach_type {
 /* Clone map from listener for newly accepted socket */
 #define BPF_F_CLONE		(1U << 9)
 
+/* Enable memory-mapping BPF map */
+#define BPF_F_MMAPABLE		(1U << 10)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
Signature · 1 line
2.17.1
Acked-by on v2 1/3
#2 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 8 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

[...]

Quoted 14 lines

Generated code for memory-mapped array:

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:27]
24: (07) r1 += 400 /* array->data offset */
25: (79) r1 = *(u64 *)(r1 +0) /* extra dereference */
26: (61) r0 = *(u32 *)(r2 +0)
27: (35) if r0 >= 0x3 goto pc+3
28: (67) r0 <<= 3
29: (0f) r0 += r1
30: (05) goto pc+1
31: (b7) r0 = 0

Quoted 3 lines
Cc Rik van Riel <[email protected]>
Cc Johannes Weiner <[email protected]>
#3 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

In reply to Andrii Nakryiko

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 84

Quoted 9 lines · Andrii Nakryiko
kernel/bpf/arraymap.c +1 -1
@@ -74,7 +78,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
 	int ret, numa_node = bpf_map_attr_numa_node(attr);
 	u32 elem_size, index_mask, max_entries;
 	bool unpriv = !capable(CAP_SYS_ADMIN);
-	u64 cost, array_size, mask64;
+	u64 cost, array_size, data_size, mask64;
 	struct bpf_map_memory mem;
 	struct bpf_array *array;
 

Please don't break reverse xmas tree where it exists.

#4 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 13 lines · Jakub Kicinski

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 84

Quoted 9 lines · Andrii Nakryiko
kernel/bpf/arraymap.c +1 -1
@@ -74,7 +78,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
      int ret, numa_node = bpf_map_attr_numa_node(attr);
      u32 elem_size, index_mask, max_entries;
      bool unpriv = !capable(CAP_SYS_ADMIN);
-     u64 cost, array_size, mask64;
+     u64 cost, array_size, data_size, mask64;
      struct bpf_map_memory mem;
      struct bpf_array *array;
 

Please don't break reverse xmas tree where it exists.

alright, will move it up

#5 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

In reply to Andrii Nakryiko
Quoted 19 lines · Andrii Nakryiko

With BPF_F_MMAPABLE array allocating data in separate chunk of memory, array_map_gen_lookup has to accomodate these changes. For non-memory-mapped there are no changes and no extra instructions. For BPF_F_MMAPABLE case, pointer to where array data is stored has to be dereferenced first.

Generated code for non-memory-mapped array:

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:19]
24: (07) r1 += 408 /* array->inline_data offset */
25: (61) r0 = *(u32 *)(r2 +0)
26: (35) if r0 >= 0x3 goto pc+3
27: (67) r0 <<= 3
28: (0f) r0 += r1
29: (05) goto pc+1
30: (b7) r0 = 0

Generated code for memory-mapped array:

Quoted 10 lines

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:27]
24: (07) r1 += 400 /* array->data offset */
25: (79) r1 = *(u64 *)(r1 +0) /* extra dereference */
26: (61) r0 = *(u32 *)(r2 +0)
27: (35) if r0 >= 0x3 goto pc+3
28: (67) r0 <<= 3
29: (0f) r0 += r1
30: (05) goto pc+1
31: (b7) r0 = 0

Would it not be possible to overallocate the memory and align the start of the bpf_map in case of BPF_F_MMAPABLE so that no extra dereference is needed?

#6 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 32 lines · Jakub Kicinski
Quoted 19 lines · Andrii Nakryiko

With BPF_F_MMAPABLE array allocating data in separate chunk of memory, array_map_gen_lookup has to accomodate these changes. For non-memory-mapped there are no changes and no extra instructions. For BPF_F_MMAPABLE case, pointer to where array data is stored has to be dereferenced first.

Generated code for non-memory-mapped array:

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:19]
24: (07) r1 += 408 /* array->inline_data offset */
25: (61) r0 = *(u32 *)(r2 +0)
26: (35) if r0 >= 0x3 goto pc+3
27: (67) r0 <<= 3
28: (0f) r0 += r1
29: (05) goto pc+1
30: (b7) r0 = 0

Generated code for memory-mapped array:

Quoted 10 lines

; p = bpf_map_lookup_elem(&data_map, &zero);
22: (18) r1 = map[id:27]
24: (07) r1 += 400 /* array->data offset */
25: (79) r1 = *(u64 *)(r1 +0) /* extra dereference */
26: (61) r0 = *(u32 *)(r2 +0)
27: (35) if r0 >= 0x3 goto pc+3
28: (67) r0 <<= 3
29: (0f) r0 += r1
30: (05) goto pc+1
31: (b7) r0 = 0

Quoted 3 lines

Would it not be possible to overallocate the memory and align the start of the bpf_map in case of BPF_F_MMAPABLE so that no extra dereference is needed?

So let's say if sizeof(struct bpf_array) is 300, then I'd have to either:

- somehow make sure that I allocate 4k (for data) + 300 (for struct bpf_array) in such a way that those 4k of data are 4k-aligned. Is there any way to do that?
- assuming there isn't, then another way would be to allocate entire 4k page for struct bpf_array itself, but put it at the end of that page, so that 4k of data is 4k-aligned. While wasteful, the bigger problem is that pointer to bpf_array is not a pointer to allocated memory anymore, so we'd need to remember that and adjust address before calling vfree().

Were you suggesting #2 as a solution? Or am I missing some other way to do this?

#7 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 12 lines · Andrii Nakryiko

So let's say if sizeof(struct bpf_array) is 300, then I'd have to either:

- somehow make sure that I allocate 4k (for data) + 300 (for struct bpf_array) in such a way that those 4k of data are 4k-aligned. Is there any way to do that?
- assuming there isn't, then another way would be to allocate entire 4k page for struct bpf_array itself, but put it at the end of that page, so that 4k of data is 4k-aligned. While wasteful, the bigger problem is that pointer to bpf_array is not a pointer to allocated memory anymore, so we'd need to remember that and adjust address before calling vfree().

Quoted 1 line

Were you suggesting #2 as a solution? Or am I missing some other way to do this?

I am suggesting #2, that's the way to do it in the kernel.

You could make the assumption that if you're allocating memory aligned to PAGE_SIZE, the address for vfree() is:

	addr = map;
	if (map->flags & MMAPABLE)
		addr = round_down(addr, PAGE_SIZE);
	vfree(addr);

Just make a note of the fact that we depend on vmalloc()s alignment in bpf_map_area_alloc().

#8 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 14 lines · Jakub Kicinski
Quoted 12 lines · Andrii Nakryiko

So let's say if sizeof(struct bpf_array) is 300, then I'd have to either:

- somehow make sure that I allocate 4k (for data) + 300 (for struct bpf_array) in such a way that those 4k of data are 4k-aligned. Is there any way to do that?
- assuming there isn't, then another way would be to allocate entire 4k page for struct bpf_array itself, but put it at the end of that page, so that 4k of data is 4k-aligned. While wasteful, the bigger problem is that pointer to bpf_array is not a pointer to allocated memory anymore, so we'd need to remember that and adjust address before calling vfree().

Quoted 3 lines
Quoted 1 line

Were you suggesting #2 as a solution? Or am I missing some other way to do this?

I am suggesting #2, that's the way to do it in the kernel.

So I'm concerned about this approach, because it feels like a bunch of unnecessarily wasted memory. While there is no way around doing round_up(PAGE_SIZE) for data itself, it certainly is not necessary to waste almost entire page for struct bpf_array. And given this is going to be used for BPF maps backing global variables, there most probably will be at least 3 (.data, .bss, .rodata) per each program, and could be more. Also, while on x86_64 page is 4k, on other architectures it can be up to 64KB, so this seems wasteful.

What's your concern exactly with the way it's implemented in this patch?

Quoted 11 lines

You could make the assumption that if you're allocating memory aligned to PAGE_SIZE, the address for vfree() is:

        addr = map;
        if (map->flags & MMAPABLE)
                addr = round_down(addr, PAGE_SIZE);
        vfree(addr);

Just make a note of the fact that we depend on vmalloc()s alignment in bpf_map_area_alloc().

will add comment for that

#9 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 18 lines · Andrii Nakryiko
Quoted 13 lines · Jakub Kicinski
Quoted 12 lines · Andrii Nakryiko

So let's say if sizeof(struct bpf_array) is 300, then I'd have to either:

- somehow make sure that I allocate 4k (for data) + 300 (for struct bpf_array) in such a way that those 4k of data are 4k-aligned. Is there any way to do that?
- assuming there isn't, then another way would be to allocate entire 4k page for struct bpf_array itself, but put it at the end of that page, so that 4k of data is 4k-aligned. While wasteful, the bigger problem is that pointer to bpf_array is not a pointer to allocated memory anymore, so we'd need to remember that and adjust address before calling vfree().

Quoted 3 lines
Quoted 1 line

Were you suggesting #2 as a solution? Or am I missing some other way to do this?

I am suggesting #2, that's the way to do it in the kernel.

Quoted 8 lines

So I'm concerned about this approach, because it feels like a bunch of unnecessarily wasted memory. While there is no way around doing round_up(PAGE_SIZE) for data itself, it certainly is not necessary to waste almost entire page for struct bpf_array. And given this is going to be used for BPF maps backing global variables, there most probably will be at least 3 (.data, .bss, .rodata) per each program, and could be more. Also, while on x86_64 page is 4k, on other architectures it can be up to 64KB, so this seems wasteful.

With the extra mutex and int you grew struct bpf_map from 192B to 256B, that's for every map on the system, unconditionally, and array map has an extra pointer even if it doesn't need it.

Increasing "wasted" space when an opt-in feature is selected doesn't seem all that terrible to me, especially that the overhead of aligning up map size to page size is already necessary.

Quoted 1 line

What's your concern exactly with the way it's implemented in this patch?

Judging by other threads we seem to care about performance of BPF (rightly so). Doing an extra pointer deref for every static data access seems like an obvious waste.

But then again, it's just an obvious suggestion, take it or leave it..

#10 lore raw

Re: [PATCH v2 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 29 lines · Jakub Kicinski
Quoted 18 lines · Andrii Nakryiko
Quoted 13 lines · Jakub Kicinski
Quoted 12 lines · Andrii Nakryiko

So let's say if sizeof(struct bpf_array) is 300, then I'd have to either:

- somehow make sure that I allocate 4k (for data) + 300 (for struct bpf_array) in such a way that those 4k of data are 4k-aligned. Is there any way to do that?
- assuming there isn't, then another way would be to allocate entire 4k page for struct bpf_array itself, but put it at the end of that page, so that 4k of data is 4k-aligned. While wasteful, the bigger problem is that pointer to bpf_array is not a pointer to allocated memory anymore, so we'd need to remember that and adjust address before calling vfree().

Quoted 3 lines
Quoted 1 line

Were you suggesting #2 as a solution? Or am I missing some other way to do this?

I am suggesting #2, that's the way to do it in the kernel.

Quoted 8 lines

So I'm concerned about this approach, because it feels like a bunch of unnecessarily wasted memory. While there is no way around doing round_up(PAGE_SIZE) for data itself, it certainly is not necessary to waste almost entire page for struct bpf_array. And given this is going to be used for BPF maps backing global variables, there most probably will be at least 3 (.data, .bss, .rodata) per each program, and could be more. Also, while on x86_64 page is 4k, on other architectures it can be up to 64KB, so this seems wasteful.

Quoted 7 lines

With the extra mutex and int you grew struct bpf_map from 192B to 256B, that's for every map on the system, unconditionally, and array map has an extra pointer even if it doesn't need it.

Increasing "wasted" space when an opt-in feature is selected doesn't seem all that terrible to me, especially that the overhead of aligning up map size to page size is already necessary.

Well, I've been talking about one more extra page for struct bpf_array itself, on top of what we already potentially waste for mmap()'ing array data. But I went ahead and posted v3 with layout we discussed here, aligning array->value on page boundary. Let's see if you like it better.

Quoted 8 lines
Quoted 2 lines

What's your concern exactly with the way it's implemented in this patch?

Judging by other threads we seem to care about performance of BPF (rightly so). Doing an extra pointer deref for every static data access seems like an obvious waste.

But then again, it's just an obvious suggestion, take it or leave it..

#11 lore raw

PATCH v2 2/3 libbpf: make global data internal arrays mmap()-able, if possible

In reply to Andrii Nakryiko

Add detection of BPF_F_MMAPABLE flag support for arrays and add it as an extra flag to internal global data maps, if supported by kernel. This allows users to memory-map global data and use it without BPF map operations, greatly simplifying user experience.

---
 tools/lib/bpf/libbpf.c | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)
tools/lib/bpf/libbpf.c +29 -2
index fde6cb3e5d41..d32899e50f8a 100644
@@ -142,6 +142,8 @@ struct bpf_capabilities {
 	__u32 btf_func:1;
 	/* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 	__u32 btf_datasec:1;
+	/* BPF_F_MMAPABLE is supported for arrays */
+	__u32 array_mmap:1;
 };
 
 /*
@@ -855,8 +857,6 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 		pr_warn("failed to alloc map name\n");
 		return -ENOMEM;
 	}
-	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
-		 map_name, map->sec_idx, map->sec_offset);
 
 	def = &map->def;
 	def->type = BPF_MAP_TYPE_ARRAY;
@@ -864,6 +864,12 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 	def->value_size = data->d_size;
 	def->max_entries = 1;
 	def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0;
+	if (obj->caps.array_mmap)
+		def->map_flags |= BPF_F_MMAPABLE;
+
+	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
+		 map_name, map->sec_idx, map->sec_offset, def->map_flags);
+
 	if (data_buff) {
 		*data_buff = malloc(data->d_size);
 		if (!*data_buff) {
@@ -2158,6 +2164,26 @@ static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
 	return 0;
 }
 
+static int bpf_object__probe_array_mmap(struct bpf_object *obj)
+{
+	struct bpf_create_map_attr attr = {
+		.map_type = BPF_MAP_TYPE_ARRAY,
+		.map_flags = BPF_F_MMAPABLE,
+		.key_size = sizeof(int),
+		.value_size = sizeof(int),
+		.max_entries = 1,
+	};
+	int fd = bpf_create_map_xattr(&attr);
+
+	if (fd >= 0) {
+		obj->caps.array_mmap = 1;
+		close(fd);
+		return 1;
+	}
+
+	return 0;
+}
+
 static int
 bpf_object__probe_caps(struct bpf_object *obj)
 {
@@ -2166,6 +2192,7 @@ bpf_object__probe_caps(struct bpf_object *obj)
 		bpf_object__probe_global_data,
 		bpf_object__probe_btf_func,
 		bpf_object__probe_btf_datasec,
+		bpf_object__probe_array_mmap,
 	};
 	int i, ret;
 
Signature · 1 line
2.17.1
#12 lore raw

Re: [PATCH v2 bpf-next 2/3] libbpf: make global data internal arrays mmap()-able, if possible

tools/lib/bpf/libbpf.c static int bpf_object__probe_btf_datasec(struct bpf_object *obj) · line 2178

Quoted 12 lines · Andrii Nakryiko
tools/lib/bpf/libbpf.c +12 -0
+static int bpf_object__probe_array_mmap(struct bpf_object *obj)
+{
+	struct bpf_create_map_attr attr = {
+		.map_type = BPF_MAP_TYPE_ARRAY,
+		.map_flags = BPF_F_MMAPABLE,
+		.key_size = sizeof(int),
+		.value_size = sizeof(int),
+		.max_entries = 1,
+	};
+	int fd = bpf_create_map_xattr(&attr);
+
+	if (fd >= 0) {

The point of the empty line between variable declarations and code in the Linux coding style is to provide a visual separation between variables and code.

If you call a complex function in variable init and then check for errors in the code that really breaks that principle.

Quoted 7 lines
tools/lib/bpf/libbpf.c +7 -0
+		obj->caps.array_mmap = 1;
+		close(fd);
+		return 1;
+	}
+
+	return 0;
+}
#13 lore raw

Re: [PATCH v2 bpf-next 2/3] libbpf: make global data internal arrays mmap()-able, if possible

Quoted 19 lines · Jakub Kicinski

tools/lib/bpf/libbpf.c static int bpf_object__probe_btf_datasec(struct bpf_object *obj) · line 2178

Quoted 12 lines · Andrii Nakryiko
tools/lib/bpf/libbpf.c +12 -0
+static int bpf_object__probe_array_mmap(struct bpf_object *obj)
+{
+     struct bpf_create_map_attr attr = {
+             .map_type = BPF_MAP_TYPE_ARRAY,
+             .map_flags = BPF_F_MMAPABLE,
+             .key_size = sizeof(int),
+             .value_size = sizeof(int),
+             .max_entries = 1,
+     };
+     int fd = bpf_create_map_xattr(&attr);
+
+     if (fd >= 0) {

The point of the empty line between variable declarations and code in the Linux coding style is to provide a visual separation between variables and code.

Quoted 2 lines

If you call a complex function in variable init and then check for errors in the code that really breaks that principle.

I'll split declaration and initialization, no problem

Quoted 9 lines
Quoted 8 lines
tools/lib/bpf/libbpf.c +7 -0
+             obj->caps.array_mmap = 1;
+             close(fd);
+             return 1;
+     }
+
+     return 0;
+}
#14 lore raw

PATCH v2 3/3 selftests/bpf: add BPF_TYPE_MAP_ARRAY mmap() tests

In reply to Andrii Nakryiko

Add selftests validating mmap()-ing BPF array maps: both single-element and multi-element ones. Check that plain bpf_map_update_elem() and bpf_map_lookup_elem() work correctly with memory-mapped array. Also convert CO-RE relocation tests to use memory-mapped views of global data.

---
 .../selftests/bpf/prog_tests/core_reloc.c     |  45 +++--
 tools/testing/selftests/bpf/prog_tests/mmap.c | 170 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_mmap.c |  41 +++++
 3 files changed, 238 insertions(+), 18 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/mmap.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_mmap.c
tools/testing/selftests/bpf/prog_tests/core_reloc.c +27 -18
index f94bd071536b..ec9e2fdd6b89 100644
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <test_progs.h>
 #include "progs/core_reloc_types.h"
+#include <sys/mman.h>
 
 #define STRUCT_TO_CHAR_PTR(struct_name) (const char *)&(struct struct_name)
 
@@ -453,8 +454,15 @@ struct data {
 	char out[256];
 };
 
+static size_t roundup_page(size_t sz)
+{
+	long page_size = sysconf(_SC_PAGE_SIZE);
+	return (sz + page_size - 1) / page_size * page_size;
+}
+
 void test_core_reloc(void)
 {
+	const size_t mmap_sz = roundup_page(sizeof(struct data));
 	struct bpf_object_load_attr load_attr = {};
 	struct core_reloc_test_case *test_case;
 	const char *tp_name, *probe_name;
@@ -463,8 +471,8 @@ void test_core_reloc(void)
 	struct bpf_map *data_map;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
-	const int zero = 0;
-	struct data data;
+	struct data *data;
+	void *mmap_data = NULL;
 
 	for (i = 0; i < ARRAY_SIZE(test_cases); i++) {
 		test_case = &test_cases[i];
@@ -476,8 +484,7 @@ void test_core_reloc(void)
 		);
 
 		obj = bpf_object__open_file(test_case->bpf_obj_file, &opts);
-		if (CHECK(IS_ERR_OR_NULL(obj), "obj_open",
-			  "failed to open '%s': %ld\n",
+		if (CHECK(IS_ERR(obj), "obj_open", "failed to open '%s': %ld\n",
 			  test_case->bpf_obj_file, PTR_ERR(obj)))
 			continue;
 
@@ -519,24 +526,22 @@ void test_core_reloc(void)
 		if (CHECK(!data_map, "find_data_map", "data map not found\n"))
 			goto cleanup;
 
-		memset(&data, 0, sizeof(data));
-		memcpy(data.in, test_case->input, test_case->input_len);
-
-		err = bpf_map_update_elem(bpf_map__fd(data_map),
-					  &zero, &data, 0);
-		if (CHECK(err, "update_data_map",
-			  "failed to update .data map: %d\n", err))
+		mmap_data = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
+				 MAP_SHARED, bpf_map__fd(data_map), 0);
+		if (CHECK(mmap_data == MAP_FAILED, "mmap",
+			  ".bss mmap failed: %d", errno)) {
+			mmap_data = NULL;
 			goto cleanup;
+		}
+		data = mmap_data;
+
+		memset(mmap_data, 0, sizeof(*data));
+		memcpy(data->in, test_case->input, test_case->input_len);
 
 		/* trigger test run */
 		usleep(1);
 
-		err = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, &data);
-		if (CHECK(err, "get_result",
-			  "failed to get output data: %d\n", err))
-			goto cleanup;
-
-		equal = memcmp(data.out, test_case->output,
+		equal = memcmp(data->out, test_case->output,
 			       test_case->output_len) == 0;
 		if (CHECK(!equal, "check_result",
 			  "input/output data don't match\n")) {
@@ -548,12 +553,16 @@ void test_core_reloc(void)
 			}
 			for (j = 0; j < test_case->output_len; j++) {
 				printf("output byte #%d: EXP 0x%02hhx GOT 0x%02hhx\n",
-				       j, test_case->output[j], data.out[j]);
+				       j, test_case->output[j], data->out[j]);
 			}
 			goto cleanup;
 		}
 
 cleanup:
+		if (mmap_data) {
+			CHECK_FAIL(munmap(mmap_data, mmap_sz));
+			mmap_data = NULL;
+		}
 		if (!IS_ERR_OR_NULL(link)) {
 			bpf_link__destroy(link);
 			link = NULL;
tools/testing/selftests/bpf/prog_tests/mmap.c new file +170 -0

170 lines not on this page · raw message

tools/testing/selftests/bpf/progs/test_mmap.c new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.17.1