Skip to contentSkip to conversation

Add support for memory-mapping BPF array maps

PATCH Superseded Andrii Nakryiko 4 participants 11 messages Nov 13, 2019

Subsystems (7)

v3 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v3 cover Add support for memory-mapping BPF array maps

This patch set adds ability to memory-map BPF array maps (single- and multi-element). The primary use case is memory-mapping BPF array maps, created to back global data variables, created by libbpf implicitly. This allows for much better usability, along with avoiding syscalls to read or update data completely.

Due to memory-mapping requirements, BPF array map that is supposed to be memory-mapped, has to be created with special BPF_F_MMAPABLE attribute, which triggers slightly different memory allocation strategy internally. See patch 1 for details.

Libbpf is extended to detect kernel support for this flag, and if supported, will specify it for all global data maps automatically.

v2->v3:
- change allocation strategy to avoid extra pointer dereference (Jakub);

v1->v2:
- fix map lookup code generation for BPF_F_MMAPABLE case;
- prevent BPF_F_MMAPABLE flag for all but plain array map type;
- centralize ref-counting in generic bpf_map_mmap();
- don't use uref counting (Alexei);
- use vfree() directly;
- print flags with %x (Song);
- extend tests to verify bpf_map_{lookup,update}_elem() logic as well.

Andrii Nakryiko (3):
bpf: add mmap() support for BPF_MAP_TYPE_ARRAY
libbpf: make global data internal arrays mmap()-able, if possible selftests/bpf: add BPF_TYPE_MAP_ARRAY mmap() tests

 include/linux/bpf.h                           |   6 +-
 include/uapi/linux/bpf.h                      |   3 +
 kernel/bpf/arraymap.c                         |  93 +++++++++-
 kernel/bpf/syscall.c                          |  47 +++++
 tools/include/uapi/linux/bpf.h                |   3 +
 tools/lib/bpf/libbpf.c                        |  32 +++-
 .../selftests/bpf/prog_tests/core_reloc.c     |  45 +++--
 tools/testing/selftests/bpf/prog_tests/mmap.c | 170 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_mmap.c |  41 +++++
 9 files changed, 413 insertions(+), 27 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/mmap.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_mmap.c
Signature · 1 line
2.17.1
#1 lore raw

PATCH v3 1/3 bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

    map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

---
 include/linux/bpf.h            |  6 ++-
 include/uapi/linux/bpf.h       |  3 ++
 kernel/bpf/arraymap.c          | 93 +++++++++++++++++++++++++++++++---
 kernel/bpf/syscall.c           | 47 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  3 ++
 5 files changed, 145 insertions(+), 7 deletions(-)
include/linux/bpf.h +5 -1
index 7c7f518811a6..ab9a24b94357 100644
@@ -12,6 +12,7 @@
 #include <linux/err.h>
 #include <linux/rbtree_latch.h>
 #include <linux/numa.h>
+#include <linux/mm_types.h>
 #include <linux/wait.h>
 #include <linux/u64_stats_sync.h>
 
@@ -66,6 +67,7 @@ struct bpf_map_ops {
 				     u64 *imm, u32 off);
 	int (*map_direct_value_meta)(const struct bpf_map *map,
 				     u64 imm, u32 *off);
+	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
 };
 
 struct bpf_map_memory {
@@ -95,7 +97,7 @@ struct bpf_map {
 	struct btf *btf;
 	struct bpf_map_memory memory;
 	bool unpriv_array;
-	bool frozen; /* write-once */
+	bool frozen; /* write-once; write-protected by freeze_mutex */
 	/* 48 bytes hole */
 
 	/* The 3rd and 4th cacheline with misc members to avoid false sharing
@@ -105,6 +107,8 @@ struct bpf_map {
 	atomic_t usercnt;
 	struct work_struct work;
 	char name[BPF_OBJ_NAME_LEN];
+	struct mutex freeze_mutex;
+	int writecnt; /* writable mmap cnt; protected by freeze_mutex */
 };
 
 static inline bool map_value_has_spin_lock(const struct bpf_map *map)
include/uapi/linux/bpf.h +3 -0
index df6809a76404..bb39b53622d9 100644
@@ -346,6 +346,9 @@ enum bpf_attach_type {
 /* Clone map from listener for newly accepted socket */
 #define BPF_F_CLONE		(1U << 9)
 
+/* Enable memory-mapping BPF map */
+#define BPF_F_MMAPABLE		(1U << 10)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
kernel/bpf/arraymap.c +87 -6

136 lines not on this page · raw message

kernel/bpf/syscall.c +47 -0

77 lines not on this page · raw message

tools/include/uapi/linux/bpf.h +3 -0
index df6809a76404..bb39b53622d9 100644
@@ -346,6 +346,9 @@ enum bpf_attach_type {
 /* Clone map from listener for newly accepted socket */
 #define BPF_F_CLONE		(1U << 9)
 
+/* Enable memory-mapping BPF map */
+#define BPF_F_MMAPABLE		(1U << 10)
+
 /* flags for BPF_PROG_QUERY */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
Signature · 1 line
2.17.1
reviewer Acked-by on v3 1/3
#2 lore raw

RE: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 28 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

    map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

[...]

Quoted 11 lines
kernel/bpf/arraymap.c +3 -3
@@ -102,10 +106,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
 	}
 
 	array_size = sizeof(*array);
-	if (percpu)
+	if (percpu) {
 		array_size += (u64) max_entries * sizeof(void *);
-	else
-		array_size += (u64) max_entries * elem_size;
+	} else {
+		/* rely on vmalloc() to return page-aligned memory and

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 122

Quoted 10 lines
kernel/bpf/arraymap.c +10 -0
+		 * ensure array->value is exactly page-aligned
+		 */
+		if (attr->map_flags & BPF_F_MMAPABLE) {
+			array_size = round_up(array_size, PAGE_SIZE);
+			array_size += (u64) max_entries * elem_size;
+			array_size = round_up(array_size, PAGE_SIZE);
+		} else {
+			array_size += (u64) max_entries * elem_size;
+		}
+	}

Thought about this chunk for a bit, assuming we don't end up with lots of small mmap arrays it should be OK. So userspace will probably need to try and optimize this to create as few mmaps as possible.

[...]

#3 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 50 lines · John Fastabend
Quoted 28 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

    map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

[...]

Quoted 17 lines
kernel/bpf/arraymap.c +9 -3
@@ -102,10 +106,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
      }
 
      array_size = sizeof(*array);
-     if (percpu)
+     if (percpu) {
              array_size += (u64) max_entries * sizeof(void *);
-     else
-             array_size += (u64) max_entries * elem_size;
+     } else {
+             /* rely on vmalloc() to return page-aligned memory and
+              * ensure array->value is exactly page-aligned
+              */
+             if (attr->map_flags & BPF_F_MMAPABLE) {
+                     array_size = round_up(array_size, PAGE_SIZE);
+                     array_size += (u64) max_entries * elem_size;
+                     array_size = round_up(array_size, PAGE_SIZE);
Quoted 8 lines

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 122

Quoted 4 lines
kernel/bpf/arraymap.c +4 -0
+             } else {
+                     array_size += (u64) max_entries * elem_size;
+             }
+     }

Thought about this chunk for a bit, assuming we don't end up with lots of small mmap arrays it should be OK. So userspace will probably need to try and optimize this to create as few mmaps as possible.

I think typically most explicitly declared maps won't be BPF_F_MMAPABLE, unless user really expects to mmap() it for use from user-space. For global data, though, the benefits are really great from being able to mmap(), which is why I'm defaulting them to BPF_F_MMAPABLE by default, if possible.

Quoted 4 lines

[...]

maintainer
#4 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

In reply to Andrii Nakryiko
Quoted 24 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

     map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

Quoted 4 lines

Overall set looks good to me! One comment below:

[...]

Quoted 9 lines
kernel/bpf/arraymap.c +4 -1
@@ -117,7 +131,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
  		return ERR_PTR(ret);
 
  	/* allocate all map elements and zero-initialize them */
-	array = bpf_map_area_alloc(array_size, numa_node);
+	if (attr->map_flags & BPF_F_MMAPABLE) {
+		void *data;
+
+		/* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 147

Quoted 10 lines
kernel/bpf/arraymap.c +10 -0
+		data = vzalloc_node(array_size, numa_node);
+		if (!data) {
+			bpf_map_charge_finish(&mem);
+			return ERR_PTR(-ENOMEM);
+		}
+		array = data + round_up(sizeof(struct bpf_array), PAGE_SIZE)
+			- offsetof(struct bpf_array, value);
+	} else {
+		array = bpf_map_area_alloc(array_size, numa_node);
+	}

Can't we place/extend all this logic inside bpf_map_area_alloc() and bpf_map_area_free() API instead of hard-coding it here?

Given this is a generic feature of which global data is just one consumer, my concern is that this reintroduces similar issues that mentioned API was trying to solve already meaning failing early instead of trying hard and triggering OOM if the array is large.

Consolidating this into bpf_map_area_alloc()/bpf_map_area_free() would make sure all the rest has same semantics.

Quoted 15 lines
kernel/bpf/arraymap.c +4 -1
  	if (!array) {
  		bpf_map_charge_finish(&mem);
  		return ERR_PTR(-ENOMEM);
@@ -365,7 +392,10 @@ static void array_map_free(struct bpf_map *map)
  	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  		bpf_array_free_percpu(array);
 
-	bpf_map_area_free(array);
+	if (array->map.map_flags & BPF_F_MMAPABLE)
+		bpf_map_area_free((void *)round_down((long)array, PAGE_SIZE));
+	else
+		bpf_map_area_free(array);
  }
 
  static void array_map_seq_show_elem(struct bpf_map *map, void *key,

[...]

#5 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 57 lines · Daniel Borkmann
Quoted 24 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

     map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

Quoted 4 lines

Overall set looks good to me! One comment below:

[...]

Quoted 9 lines
kernel/bpf/arraymap.c +4 -1
@@ -117,7 +131,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
              return ERR_PTR(ret);
 
      /* allocate all map elements and zero-initialize them */
-     array = bpf_map_area_alloc(array_size, numa_node);
+     if (attr->map_flags & BPF_F_MMAPABLE) {
+             void *data;
+
+             /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 147

Quoted 10 lines
kernel/bpf/arraymap.c +10 -0
+             data = vzalloc_node(array_size, numa_node);
+             if (!data) {
+                     bpf_map_charge_finish(&mem);
+                     return ERR_PTR(-ENOMEM);
+             }
+             array = data + round_up(sizeof(struct bpf_array), PAGE_SIZE)
+                     - offsetof(struct bpf_array, value);
+     } else {
+             array = bpf_map_area_alloc(array_size, numa_node);
+     }

Can't we place/extend all this logic inside bpf_map_area_alloc() and bpf_map_area_free() API instead of hard-coding it here?

Quoted 7 lines

Given this is a generic feature of which global data is just one consumer, my concern is that this reintroduces similar issues that mentioned API was trying to solve already meaning failing early instead of trying hard and triggering OOM if the array is large.

Consolidating this into bpf_map_area_alloc()/bpf_map_area_free() would make sure all the rest has same semantics.

So a bunch of this (e.g, array pointer adjustment in mmapable case) depends on specific layout of bpf_array, while bpf_map_area_alloc() is called for multitude of different maps. What we can generalize, though, is this enforcement of vmalloc() for mmapable case: enforce size is multiple of PAGE_SIZE, bypass kmalloc, etc. I can do that part easily, I refrained because it would require extra bool mmapable flag to bpf_map_area_alloc() and (trivial) update to 13 call sites passing false, I wasn't sure people would like code churn.

As for bpf_map_areas_free(), again, adjustment is specific to bpf_array and its memory layout w.r.t. data placement, so I don't think we can generalize it that much.

After talking with Johannes, I'm also adding new vmalloc_user_node_flags() API and will specify same RETRY_MAYFAIL and NOWARN flags, so behavior will stay the same.

Let me know if you want bool mmapable added to bpf_map_area_alloc(). And also if I'm missing how you wanted to generalize other parts, please explain in more details.

Quoted 17 lines
Quoted 16 lines
kernel/bpf/arraymap.c +4 -1
      if (!array) {
              bpf_map_charge_finish(&mem);
              return ERR_PTR(-ENOMEM);
@@ -365,7 +392,10 @@ static void array_map_free(struct bpf_map *map)
      if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
              bpf_array_free_percpu(array);
 
-     bpf_map_area_free(array);
+     if (array->map.map_flags & BPF_F_MMAPABLE)
+             bpf_map_area_free((void *)round_down((long)array, PAGE_SIZE));
+     else
+             bpf_map_area_free(array);
  }
 
  static void array_map_seq_show_elem(struct bpf_map *map, void *key,

[...]

maintainer
#6 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 78 lines · Andrii Nakryiko
Quoted 56 lines · Daniel Borkmann
Quoted 24 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

     map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

Quoted 4 lines

Overall set looks good to me! One comment below:

[...]

Quoted 9 lines
kernel/bpf/arraymap.c +4 -1
@@ -117,7 +131,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
              return ERR_PTR(ret);
 
      /* allocate all map elements and zero-initialize them */
-     array = bpf_map_area_alloc(array_size, numa_node);
+     if (attr->map_flags & BPF_F_MMAPABLE) {
+             void *data;
+
+             /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 147

Quoted 10 lines
kernel/bpf/arraymap.c +10 -0
+             data = vzalloc_node(array_size, numa_node);
+             if (!data) {
+                     bpf_map_charge_finish(&mem);
+                     return ERR_PTR(-ENOMEM);
+             }
+             array = data + round_up(sizeof(struct bpf_array), PAGE_SIZE)
+                     - offsetof(struct bpf_array, value);
+     } else {
+             array = bpf_map_area_alloc(array_size, numa_node);
+     }

Can't we place/extend all this logic inside bpf_map_area_alloc() and bpf_map_area_free() API instead of hard-coding it here?

Quoted 7 lines

Given this is a generic feature of which global data is just one consumer, my concern is that this reintroduces similar issues that mentioned API was trying to solve already meaning failing early instead of trying hard and triggering OOM if the array is large.

Consolidating this into bpf_map_area_alloc()/bpf_map_area_free() would make sure all the rest has same semantics.

So a bunch of this (e.g, array pointer adjustment in mmapable case) depends on specific layout of bpf_array, while bpf_map_area_alloc() is called for multitude of different maps. What we can generalize, though, is this enforcement of vmalloc() for mmapable case: enforce size is multiple of PAGE_SIZE, bypass kmalloc, etc. I can do that part easily, I refrained because it would require extra bool mmapable flag to bpf_map_area_alloc() and (trivial) update to 13 call sites passing false, I wasn't sure people would like code churn.

As for bpf_map_areas_free(), again, adjustment is specific to bpf_array and its memory layout w.r.t. data placement, so I don't think we can generalize it that much.

Quoted 7 lines

After talking with Johannes, I'm also adding new vmalloc_user_node_flags() API and will specify same RETRY_MAYFAIL and NOWARN flags, so behavior will stay the same.

Let me know if you want bool mmapable added to bpf_map_area_alloc(). And also if I'm missing how you wanted to generalize other parts, please explain in more details.

Why changing all call-sites? You could have two pair of API helpers, e.g. bpf_map_area_{,mmapable}_alloc() and bpf_map_area_{,mmapable}_free() and they both call into __bpf_map_area_alloc() and __bpf_map_area_free() which are private and common to both, so whenever we need to go and change internals, they are fixed for all users. Call-sites would remain as-is just for array map you'd select between the two.

Quoted 16 lines
Quoted 16 lines
Quoted 15 lines
kernel/bpf/arraymap.c +4 -1
      if (!array) {
              bpf_map_charge_finish(&mem);
              return ERR_PTR(-ENOMEM);
@@ -365,7 +392,10 @@ static void array_map_free(struct bpf_map *map)
      if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
              bpf_array_free_percpu(array);
 
-     bpf_map_area_free(array);
+     if (array->map.map_flags & BPF_F_MMAPABLE)
+             bpf_map_area_free((void *)round_down((long)array, PAGE_SIZE));
+     else
+             bpf_map_area_free(array);
  }
 
  static void array_map_seq_show_elem(struct bpf_map *map, void *key,

[...]

#7 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: add mmap() support for BPF_MAP_TYPE_ARRAY

Quoted 88 lines · Daniel Borkmann
Quoted 78 lines · Andrii Nakryiko
Quoted 56 lines · Daniel Borkmann
Quoted 24 lines · Andrii Nakryiko

Add ability to memory-map contents of BPF array map. This is extremely useful for working with BPF global data from userspace programs. It allows to avoid typical bpf_map_{lookup,update}_elem operations, improving both performance and usability.

There had to be special considerations for map freezing, to avoid having writable memory view into a frozen map. To solve this issue, map freezing and mmap-ing is happening under mutex now:
- if map is already frozen, no writable mapping is allowed;
- if map has writable memory mappings active (accounted in map->writecnt),

     map freezing will keep failing with -EBUSY;

- once number of writable memory mappings drops to zero, map freezing can be performed again.

Only non-per-CPU plain arrays are supported right now. Maps with spinlocks can't be memory mapped either.

For BPF_F_MMAPABLE array, memory allocation has to be done through vmalloc() to be mmap()'able. We also need to make sure that array data memory is page-sized and page-aligned, so we over-allocate memory in such a way that struct bpf_array is at the end of a single page of memory with array->value being aligned with the start of the second page. On deallocation we need to accomodate this memory arrangement to free vmalloc()'ed memory correctly.

Quoted 4 lines

Overall set looks good to me! One comment below:

[...]

Quoted 9 lines
kernel/bpf/arraymap.c +4 -1
@@ -117,7 +131,20 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
              return ERR_PTR(ret);
 
      /* allocate all map elements and zero-initialize them */
-     array = bpf_map_area_alloc(array_size, numa_node);
+     if (attr->map_flags & BPF_F_MMAPABLE) {
+             void *data;
+
+             /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */

kernel/bpf/arraymap.c static struct bpf_map *array_map_alloc(union bpf_attr *attr) · line 147

Quoted 10 lines
kernel/bpf/arraymap.c +10 -0
+             data = vzalloc_node(array_size, numa_node);
+             if (!data) {
+                     bpf_map_charge_finish(&mem);
+                     return ERR_PTR(-ENOMEM);
+             }
+             array = data + round_up(sizeof(struct bpf_array), PAGE_SIZE)
+                     - offsetof(struct bpf_array, value);
+     } else {
+             array = bpf_map_area_alloc(array_size, numa_node);
+     }

Can't we place/extend all this logic inside bpf_map_area_alloc() and bpf_map_area_free() API instead of hard-coding it here?

Quoted 7 lines

Given this is a generic feature of which global data is just one consumer, my concern is that this reintroduces similar issues that mentioned API was trying to solve already meaning failing early instead of trying hard and triggering OOM if the array is large.

Consolidating this into bpf_map_area_alloc()/bpf_map_area_free() would make sure all the rest has same semantics.

So a bunch of this (e.g, array pointer adjustment in mmapable case) depends on specific layout of bpf_array, while bpf_map_area_alloc() is called for multitude of different maps. What we can generalize, though, is this enforcement of vmalloc() for mmapable case: enforce size is multiple of PAGE_SIZE, bypass kmalloc, etc. I can do that part easily, I refrained because it would require extra bool mmapable flag to bpf_map_area_alloc() and (trivial) update to 13 call sites passing false, I wasn't sure people would like code churn.

As for bpf_map_areas_free(), again, adjustment is specific to bpf_array and its memory layout w.r.t. data placement, so I don't think we can generalize it that much.

Quoted 7 lines

After talking with Johannes, I'm also adding new vmalloc_user_node_flags() API and will specify same RETRY_MAYFAIL and NOWARN flags, so behavior will stay the same.

Let me know if you want bool mmapable added to bpf_map_area_alloc(). And also if I'm missing how you wanted to generalize other parts, please explain in more details.

Quoted 6 lines

Why changing all call-sites? You could have two pair of API helpers, e.g. bpf_map_area_{,mmapable}_alloc() and bpf_map_area_{,mmapable}_free() and they both call into __bpf_map_area_alloc() and __bpf_map_area_free() which are private and common to both, so whenever we need to go and change internals, they are fixed for all users. Call-sites would remain as-is just for array map you'd select between the two.

ok, can do bpf_map_area_alloc() and bpf_map_area_mmapable_alloc(). Still don't see how I can do bpf_map_area_mmapable_free() (normal bpf_map_area_free() will be able to free mmapable allocated memory, if caller adjusts the pointer correctly; adjustment though is specific to each use case, so can't be generalized).

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
kernel/bpf/arraymap.c +4 -1
      if (!array) {
              bpf_map_charge_finish(&mem);
              return ERR_PTR(-ENOMEM);
@@ -365,7 +392,10 @@ static void array_map_free(struct bpf_map *map)
      if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
              bpf_array_free_percpu(array);
 
-     bpf_map_area_free(array);
+     if (array->map.map_flags & BPF_F_MMAPABLE)
+             bpf_map_area_free((void *)round_down((long)array, PAGE_SIZE));
+     else
+             bpf_map_area_free(array);
  }
 
  static void array_map_seq_show_elem(struct bpf_map *map, void *key,

[...]

#8 lore raw

PATCH v3 2/3 libbpf: make global data internal arrays mmap()-able, if possible

In reply to Andrii Nakryiko

Add detection of BPF_F_MMAPABLE flag support for arrays and add it as an extra flag to internal global data maps, if supported by kernel. This allows users to memory-map global data and use it without BPF map operations, greatly simplifying user experience.

---
 tools/lib/bpf/libbpf.c | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)
tools/lib/bpf/libbpf.c +30 -2
index 96ef18cfeffb..7e2be1288fa1 100644
@@ -142,6 +142,8 @@ struct bpf_capabilities {
 	__u32 btf_func:1;
 	/* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 	__u32 btf_datasec:1;
+	/* BPF_F_MMAPABLE is supported for arrays */
+	__u32 array_mmap:1;
 };
 
 /*
@@ -856,8 +858,6 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 		pr_warn("failed to alloc map name\n");
 		return -ENOMEM;
 	}
-	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
-		 map_name, map->sec_idx, map->sec_offset);
 
 	def = &map->def;
 	def->type = BPF_MAP_TYPE_ARRAY;
@@ -865,6 +865,12 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 	def->value_size = data->d_size;
 	def->max_entries = 1;
 	def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0;
+	if (obj->caps.array_mmap)
+		def->map_flags |= BPF_F_MMAPABLE;
+
+	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
+		 map_name, map->sec_idx, map->sec_offset, def->map_flags);
+
 	if (data_buff) {
 		*data_buff = malloc(data->d_size);
 		if (!*data_buff) {
@@ -2160,6 +2166,27 @@ static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
 	return 0;
 }
 
+static int bpf_object__probe_array_mmap(struct bpf_object *obj)
+{
+	struct bpf_create_map_attr attr = {
+		.map_type = BPF_MAP_TYPE_ARRAY,
+		.map_flags = BPF_F_MMAPABLE,
+		.key_size = sizeof(int),
+		.value_size = sizeof(int),
+		.max_entries = 1,
+	};
+	int fd;
+
+	fd = bpf_create_map_xattr(&attr);
+	if (fd >= 0) {
+		obj->caps.array_mmap = 1;
+		close(fd);
+		return 1;
+	}
+
+	return 0;
+}
+
 static int
 bpf_object__probe_caps(struct bpf_object *obj)
 {
@@ -2168,6 +2195,7 @@ bpf_object__probe_caps(struct bpf_object *obj)
 		bpf_object__probe_global_data,
 		bpf_object__probe_btf_func,
 		bpf_object__probe_btf_datasec,
+		bpf_object__probe_array_mmap,
 	};
 	int i, ret;
 
Signature · 1 line
2.17.1
reviewer Acked-by on v3 2/3
#9 lore raw

RE: [PATCH v3 bpf-next 2/3] libbpf: make global data internal arrays mmap()-able, if possible

Quoted 8 lines · Andrii Nakryiko

Add detection of BPF_F_MMAPABLE flag support for arrays and add it as an extra flag to internal global data maps, if supported by kernel. This allows users to memory-map global data and use it without BPF map operations, greatly simplifying user experience.

---

[...]

Quoted 24 lines

/*

tools/lib/bpf/libbpf.c +6 -2
@@ -856,8 +858,6 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 		pr_warn("failed to alloc map name\n");
 		return -ENOMEM;
 	}
-	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
-		 map_name, map->sec_idx, map->sec_offset);
 
 	def = &map->def;
 	def->type = BPF_MAP_TYPE_ARRAY;
@@ -865,6 +865,12 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 	def->value_size = data->d_size;
 	def->max_entries = 1;
 	def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0;
+	if (obj->caps.array_mmap)
+		def->map_flags |= BPF_F_MMAPABLE;
+
+	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
+		 map_name, map->sec_idx, map->sec_offset, def->map_flags);
+
 	if (data_buff) {
 		*data_buff = malloc(data->d_size);
 		if (!*data_buff) {

tools/lib/bpf/libbpf.c static int bpf_object__probe_btf_datasec(struct bpf_object *obj) · line 2167

Quoted 2 lines
tools/lib/bpf/libbpf.c +0 -0
@@ -2160,6 +2166,27 @@ static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
 	return 0;
 }

I was a bit concerned we should fall back to making the call without the BPF_F_MMAPABLE flag set if it fails but did a quick walk through the call path and it seems like it shouldn't fail except if vmalloc/vzalloc failures so seems fine.

#10 lore raw

PATCH v3 3/3 selftests/bpf: add BPF_TYPE_MAP_ARRAY mmap() tests

In reply to Andrii Nakryiko

Add selftests validating mmap()-ing BPF array maps: both single-element and multi-element ones. Check that plain bpf_map_update_elem() and bpf_map_lookup_elem() work correctly with memory-mapped array. Also convert CO-RE relocation tests to use memory-mapped views of global data.

---
 .../selftests/bpf/prog_tests/core_reloc.c     |  45 +++--
 tools/testing/selftests/bpf/prog_tests/mmap.c | 170 ++++++++++++++++++
 tools/testing/selftests/bpf/progs/test_mmap.c |  41 +++++
 3 files changed, 238 insertions(+), 18 deletions(-)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/mmap.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_mmap.c
tools/testing/selftests/bpf/prog_tests/core_reloc.c +27 -18
index f94bd071536b..ec9e2fdd6b89 100644
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <test_progs.h>
 #include "progs/core_reloc_types.h"
+#include <sys/mman.h>
 
 #define STRUCT_TO_CHAR_PTR(struct_name) (const char *)&(struct struct_name)
 
@@ -453,8 +454,15 @@ struct data {
 	char out[256];
 };
 
+static size_t roundup_page(size_t sz)
+{
+	long page_size = sysconf(_SC_PAGE_SIZE);
+	return (sz + page_size - 1) / page_size * page_size;
+}
+
 void test_core_reloc(void)
 {
+	const size_t mmap_sz = roundup_page(sizeof(struct data));
 	struct bpf_object_load_attr load_attr = {};
 	struct core_reloc_test_case *test_case;
 	const char *tp_name, *probe_name;
@@ -463,8 +471,8 @@ void test_core_reloc(void)
 	struct bpf_map *data_map;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
-	const int zero = 0;
-	struct data data;
+	struct data *data;
+	void *mmap_data = NULL;
 
 	for (i = 0; i < ARRAY_SIZE(test_cases); i++) {
 		test_case = &test_cases[i];
@@ -476,8 +484,7 @@ void test_core_reloc(void)
 		);
 
 		obj = bpf_object__open_file(test_case->bpf_obj_file, &opts);
-		if (CHECK(IS_ERR_OR_NULL(obj), "obj_open",
-			  "failed to open '%s': %ld\n",
+		if (CHECK(IS_ERR(obj), "obj_open", "failed to open '%s': %ld\n",
 			  test_case->bpf_obj_file, PTR_ERR(obj)))
 			continue;
 
@@ -519,24 +526,22 @@ void test_core_reloc(void)
 		if (CHECK(!data_map, "find_data_map", "data map not found\n"))
 			goto cleanup;
 
-		memset(&data, 0, sizeof(data));
-		memcpy(data.in, test_case->input, test_case->input_len);
-
-		err = bpf_map_update_elem(bpf_map__fd(data_map),
-					  &zero, &data, 0);
-		if (CHECK(err, "update_data_map",
-			  "failed to update .data map: %d\n", err))
+		mmap_data = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
+				 MAP_SHARED, bpf_map__fd(data_map), 0);
+		if (CHECK(mmap_data == MAP_FAILED, "mmap",
+			  ".bss mmap failed: %d", errno)) {
+			mmap_data = NULL;
 			goto cleanup;
+		}
+		data = mmap_data;
+
+		memset(mmap_data, 0, sizeof(*data));
+		memcpy(data->in, test_case->input, test_case->input_len);
 
 		/* trigger test run */
 		usleep(1);
 
-		err = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, &data);
-		if (CHECK(err, "get_result",
-			  "failed to get output data: %d\n", err))
-			goto cleanup;
-
-		equal = memcmp(data.out, test_case->output,
+		equal = memcmp(data->out, test_case->output,
 			       test_case->output_len) == 0;
 		if (CHECK(!equal, "check_result",
 			  "input/output data don't match\n")) {
@@ -548,12 +553,16 @@ void test_core_reloc(void)
 			}
 			for (j = 0; j < test_case->output_len; j++) {
 				printf("output byte #%d: EXP 0x%02hhx GOT 0x%02hhx\n",
-				       j, test_case->output[j], data.out[j]);
+				       j, test_case->output[j], data->out[j]);
 			}
 			goto cleanup;
 		}
 
 cleanup:
+		if (mmap_data) {
+			CHECK_FAIL(munmap(mmap_data, mmap_sz));
+			mmap_data = NULL;
+		}
 		if (!IS_ERR_OR_NULL(link)) {
 			bpf_link__destroy(link);
 			link = NULL;
tools/testing/selftests/bpf/prog_tests/mmap.c new file +170 -0

170 lines not on this page · raw message

tools/testing/selftests/bpf/progs/test_mmap.c new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.17.1