Skip to contentSkip to conversation

libbpf: add bpf_map__reuse_fd_from_loader_ctx() to allow shared maps

PATCH In review John Fastabend 2 participants 4 messages Jun 26 – Jul 1

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 libbpf: add bpf_map__reuse_fd_from_loader_ctx() to allow shared maps

This lets the gen_loader use a map that may already be loaded/pinned on the target system. When we are loading many signed programs on the target we often need them to share maps to communicate/pass data between the programs.

Note it is slightly complicated by needing to ensure on error/close that we only emit_sys_close_blob() for map ids that are not these shared type otherwise we could close a shared map unintentionally. To track this we added a borrowed_fd_array.

---
 tools/lib/bpf/bpf_gen_internal.h |  2 ++
 tools/lib/bpf/gen_loader.c       | 50 ++++++++++++++++++++++++++++++--
 tools/lib/bpf/libbpf.c           | 11 +++++++
 tools/lib/bpf/libbpf.h           |  1 +
 tools/lib/bpf/libbpf.map         |  1 +
 5 files changed, 63 insertions(+), 2 deletions(-)
tools/lib/bpf/bpf_gen_internal.h +2 -0
index 49af4260b8e6..e555c3165105 100644
@@ -50,6 +50,7 @@ struct bpf_gen {
 	struct ksym_desc *ksyms;
 	__u32 nr_ksyms;
 	int fd_array;
+	int borrowed_fd_array;
 	int nr_fd_array;
 	int hash_insn_offset[SHA256_DWORD_SIZE];
 };
@@ -62,6 +63,7 @@ void bpf_gen__map_create(struct bpf_gen *gen,
 			 enum bpf_map_type map_type, const char *map_name,
 			 __u32 key_size, __u32 value_size, __u32 max_entries,
 			 struct bpf_map_create_opts *map_attr, int map_idx);
+void bpf_gen__map_reuse_fd(struct bpf_gen *gen, int map_idx);
 void bpf_gen__prog_load(struct bpf_gen *gen,
 			enum bpf_prog_type prog_type, const char *prog_name,
 			const char *license, struct bpf_insn *insns, size_t insn_cnt,
tools/lib/bpf/gen_loader.c +48 -2

100 lines not on this page · raw message

tools/lib/bpf/libbpf.c +11 -0
index 7162146280a8..af1553e9d4df 100644
@@ -5085,6 +5085,15 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd)
 	return libbpf_err(err);
 }
 
+int bpf_map__reuse_fd_from_loader_ctx(struct bpf_map *map)
+{
+	if (map_is_created(map))
+		return libbpf_err(-EBUSY);
+
+	map->reused = true;
+	return 0;
+}
+
 __u32 bpf_map__max_entries(const struct bpf_map *map)
 {
 	return map->def.max_entries;
@@ -5659,6 +5668,8 @@ bpf_object__create_maps(struct bpf_object *obj)
 		}
 
 		if (map->reused) {
+			if (obj->gen_loader)
+				bpf_gen__map_reuse_fd(obj->gen_loader, map - obj->maps);
 			pr_debug("map '%s': skipping creation (preset fd=%d)\n",
 				 map->name, map->fd);
 		} else {
tools/lib/bpf/libbpf.h +1 -0
index b965ad571540..4145a16c8963 100644
@@ -1135,6 +1135,7 @@ LIBBPF_API bool bpf_map__autoattach(const struct bpf_map *map);
  */
 LIBBPF_API int bpf_map__fd(const struct bpf_map *map);
 LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd);
+LIBBPF_API int bpf_map__reuse_fd_from_loader_ctx(struct bpf_map *map);
 /* get map name */
 LIBBPF_API const char *bpf_map__name(const struct bpf_map *map);
 /* get/set map type */
tools/lib/bpf/libbpf.map +1 -0
index b731df19ae69..d6c2f1d75cda 100644
@@ -458,6 +458,7 @@ LIBBPF_1.7.0 {
 
 LIBBPF_1.8.0 {
 	global:
+		bpf_map__reuse_fd_from_loader_ctx;
 		bpf_program__attach_tracing_multi;
 		bpf_program__clone;
 		btf__new_empty_opts;
Signature · 1 line
2.53.0
#1 lore raw

PATCH 2/2 libbpf: selftest for bpf_map__reuse_fd_from_loader_ctx()

Extend signed loader test case to exercise __reuse_fd_from_loader_ctx().

---
 .../selftests/bpf/prog_tests/signed_loader.c  | 115 ++++++++++++++++++
 1 file changed, 115 insertions(+)
tools/testing/selftests/bpf/prog_tests/signed_loader.c +115 -0
index 5fc417e31fc6..0af33362ab1b 100644
@@ -8,8 +8,10 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <limits.h>
+#include <unistd.h>
 #include <linux/keyctl.h>
 #include <linux/bpf.h>
+#include <linux/if_ether.h>
 
 #include "bpf/libbpf_internal.h" /* for libbpf_sha256() */
 #include "bpf/skel_internal.h"	 /* for loader ctx layout (bpf_loader_ctx etc) */
@@ -699,6 +701,117 @@ static void metadata_ctx_initial_value_ignored(void)
 	test_signed_loader_data__destroy(skel);
 }
 
+static void loader_ctx_reuse_fd_shared_map(void)
+{
+	LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true);
+	struct test_signed_loader_map *skel = NULL;
+	struct bpf_map_info shared_info, reused_info;
+	__u32 shared_info_len, reused_info_len;
+	__u8 excl[SHA256_DIGEST_LENGTH];
+	__u8 pkt[ETH_HLEN] = {};
+	__u32 key = 0, expected_retval = 42;
+	__u64 shared_value = expected_retval;
+	LIBBPF_OPTS(bpf_test_run_opts, topts,
+		.data_in = pkt,
+		.data_size_in = sizeof(pkt),
+		.repeat = 1,
+	);
+	int nr_maps = 0, nr_progs = 0, ctx_fd = -1, shared_fd = -1;
+	int supplied_fd, r;
+	struct bpf_prog_desc *pd;
+	struct bpf_map_desc *md;
+	struct bpf_program *p;
+	struct bpf_map *m;
+	unsigned char *blob = NULL;
+	__u32 ctx_sz, data_sz;
+	void *ctx = NULL;
+	bool ran;
+
+	skel = test_signed_loader_map__open();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_map__reuse_fd_from_loader_ctx(skel->maps.amap),
+		       "reuse_fd_from_loader_ctx"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load"))
+		goto cleanup;
+
+	bpf_object__for_each_program(p, skel->obj)
+		nr_progs++;
+	bpf_object__for_each_map(m, skel->obj)
+		nr_maps++;
+	if (!ASSERT_EQ(nr_maps, 1, "one map") ||
+	    !ASSERT_EQ(nr_progs, 1, "one prog"))
+		goto cleanup;
+
+	shared_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "ctx_reuse",
+				   sizeof(__u32), sizeof(__u64), 4, NULL);
+	if (!ASSERT_OK_FD(shared_fd, "shared_map"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_map_update_elem(shared_fd, &key, &shared_value, BPF_ANY),
+		       "shared_map_update"))
+		goto cleanup;
+
+	ctx_sz = sizeof(struct bpf_loader_ctx) +
+		 nr_maps * sizeof(struct bpf_map_desc) +
+		 nr_progs * sizeof(struct bpf_prog_desc);
+	ctx = calloc(1, ctx_sz);
+	if (!ASSERT_OK_PTR(ctx, "ctx_alloc"))
+		goto cleanup;
+	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
+
+	md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx));
+	pd = (struct bpf_prog_desc *)(md + nr_maps);
+	ctx_fd = dup(shared_fd);
+	if (!ASSERT_OK_FD(ctx_fd, "ctx_map_fd"))
+		goto cleanup;
+	md[0].map_fd = ctx_fd;
+	ctx_fd = -1;
+	supplied_fd = md[0].map_fd;
+
+	libbpf_sha256(gopts.insns, gopts.insns_sz, excl);
+	data_sz = gopts.data_sz;
+	blob = malloc(data_sz);
+	if (!ASSERT_OK_PTR(blob, "blob_alloc"))
+		goto cleanup;
+	memcpy(blob, gopts.data, data_sz);
+
+	r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz,
+			   excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran);
+	if (!ASSERT_TRUE(ran, "loader ran") ||
+	    !ASSERT_EQ(r, 0, "loader retval"))
+		goto cleanup;
+
+	ASSERT_EQ(md[0].map_fd, supplied_fd, "ctx map fd preserved");
+	ASSERT_GT(pd[0].prog_fd, 0, "prog fd populated");
+	if (ASSERT_OK(bpf_prog_test_run_opts(pd[0].prog_fd, &topts),
+		      "shared map prog run"))
+		ASSERT_EQ(topts.retval, expected_retval, "shared map value");
+
+	memset(&shared_info, 0, sizeof(shared_info));
+	memset(&reused_info, 0, sizeof(reused_info));
+	shared_info_len = sizeof(shared_info);
+	reused_info_len = sizeof(reused_info);
+	if (ASSERT_OK(bpf_map_get_info_by_fd(shared_fd, &shared_info, &shared_info_len),
+		      "shared map info") &&
+	    ASSERT_OK(bpf_map_get_info_by_fd(md[0].map_fd, &reused_info, &reused_info_len),
+		      "reused map info"))
+		ASSERT_EQ(reused_info.id, shared_info.id, "shared map reused");
+
+cleanup:
+	if (ctx)
+		close_loader_ctx_fds(ctx, nr_maps, nr_progs);
+	if (ctx_fd >= 0)
+		close(ctx_fd);
+	if (shared_fd >= 0)
+		close(shared_fd);
+	free(blob);
+	free(ctx);
+	test_signed_loader_map__destroy(skel);
+}
+
 /*
  * The load-time signature must authenticate the loader instructions: a valid
  * signature loads, and the very same signature over one-byte-tampered insns is
@@ -1112,6 +1225,8 @@ void test_signed_loader(void)
 		metadata_ctx_max_entries_ignored();
 	if (test__start_subtest("metadata_ctx_initial_value_ignored"))
 		metadata_ctx_initial_value_ignored();
+	if (test__start_subtest("loader_ctx_reuse_fd_shared_map"))
+		loader_ctx_reuse_fd_shared_map();
 	if (test__start_subtest("signature_authenticates_insns"))
 		signature_authenticates_insns();
 	if (test__start_subtest("hash_requires_frozen"))
Signature · 1 line
2.53.0
#2 lore raw
In reply to John Fastabend
Quoted 20 lines · John Fastabend

This lets the gen_loader use a map that may already be loaded/pinned on the target system. When we are loading many signed programs on the target we often need them to share maps to communicate/pass data between the programs.

Note it is slightly complicated by needing to ensure on error/close that we only emit_sys_close_blob() for map ids that are not these shared type otherwise we could close a shared map unintentionally. To track this we added a borrowed_fd_array.

---
 tools/lib/bpf/bpf_gen_internal.h |  2 ++
 tools/lib/bpf/gen_loader.c       | 50 ++++++++++++++++++++++++++++++--
 tools/lib/bpf/libbpf.c           | 11 +++++++
 tools/lib/bpf/libbpf.h           |  1 +
 tools/lib/bpf/libbpf.map         |  1 +
 5 files changed, 63 insertions(+), 2 deletions(-)

[...]

Quoted 33 lines
tools/lib/bpf/gen_loader.c +22 -0
+void bpf_gen__map_reuse_fd(struct bpf_gen *gen, int map_idx)
+{
+       int idx;
+
+       if (map_idx != gen->nr_maps) {
+               gen->error = -EDOM;
+               return;
+       }
+
+       idx = add_map_fd(gen);
+       if (gen->error)
+               return;
+
+       set_blob_borrowed_fd_array_off(gen, idx);
+       move_ctx2blob(gen, blob_fd_array_off(gen, idx), 4,
+                     sizeof(struct bpf_loader_ctx) +
+                     sizeof(struct bpf_map_desc) * map_idx +
+                     offsetof(struct bpf_map_desc, map_fd),
+                     false);
+       pr_debug("gen: map_reuse_fd idx %d\n", map_idx);
+}
+
 static void emit_signature_match(struct bpf_gen *gen)
 {
        __s64 off;
tools/lib/bpf/libbpf.c +0 -0
index 7162146280a8..af1553e9d4df 100644
@@ -5085,6 +5085,15 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd)
        return libbpf_err(err);
 }

tools/lib/bpf/libbpf.c int bpf_map__reuse_fd(struct bpf_map *map, int fd) · line 5096

Quoted 10 lines
tools/lib/bpf/libbpf.c +9 -0
 
+int bpf_map__reuse_fd_from_loader_ctx(struct bpf_map *map)
+{
+       if (map_is_created(map))
+               return libbpf_err(-EBUSY);
+
+       map->reused = true;
+       return 0;
+}
+

Can you please give a small example on how this is going to be used? E.g., who calls this API and at which point?

Generally speaking, I'm not a fan of such specific narrow-purpose APIs. Why can't we just teach bpf_map__reuse_fd() to do this if gen_loader is set for the object?.. Sure, fd would be ignored and/or can be -1 or something, but that's a gen_loader situation, that's fine, right?

pw-bot: cr

Quoted 40 lines
tools/lib/bpf/libbpf.c +2 -0
 __u32 bpf_map__max_entries(const struct bpf_map *map)
 {
        return map->def.max_entries;
@@ -5659,6 +5668,8 @@ bpf_object__create_maps(struct bpf_object *obj)
                }
 
                if (map->reused) {
+                       if (obj->gen_loader)
+                               bpf_gen__map_reuse_fd(obj->gen_loader, map - obj->maps);
                        pr_debug("map '%s': skipping creation (preset fd=%d)\n",
                                 map->name, map->fd);
                } else {
tools/lib/bpf/libbpf.h +1 -0
index b965ad571540..4145a16c8963 100644
@@ -1135,6 +1135,7 @@ LIBBPF_API bool bpf_map__autoattach(const struct bpf_map *map);
  */
 LIBBPF_API int bpf_map__fd(const struct bpf_map *map);
 LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd);
+LIBBPF_API int bpf_map__reuse_fd_from_loader_ctx(struct bpf_map *map);
 /* get map name */
 LIBBPF_API const char *bpf_map__name(const struct bpf_map *map);
 /* get/set map type */
tools/lib/bpf/libbpf.map +1 -1
index b731df19ae69..d6c2f1d75cda 100644
@@ -458,6 +458,7 @@ LIBBPF_1.7.0 {
 
 LIBBPF_1.8.0 {
        global:
+               bpf_map__reuse_fd_from_loader_ctx;
                bpf_program__attach_tracing_multi;
                bpf_program__clone;
                btf__new_empty_opts;
--

2.53.0

#3 lore raw
Quoted 58 lines · Andrii Nakryiko
Quoted 20 lines · John Fastabend

This lets the gen_loader use a map that may already be loaded/pinned on the target system. When we are loading many signed programs on the target we often need them to share maps to communicate/pass data between the programs.

Note it is slightly complicated by needing to ensure on error/close that we only emit_sys_close_blob() for map ids that are not these shared type otherwise we could close a shared map unintentionally. To track this we added a borrowed_fd_array.

---
 tools/lib/bpf/bpf_gen_internal.h |  2 ++
 tools/lib/bpf/gen_loader.c       | 50 ++++++++++++++++++++++++++++++--
 tools/lib/bpf/libbpf.c           | 11 +++++++
 tools/lib/bpf/libbpf.h           |  1 +
 tools/lib/bpf/libbpf.map         |  1 +
 5 files changed, 63 insertions(+), 2 deletions(-)

[...]

Quoted 34 lines
tools/lib/bpf/gen_loader.c +22 -0
+void bpf_gen__map_reuse_fd(struct bpf_gen *gen, int map_idx)
+{
+       int idx;
+
+       if (map_idx != gen->nr_maps) {
+               gen->error = -EDOM;
+               return;
+       }
+
+       idx = add_map_fd(gen);
+       if (gen->error)
+               return;
+
+       set_blob_borrowed_fd_array_off(gen, idx);
+       move_ctx2blob(gen, blob_fd_array_off(gen, idx), 4,
+                     sizeof(struct bpf_loader_ctx) +
+                     sizeof(struct bpf_map_desc) * map_idx +
+                     offsetof(struct bpf_map_desc, map_fd),
+                     false);
+       pr_debug("gen: map_reuse_fd idx %d\n", map_idx);
+}
+
 static void emit_signature_match(struct bpf_gen *gen)
 {
        __s64 off;
tools/lib/bpf/libbpf.c +0 -0
index 7162146280a8..af1553e9d4df 100644
@@ -5085,6 +5085,15 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd)
        return libbpf_err(err);
 }
 
Quoted 12 lines

tools/lib/bpf/libbpf.c int bpf_map__reuse_fd(struct bpf_map *map, int fd) · line 5096

Quoted 9 lines
tools/lib/bpf/libbpf.c +9 -0
+int bpf_map__reuse_fd_from_loader_ctx(struct bpf_map *map)
+{
+       if (map_is_created(map))
+               return libbpf_err(-EBUSY);
+
+       map->reused = true;
+       return 0;
+}
+

Can you please give a small example on how this is going to be used? E.g., who calls this API and at which point?

This is our package builder creating signed package for loading on remote systems. Here different packages want to share an arena. I'll create a more complete example.

Quoted 6 lines

Generally speaking, I'm not a fan of such specific narrow-purpose APIs. Why can't we just teach bpf_map__reuse_fd() to do this if gen_loader is set for the object?.. Sure, fd would be ignored and/or can be -1 or something, but that's a gen_loader situation, that's fine, right?

If -1 is good that will work then we have bpf_map__reuse_fd(map, -1); We can just define REUSE_FD_SHARED_MAP -1.

I'll send a v2 using bpf_map__reuse_fd(map, REUSE_FD_SHARED_MAP) semantic.

Thanks.