Skip to contentSkip to conversation

libbpf: Support uniform BTF-defined key/value specification across all BPF maps

PATCH In review Hengqi Chen 2 participants 8 messages Sep 5, 2021 – Oct 2, 2021

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 libbpf: Support uniform BTF-defined key/value specification across all BPF maps

A bunch of BPF maps do not support specifying types for key and value. This is non-uniform and inconvenient[0]. Currently, libbpf uses a retry logic which removes BTF type IDs when BPF map creation failed. Instead of retrying, this commit recognizes those specialized map and removes BTF type IDs when creating BPF map.

[0] Closes: https://github.com/libbpf/libbpf/issues/355

---
 tools/lib/bpf/libbpf.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
tools/lib/bpf/libbpf.c +20 -15
index 88d8825fc6f6..7068c4d07337 100644
@@ -4613,6 +4613,26 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
 			create_attr.inner_map_fd = map->inner_map_fd;
 	}
 
+	if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
+	    def->type == BPF_MAP_TYPE_STACK_TRACE ||
+	    def->type == BPF_MAP_TYPE_CGROUP_ARRAY ||
+	    def->type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
+	    def->type == BPF_MAP_TYPE_HASH_OF_MAPS ||
+	    def->type == BPF_MAP_TYPE_DEVMAP ||
+	    def->type == BPF_MAP_TYPE_SOCKMAP ||
+	    def->type == BPF_MAP_TYPE_CPUMAP ||
+	    def->type == BPF_MAP_TYPE_XSKMAP ||
+	    def->type == BPF_MAP_TYPE_SOCKHASH ||
+	    def->type == BPF_MAP_TYPE_QUEUE ||
+	    def->type == BPF_MAP_TYPE_STACK ||
+	    def->type == BPF_MAP_TYPE_DEVMAP_HASH) {
+		create_attr.btf_fd = 0;
+		create_attr.btf_key_type_id = 0;
+		create_attr.btf_value_type_id = 0;
+		map->btf_key_type_id = 0;
+		map->btf_value_type_id = 0;
+	}
+
 	if (obj->gen_loader) {
 		bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps);
 		/* Pretend to have valid FD to pass various fd >= 0 checks.
@@ -4622,21 +4642,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
 	} else {
 		map->fd = bpf_create_map_xattr(&create_attr);
 	}
-	if (map->fd < 0 && (create_attr.btf_key_type_id ||
-			    create_attr.btf_value_type_id)) {
-		char *cp, errmsg[STRERR_BUFSIZE];
-
-		err = -errno;
-		cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
-		pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
-			map->name, cp, err);
-		create_attr.btf_fd = 0;
-		create_attr.btf_key_type_id = 0;
-		create_attr.btf_value_type_id = 0;
-		map->btf_key_type_id = 0;
-		map->btf_value_type_id = 0;
-		map->fd = bpf_create_map_xattr(&create_attr);
-	}
 
 	err = map->fd < 0 ? -errno : 0;
 
Signature · 1 line
2.25.1
#1 lore raw

PATCH 2/2 selftests/bpf: Test BPF map creation using BTF-defined key/value

Test BPF map creation using BTF-defined key/value. The test defines some specialized maps by specifying BTF types for key/value and checks those maps are correctly initialized and loaded.

---
 .../selftests/bpf/prog_tests/map_create.c     |  87 ++++++++++++++
 .../selftests/bpf/progs/test_map_create.c     | 110 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/map_create.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_map_create.c
tools/testing/selftests/bpf/prog_tests/map_create.c new file +87 -0
new file mode 100644
index 000000000000..6ca32d0dffd2
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#include <test_progs.h>
+#include "test_map_create.skel.h"
+
+void test_map_create(void)
+{
+	struct test_map_create *skel;
+	int err, fd;
+
+	skel = test_map_create__open();
+	if (!ASSERT_OK_PTR(skel, "test_map_create__open failed"))
+		return;
+
+	err = test_map_create__load(skel);
+	if (!ASSERT_OK(err, "test_map_create__load failed"))
+		goto cleanup;
+
+	fd = bpf_map__fd(skel->maps.map1);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map2);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map3);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map4);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map5);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map6);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map7);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map8);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map9);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map10);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map11);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map12);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+	fd = bpf_map__fd(skel->maps.map13);
+	if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+		goto cleanup;
+	close(fd);
+
+cleanup:
+	test_map_create__destroy(skel);
+}
tools/testing/selftests/bpf/progs/test_map_create.c new file +110 -0

110 lines not on this page · raw message

Signature · 1 line
2.25.1
#2 lore raw

Re: [PATCH bpf-next 2/2] selftests/bpf: Test BPF map creation using BTF-defined key/value

Quoted 25 lines · Hengqi Chen

Test BPF map creation using BTF-defined key/value. The test defines some specialized maps by specifying BTF types for key/value and checks those maps are correctly initialized and loaded.

---
 .../selftests/bpf/prog_tests/map_create.c     |  87 ++++++++++++++
 .../selftests/bpf/progs/test_map_create.c     | 110 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/map_create.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_map_create.c
tools/testing/selftests/bpf/prog_tests/map_create.c new file +6 -0
new file mode 100644
index 000000000000..6ca32d0dffd2
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#include <test_progs.h>
+#include "test_map_create.skel.h"
+

tools/testing/selftests/bpf/prog_tests/map_create.c line 16

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +10 -0
+void test_map_create(void)
+{
+       struct test_map_create *skel;
+       int err, fd;
+
+       skel = test_map_create__open();
+       if (!ASSERT_OK_PTR(skel, "test_map_create__open failed"))
+               return;
+
+       err = test_map_create__load(skel);

If load() succeeds, all the maps will definitely be created, so all the below tests are meaningless.

I think it's better to just change all the existing map definitions used throughout selftests to use key/value types, instead of key_size/value_size. That will automatically test this feature without adding an extra test. Unfortunately to really test that the logic is working, we'd need to check that libbpf doesn't emit the warning about retrying map creation w/o BTF, but I think one-time manual check (please use ./test_progs -v to see libbpf warnings during tests) should be sufficient for this.

Quoted 12 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +12 -0
+       if (!ASSERT_OK(err, "test_map_create__load failed"))
+               goto cleanup;
+
+       fd = bpf_map__fd(skel->maps.map1);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);
+
+       fd = bpf_map__fd(skel->maps.map2);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);

[...]

#3 lore raw

Re: [PATCH bpf-next 2/2] selftests/bpf: Test BPF map creation using BTF-defined key/value

Quoted 40 lines · Andrii Nakryiko
Quoted 25 lines · Hengqi Chen

Test BPF map creation using BTF-defined key/value. The test defines some specialized maps by specifying BTF types for key/value and checks those maps are correctly initialized and loaded.

---
 .../selftests/bpf/prog_tests/map_create.c     |  87 ++++++++++++++
 .../selftests/bpf/progs/test_map_create.c     | 110 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/map_create.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_map_create.c
tools/testing/selftests/bpf/prog_tests/map_create.c new file +6 -0
new file mode 100644
index 000000000000..6ca32d0dffd2
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#include <test_progs.h>
+#include "test_map_create.skel.h"
+

tools/testing/selftests/bpf/prog_tests/map_create.c line 16

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +10 -0
+void test_map_create(void)
+{
+       struct test_map_create *skel;
+       int err, fd;
+
+       skel = test_map_create__open();
+       if (!ASSERT_OK_PTR(skel, "test_map_create__open failed"))
+               return;
+
+       err = test_map_create__load(skel);

If load() succeeds, all the maps will definitely be created, so all the below tests are meaningless.

Quoted 9 lines

I think it's better to just change all the existing map definitions used throughout selftests to use key/value types, instead of key_size/value_size. That will automatically test this feature without adding an extra test. Unfortunately to really test that the logic is working, we'd need to check that libbpf doesn't emit the warning about retrying map creation w/o BTF, but I think one-time manual check (please use ./test_progs -v to see libbpf warnings during tests) should be sufficient for this.

Hello, Andrii

I updated these existing tests as you suggested, but I was unable to run the whole bpf selftests locally.

Running ./test_progs -v made my system hang up,
didn't find the root cause yet.

Instead I just run the modified test with the following commands:

sudo ./test_progs -v --name=kfree_skb,perf_event_stackmap,btf_map_in_map,pe_preserve_elems,stacktrace_map,stacktrace_build_id,xdp_bpf2bpf,select_reuseport,tcpbpf_user

Quoted 15 lines
Quoted 12 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +12 -0
+       if (!ASSERT_OK(err, "test_map_create__load failed"))
+               goto cleanup;
+
+       fd = bpf_map__fd(skel->maps.map1);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);
+
+       fd = bpf_map__fd(skel->maps.map2);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);

[...]

#4 lore raw

Re: [PATCH bpf-next 2/2] selftests/bpf: Test BPF map creation using BTF-defined key/value

Quoted 54 lines · Hengqi Chen
Quoted 40 lines · Andrii Nakryiko
Quoted 25 lines · Hengqi Chen

Test BPF map creation using BTF-defined key/value. The test defines some specialized maps by specifying BTF types for key/value and checks those maps are correctly initialized and loaded.

---
 .../selftests/bpf/prog_tests/map_create.c     |  87 ++++++++++++++
 .../selftests/bpf/progs/test_map_create.c     | 110 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/map_create.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_map_create.c
tools/testing/selftests/bpf/prog_tests/map_create.c new file +6 -0
new file mode 100644
index 000000000000..6ca32d0dffd2
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#include <test_progs.h>
+#include "test_map_create.skel.h"
+

tools/testing/selftests/bpf/prog_tests/map_create.c line 16

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +10 -0
+void test_map_create(void)
+{
+       struct test_map_create *skel;
+       int err, fd;
+
+       skel = test_map_create__open();
+       if (!ASSERT_OK_PTR(skel, "test_map_create__open failed"))
+               return;
+
+       err = test_map_create__load(skel);

If load() succeeds, all the maps will definitely be created, so all the below tests are meaningless.

Quoted 9 lines

I think it's better to just change all the existing map definitions used throughout selftests to use key/value types, instead of key_size/value_size. That will automatically test this feature without adding an extra test. Unfortunately to really test that the logic is working, we'd need to check that libbpf doesn't emit the warning about retrying map creation w/o BTF, but I think one-time manual check (please use ./test_progs -v to see libbpf warnings during tests) should be sufficient for this.

Quoted 7 lines

Hello, Andrii

I updated these existing tests as you suggested, but I was unable to run the whole bpf selftests locally.

Running ./test_progs -v made my system hang up,
didn't find the root cause yet.

This is strange. Do you know at which test this happens? Do you get kernel warning/oops when this happens in dmesg?

But overall, the development and testing workflow for people working on bpf/bpf-next involves building latest kernel and running selftests inside the QEMU VM for testing. We also have vmtest.sh script in selftests/bpf that automates a lot of building steps. It will build kernel, test_progs and other selftest binaries, and will spin up QEMU VM with the same image that our BPF CIs are using. You just need to have very recent/latest Clang available and similarly pahole (from dwarves package) should be up to date and available through $PATH. After that, running ./vmtest.sh will just run all ./test_progs automatically.

Either way, our CI will also run your changes through the tests (except there are some intermittent issues right now, so we'll have to wait a bit for that to kick in). You can monitor [0], or the link will actually appear on each of your patches (e.g., [1]) in "Checks" section, once everything is up and running properly.

[0] https://github.com/kernel-patches/bpf/pulls
[1] https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

Quoted 20 lines

Instead I just run the modified test with the following commands:

sudo ./test_progs -v --name=kfree_skb,perf_event_stackmap,btf_map_in_map,pe_preserve_elems,stacktrace_map,stacktrace_build_id,xdp_bpf2bpf,select_reuseport,tcpbpf_user

Quoted 15 lines
Quoted 12 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +12 -0
+       if (!ASSERT_OK(err, "test_map_create__load failed"))
+               goto cleanup;
+
+       fd = bpf_map__fd(skel->maps.map1);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);
+
+       fd = bpf_map__fd(skel->maps.map2);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);

[...]

#5 lore raw

Re: [PATCH bpf-next 2/2] selftests/bpf: Test BPF map creation using BTF-defined key/value

Quoted 57 lines · Andrii Nakryiko
Quoted 56 lines · Hengqi Chen
Quoted 40 lines · Andrii Nakryiko
Quoted 25 lines · Hengqi Chen

Test BPF map creation using BTF-defined key/value. The test defines some specialized maps by specifying BTF types for key/value and checks those maps are correctly initialized and loaded.

---
 .../selftests/bpf/prog_tests/map_create.c     |  87 ++++++++++++++
 .../selftests/bpf/progs/test_map_create.c     | 110 ++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/prog_tests/map_create.c
 create mode 100644 tools/testing/selftests/bpf/progs/test_map_create.c
tools/testing/selftests/bpf/prog_tests/map_create.c new file +6 -0
new file mode 100644
index 000000000000..6ca32d0dffd2
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021 Hengqi Chen */
+
+#include <test_progs.h>
+#include "test_map_create.skel.h"
+

tools/testing/selftests/bpf/prog_tests/map_create.c line 16

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +10 -0
+void test_map_create(void)
+{
+       struct test_map_create *skel;
+       int err, fd;
+
+       skel = test_map_create__open();
+       if (!ASSERT_OK_PTR(skel, "test_map_create__open failed"))
+               return;
+
+       err = test_map_create__load(skel);

If load() succeeds, all the maps will definitely be created, so all the below tests are meaningless.

Quoted 9 lines

I think it's better to just change all the existing map definitions used throughout selftests to use key/value types, instead of key_size/value_size. That will automatically test this feature without adding an extra test. Unfortunately to really test that the logic is working, we'd need to check that libbpf doesn't emit the warning about retrying map creation w/o BTF, but I think one-time manual check (please use ./test_progs -v to see libbpf warnings during tests) should be sufficient for this.

Hello, Andrii

Quoted 9 lines
Quoted 5 lines

I updated these existing tests as you suggested, but I was unable to run the whole bpf selftests locally.

Running ./test_progs -v made my system hang up,
didn't find the root cause yet.

This is strange. Do you know at which test this happens? Do you get kernel warning/oops when this happens in dmesg?

No, when this situation occurred, I just restarted my laptop. Will look into this later.

Quoted 11 lines

But overall, the development and testing workflow for people working on bpf/bpf-next involves building latest kernel and running selftests inside the QEMU VM for testing. We also have vmtest.sh script in selftests/bpf that automates a lot of building steps. It will build kernel, test_progs and other selftest binaries, and will spin up QEMU VM with the same image that our BPF CIs are using. You just need to have very recent/latest Clang available and similarly pahole (from dwarves package) should be up to date and available through $PATH. After that, running ./vmtest.sh will just run all ./test_progs automatically.

This workflow info is quite useful for me. Thanks.

Quoted 29 lines

Either way, our CI will also run your changes through the tests (except there are some intermittent issues right now, so we'll have to wait a bit for that to kick in). You can monitor [0], or the link will actually appear on each of your patches (e.g., [1]) in "Checks" section, once everything is up and running properly.

[0] https://github.com/kernel-patches/bpf/pulls
[1] https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/

Quoted 20 lines

Instead I just run the modified test with the following commands:

sudo ./test_progs -v --name=kfree_skb,perf_event_stackmap,btf_map_in_map,pe_preserve_elems,stacktrace_map,stacktrace_build_id,xdp_bpf2bpf,select_reuseport,tcpbpf_user

Quoted 15 lines
Quoted 12 lines
tools/testing/selftests/bpf/prog_tests/map_create.c new file +12 -0
+       if (!ASSERT_OK(err, "test_map_create__load failed"))
+               goto cleanup;
+
+       fd = bpf_map__fd(skel->maps.map1);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);
+
+       fd = bpf_map__fd(skel->maps.map2);
+       if (!ASSERT_GT(fd, 0, "bpf_map__fd failed"))
+               goto cleanup;
+       close(fd);

[...]

#6 lore raw
In reply to Hengqi Chen
Quoted 2 lines · Hengqi Chen

A bunch of BPF maps do not support specifying types for key and value.

s/types/BTF types/, it's a bit confusing otherwise

Quoted 3 lines

This is non-uniform and inconvenient[0]. Currently, libbpf uses a retry logic which removes BTF type IDs when BPF map creation failed. Instead of retrying, this commit recognizes those specialized map and removes

s/map/maps/

Quoted 6 lines

BTF type IDs when creating BPF map.

[0] Closes: https://github.com/libbpf/libbpf/issues/355

---

For patch sets consisting of two or more patches, we ask for a cover letter, so for the next revision please provide a cover letter with an overall description of what the series is about.

Quoted 20 lines
 tools/lib/bpf/libbpf.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
tools/lib/bpf/libbpf.c +9 -0
index 88d8825fc6f6..7068c4d07337 100644
@@ -4613,6 +4613,26 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
                        create_attr.inner_map_fd = map->inner_map_fd;
        }
 
+       if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
+           def->type == BPF_MAP_TYPE_STACK_TRACE ||
+           def->type == BPF_MAP_TYPE_CGROUP_ARRAY ||
+           def->type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
+           def->type == BPF_MAP_TYPE_HASH_OF_MAPS ||
+           def->type == BPF_MAP_TYPE_DEVMAP ||
+           def->type == BPF_MAP_TYPE_SOCKMAP ||
+           def->type == BPF_MAP_TYPE_CPUMAP ||
+           def->type == BPF_MAP_TYPE_XSKMAP ||

tools/lib/bpf/libbpf.c static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · line 4634

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+           def->type == BPF_MAP_TYPE_SOCKHASH ||
+           def->type == BPF_MAP_TYPE_QUEUE ||
+           def->type == BPF_MAP_TYPE_STACK ||
+           def->type == BPF_MAP_TYPE_DEVMAP_HASH) {
+               create_attr.btf_fd = 0;
+               create_attr.btf_key_type_id = 0;
+               create_attr.btf_value_type_id = 0;
+               map->btf_key_type_id = 0;
+               map->btf_value_type_id = 0;
+       }

Let's do this as a more succinct switch statement. Consider also slightly rearranging entries to keep "related" map types together:
- SOCKMAP + SOCKHASH
- DEVMAP + DEVMAP_HASH + CPUMAP + XSKMAP

Thanks!

Quoted 14 lines
tools/lib/bpf/libbpf.c +1 -6
+
        if (obj->gen_loader) {
                bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps);
                /* Pretend to have valid FD to pass various fd >= 0 checks.
@@ -4622,21 +4642,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
        } else {
                map->fd = bpf_create_map_xattr(&create_attr);
        }
-       if (map->fd < 0 && (create_attr.btf_key_type_id ||
-                           create_attr.btf_value_type_id)) {
-               char *cp, errmsg[STRERR_BUFSIZE];
-
-               err = -errno;
-               cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));

tools/lib/bpf/libbpf.c static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · line 4645

Quoted 10 lines
tools/lib/bpf/libbpf.c +0 -9
-               pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
-                       map->name, cp, err);
-               create_attr.btf_fd = 0;
-               create_attr.btf_key_type_id = 0;
-               create_attr.btf_value_type_id = 0;
-               map->btf_key_type_id = 0;
-               map->btf_value_type_id = 0;
-               map->fd = bpf_create_map_xattr(&create_attr);
-       }
 

Please don't remove this fallback logic. There are multiple situations where libbpf might need to retry map creation without BTF.

Quoted 5 lines
tools/lib/bpf/libbpf.c +0 -1
        err = map->fd < 0 ? -errno : 0;
 
--

2.25.1

#7 lore raw
Quoted 14 lines · Andrii Nakryiko
Quoted 2 lines · Hengqi Chen

A bunch of BPF maps do not support specifying types for key and value.

s/types/BTF types/, it's a bit confusing otherwise

Quoted 3 lines

This is non-uniform and inconvenient[0]. Currently, libbpf uses a retry logic which removes BTF type IDs when BPF map creation failed. Instead of retrying, this commit recognizes those specialized map and removes

s/map/maps/

Quoted 2 lines

BTF type IDs when creating BPF map.

Quoted 9 lines
Quoted 4 lines

[0] Closes: https://github.com/libbpf/libbpf/issues/355

---

For patch sets consisting of two or more patches, we ask for a cover letter, so for the next revision please provide a cover letter with an overall description of what the series is about.

Hello, Andrii

Sorry for the long delay. Will send a v2 for review.

Quoted 71 lines
Quoted 20 lines
 tools/lib/bpf/libbpf.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
tools/lib/bpf/libbpf.c +9 -0
index 88d8825fc6f6..7068c4d07337 100644
@@ -4613,6 +4613,26 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
                        create_attr.inner_map_fd = map->inner_map_fd;
        }
 
+       if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
+           def->type == BPF_MAP_TYPE_STACK_TRACE ||
+           def->type == BPF_MAP_TYPE_CGROUP_ARRAY ||
+           def->type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
+           def->type == BPF_MAP_TYPE_HASH_OF_MAPS ||
+           def->type == BPF_MAP_TYPE_DEVMAP ||
+           def->type == BPF_MAP_TYPE_SOCKMAP ||
+           def->type == BPF_MAP_TYPE_CPUMAP ||
+           def->type == BPF_MAP_TYPE_XSKMAP ||

tools/lib/bpf/libbpf.c static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · line 4634

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+           def->type == BPF_MAP_TYPE_SOCKHASH ||
+           def->type == BPF_MAP_TYPE_QUEUE ||
+           def->type == BPF_MAP_TYPE_STACK ||
+           def->type == BPF_MAP_TYPE_DEVMAP_HASH) {
+               create_attr.btf_fd = 0;
+               create_attr.btf_key_type_id = 0;
+               create_attr.btf_value_type_id = 0;
+               map->btf_key_type_id = 0;
+               map->btf_value_type_id = 0;
+       }

Let's do this as a more succinct switch statement. Consider also slightly rearranging entries to keep "related" map types together:
- SOCKMAP + SOCKHASH
- DEVMAP + DEVMAP_HASH + CPUMAP + XSKMAP

Thanks!

Quoted 15 lines
tools/lib/bpf/libbpf.c +1 -6
+
        if (obj->gen_loader) {
                bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps);
                /* Pretend to have valid FD to pass various fd >= 0 checks.
@@ -4622,21 +4642,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
        } else {
                map->fd = bpf_create_map_xattr(&create_attr);
        }
-       if (map->fd < 0 && (create_attr.btf_key_type_id ||
-                           create_attr.btf_value_type_id)) {
-               char *cp, errmsg[STRERR_BUFSIZE];
-
-               err = -errno;
-               cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));

tools/lib/bpf/libbpf.c static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · line 4645

Quoted 10 lines
tools/lib/bpf/libbpf.c +0 -9
-               pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
-                       map->name, cp, err);
-               create_attr.btf_fd = 0;
-               create_attr.btf_key_type_id = 0;
-               create_attr.btf_value_type_id = 0;
-               map->btf_key_type_id = 0;
-               map->btf_value_type_id = 0;
-               map->fd = bpf_create_map_xattr(&create_attr);
-       }
 

Please don't remove this fallback logic. There are multiple situations where libbpf might need to retry map creation without BTF.

Quoted 5 lines
tools/lib/bpf/libbpf.c +0 -1
        err = map->fd < 0 ? -errno : 0;
 
--

2.25.1