Skip to contentSkip to conversation

Add generic and raw BTF parsing APIs to libbpf

PATCH Applied Andrii Nakryiko 5 participants 8 messages Aug 2, 2020 – Aug 6, 2020

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

cover Add generic and raw BTF parsing APIs to libbpf

It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.

Patches #2 and #3 removes re-implementations of such APIs from bpftool and resolve_btfids tools.

Andrii Nakryiko (3):
libbpf: add btf__parse_raw() and generic btf__parse() APIs tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file tools/resolve_btfids: use libbpf's btf__parse() API

 tools/bpf/bpftool/btf.c             |  54 +------------
 tools/bpf/resolve_btfids/.gitignore |   4 +
 tools/bpf/resolve_btfids/main.c     |  58 +-------------
 tools/lib/bpf/btf.c                 | 114 +++++++++++++++++++---------
 tools/lib/bpf/btf.h                 |   5 +-
 tools/lib/bpf/libbpf.map            |   2 +
 6 files changed, 89 insertions(+), 148 deletions(-)
 create mode 100644 tools/bpf/resolve_btfids/.gitignore
Signature · 1 line
2.24.1
#1 lore raw

PATCH 1/3 libbpf: add btf__parse_raw() and generic btf__parse() APIs

Add public APIs to parse BTF from raw data file (e.g., /sys/kernel/btf/vmlinux), as well as generic btf__parse(), which will try to determine correct format, currently either raw or ELF.

---
 tools/lib/bpf/btf.c      | 114 ++++++++++++++++++++++++++-------------
 tools/lib/bpf/btf.h      |   5 +-
 tools/lib/bpf/libbpf.map |   2 +
 3 files changed, 83 insertions(+), 38 deletions(-)
tools/lib/bpf/btf.c +78 -36
index ded5b29965f9..856b09a04563 100644
@@ -562,6 +562,83 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
 	return btf;
 }
 
+struct btf *btf__parse_raw(const char *path)
+{
+	void *data = NULL;
+	struct btf *btf;
+	FILE *f = NULL;
+	__u16 magic;
+	int err = 0;
+	long sz;
+
+	f = fopen(path, "rb");
+	if (!f) {
+		err = -errno;
+		goto err_out;
+	}
+
+	/* check BTF magic */
+	if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
+		err = -EIO;
+		goto err_out;
+	}
+	if (magic != BTF_MAGIC) {
+		/* definitely not a raw BTF */
+		err = -EPROTO;
+		goto err_out;
+	}
+
+	/* get file size */
+	if (fseek(f, 0, SEEK_END)) {
+		err = -errno;
+		goto err_out;
+	}
+	sz = ftell(f);
+	if (sz < 0) {
+		err = -errno;
+		goto err_out;
+	}
+	/* rewind to the start */
+	if (fseek(f, 0, SEEK_SET)) {
+		err = -errno;
+		goto err_out;
+	}
+
+	/* pre-alloc memory and read all of BTF data */
+	data = malloc(sz);
+	if (!data) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+	if (fread(data, 1, sz, f) < sz) {
+		err = -EIO;
+		goto err_out;
+	}
+
+	/* finally parse BTF data */
+	btf = btf__new(data, sz);
+
+err_out:
+	free(data);
+	if (f)
+		fclose(f);
+	return err ? ERR_PTR(err) : btf;
+}
+
+struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
+{
+	struct btf *btf;
+
+	if (btf_ext)
+		*btf_ext = NULL;
+
+	btf = btf__parse_raw(path);
+	if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO)
+		return btf;
+
+	return btf__parse_elf(path, btf_ext);
+}
+
 static int compare_vsi_off(const void *_a, const void *_b)
 {
 	const struct btf_var_secinfo *a = _a;
@@ -2951,41 +3028,6 @@ static int btf_dedup_remap_types(struct btf_dedup *d)
 	return 0;
 }
 
-static struct btf *btf_load_raw(const char *path)
-{
-	struct btf *btf;
-	size_t read_cnt;
-	struct stat st;
-	void *data;
-	FILE *f;
-
-	if (stat(path, &st))
-		return ERR_PTR(-errno);
-
-	data = malloc(st.st_size);
-	if (!data)
-		return ERR_PTR(-ENOMEM);
-
-	f = fopen(path, "rb");
-	if (!f) {
-		btf = ERR_PTR(-errno);
-		goto cleanup;
-	}
-
-	read_cnt = fread(data, 1, st.st_size, f);
-	fclose(f);
-	if (read_cnt < st.st_size) {
-		btf = ERR_PTR(-EBADF);
-		goto cleanup;
-	}
-
-	btf = btf__new(data, read_cnt);
-
-cleanup:
-	free(data);
-	return btf;
-}
-
 /*
  * Probe few well-known locations for vmlinux kernel image and try to load BTF
  * data out of it to use for target BTF.
@@ -3021,7 +3063,7 @@ struct btf *libbpf_find_kernel_btf(void)
 			continue;
 
 		if (locations[i].raw_btf)
-			btf = btf_load_raw(path);
+			btf = btf__parse_raw(path);
 		else
 			btf = btf__parse_elf(path, NULL);
 
tools/lib/bpf/btf.h +3 -2
index 491c7b41ffdc..f4a1a1d2b9a3 100644
@@ -64,8 +64,9 @@ struct btf_ext_header {
 
 LIBBPF_API void btf__free(struct btf *btf);
 LIBBPF_API struct btf *btf__new(const void *data, __u32 size);
-LIBBPF_API struct btf *btf__parse_elf(const char *path,
-				      struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_raw(const char *path);
 LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
 LIBBPF_API int btf__load(struct btf *btf);
 LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
tools/lib/bpf/libbpf.map +2 -0
index ca49a6a7e5b2..edf6a38807ea 100644
@@ -291,5 +291,7 @@ LIBBPF_0.1.0 {
 		bpf_program__is_sk_lookup;
 		bpf_program__set_autoload;
 		bpf_program__set_sk_lookup;
+		btf__parse;
+		btf__parse_raw;
 		btf__set_fd;
 } LIBBPF_0.0.9;
Signature · 1 line
2.24.1
#2 lore raw

PATCH 2/3 tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file

In reply to Andrii Nakryiko

Use generic libbpf API to parse BTF data from file, instead of re-implementing it in bpftool.

---
 tools/bpf/bpftool/btf.c | 54 +----------------------------------------
 1 file changed, 1 insertion(+), 53 deletions(-)
tools/bpf/bpftool/btf.c +1 -53
index fc9bc7a23db6..42d1df2c1939 100644
@@ -422,54 +422,6 @@ static int dump_btf_c(const struct btf *btf,
 	return err;
 }
 
-static struct btf *btf__parse_raw(const char *file)
-{
-	struct btf *btf;
-	struct stat st;
-	__u8 *buf;
-	FILE *f;
-
-	if (stat(file, &st))
-		return NULL;
-
-	f = fopen(file, "rb");
-	if (!f)
-		return NULL;
-
-	buf = malloc(st.st_size);
-	if (!buf) {
-		btf = ERR_PTR(-ENOMEM);
-		goto exit_close;
-	}
-
-	if ((size_t) st.st_size != fread(buf, 1, st.st_size, f)) {
-		btf = ERR_PTR(-EINVAL);
-		goto exit_free;
-	}
-
-	btf = btf__new(buf, st.st_size);
-
-exit_free:
-	free(buf);
-exit_close:
-	fclose(f);
-	return btf;
-}
-
-static bool is_btf_raw(const char *file)
-{
-	__u16 magic = 0;
-	int fd, nb_read;
-
-	fd = open(file, O_RDONLY);
-	if (fd < 0)
-		return false;
-
-	nb_read = read(fd, &magic, sizeof(magic));
-	close(fd);
-	return nb_read == sizeof(magic) && magic == BTF_MAGIC;
-}
-
 static int do_dump(int argc, char **argv)
 {
 	struct btf *btf = NULL;
@@ -547,11 +499,7 @@ static int do_dump(int argc, char **argv)
 		}
 		NEXT_ARG();
 	} else if (is_prefix(src, "file")) {
-		if (is_btf_raw(*argv))
-			btf = btf__parse_raw(*argv);
-		else
-			btf = btf__parse_elf(*argv, NULL);
-
+		btf = btf__parse(*argv, NULL);
 		if (IS_ERR(btf)) {
 			err = -PTR_ERR(btf);
 			btf = NULL;
Signature · 1 line
2.24.1
#3 lore raw

PATCH 3/3 tools/resolve_btfids: use libbpf's btf__parse() API

In reply to Andrii Nakryiko

Instead of re-implementing generic BTF parsing logic, use libbpf's API. Also add .gitignore for resolve_btfids's build artifacts.

---
 tools/bpf/resolve_btfids/.gitignore |  4 ++
 tools/bpf/resolve_btfids/main.c     | 58 +----------------------------
 2 files changed, 5 insertions(+), 57 deletions(-)
 create mode 100644 tools/bpf/resolve_btfids/.gitignore
tools/bpf/resolve_btfids/.gitignore new file +4 -0
new file mode 100644
index 000000000000..a026df7dc280
@@ -0,0 +1,4 @@
+/FEATURE-DUMP.libbpf
+/bpf_helper_defs.h
+/fixdep
+/resolve_btfids
tools/bpf/resolve_btfids/main.c +1 -57

70 lines not on this page · raw message

Signature · 1 line
2.24.1
maintainer Applied
#4 lore raw
In reply to Andrii Nakryiko
Quoted 15 lines · Andrii Nakryiko

It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.

Patches #2 and #3 removes re-implementations of such APIs from bpftool and resolve_btfids tools.

Andrii Nakryiko (3):
libbpf: add btf__parse_raw() and generic btf__parse() APIs tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file tools/resolve_btfids: use libbpf's btf__parse() API

Quoted 9 lines
  tools/bpf/bpftool/btf.c             |  54 +------------
  tools/bpf/resolve_btfids/.gitignore |   4 +
  tools/bpf/resolve_btfids/main.c     |  58 +-------------
  tools/lib/bpf/btf.c                 | 114 +++++++++++++++++++---------
  tools/lib/bpf/btf.h                 |   5 +-
  tools/lib/bpf/libbpf.map            |   2 +
  6 files changed, 89 insertions(+), 148 deletions(-)

create mode 100644 tools/bpf/resolve_btfids/.gitignore

Applied, thanks!

#5 lore raw
In reply to Andrii Nakryiko

Em Sat, Aug 01, 2020 at 06:32:16PM -0700, Andrii Nakryiko escreveu:

Quoted 7 lines

It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.

Quoted 7 lines

Patches #2 and #3 removes re-implementations of such APIs from bpftool and resolve_btfids tools.

Andrii Nakryiko (3):
libbpf: add btf__parse_raw() and generic btf__parse() APIs tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file tools/resolve_btfids: use libbpf's btf__parse() API

I haven't checked which of the patches, or some in other series caused this on Clear Linux:

21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1

gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41

btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]

    625 |  return err ? ERR_PTR(err) : btf;
        |         ~~~~~~~~~~~~~~~~~~~^~~~~

This is what I have:

[acme@quaco perf]$ git log -10 --oneline tools/lib/bpf
94a1fedd63ed libbpf: Add btf__parse_raw() and generic btf__parse() APIs 2e49527e5248 libbpf: Add bpf_link detach APIs
1acf8f90ea7e libbpf: Fix register in PT_REGS MIPS macros 50450fc716c1 libbpf: Make destructors more robust by handling ERR_PTR(err) cases dc8698cac7aa libbpf: Add support for BPF XDP link
d4b4dd6ce770 libbpf: Print hint when PERF_EVENT_IOC_SET_BPF returns -EPROTO cd31039a7347 tools/libbpf: Add support for bpf map element iterator da7a35062bcc libbpf bpf_helpers: Use __builtin_offsetof for offsetof 499dd29d90bb libbpf: Add support for SK_LOOKUP program type 4be556cf5aef libbpf: Add SEC name for xdp programs attached to CPUMAP [acme@quaco perf]$

Quoted 12 lines
 tools/bpf/bpftool/btf.c             |  54 +------------
 tools/bpf/resolve_btfids/.gitignore |   4 +
 tools/bpf/resolve_btfids/main.c     |  58 +-------------
 tools/lib/bpf/btf.c                 | 114 +++++++++++++++++++---------
 tools/lib/bpf/btf.h                 |   5 +-
 tools/lib/bpf/libbpf.map            |   2 +
 6 files changed, 89 insertions(+), 148 deletions(-)
 create mode 100644 tools/bpf/resolve_btfids/.gitignore
Signature · 1 line
2.24.1
Signature · 1 line
- Arnaldo
#6 lore raw
Quoted 23 lines · Arnaldo Carvalho de Melo

Em Sat, Aug 01, 2020 at 06:32:16PM -0700, Andrii Nakryiko escreveu:

Quoted 7 lines

It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.

Quoted 7 lines

Patches #2 and #3 removes re-implementations of such APIs from bpftool and resolve_btfids tools.

Andrii Nakryiko (3):
libbpf: add btf__parse_raw() and generic btf__parse() APIs tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file tools/resolve_btfids: use libbpf's btf__parse() API

I haven't checked which of the patches, or some in other series caused this on Clear Linux:

21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1

Quoted 7 lines

gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41

btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]

    625 |  return err ? ERR_PTR(err) : btf;
        |         ~~~~~~~~~~~~~~~~~~~^~~~~

Yeah, fixed in https://patchwork.ozlabs.org/project/netdev/patch/[email protected]/

Quoted 31 lines

This is what I have:

[acme@quaco perf]$ git log -10 --oneline tools/lib/bpf
94a1fedd63ed libbpf: Add btf__parse_raw() and generic btf__parse() APIs 2e49527e5248 libbpf: Add bpf_link detach APIs
1acf8f90ea7e libbpf: Fix register in PT_REGS MIPS macros 50450fc716c1 libbpf: Make destructors more robust by handling ERR_PTR(err) cases dc8698cac7aa libbpf: Add support for BPF XDP link
d4b4dd6ce770 libbpf: Print hint when PERF_EVENT_IOC_SET_BPF returns -EPROTO cd31039a7347 tools/libbpf: Add support for bpf map element iterator da7a35062bcc libbpf bpf_helpers: Use __builtin_offsetof for offsetof 499dd29d90bb libbpf: Add support for SK_LOOKUP program type 4be556cf5aef libbpf: Add SEC name for xdp programs attached to CPUMAP [acme@quaco perf]$

Quoted 9 lines
 tools/bpf/bpftool/btf.c             |  54 +------------
 tools/bpf/resolve_btfids/.gitignore |   4 +
 tools/bpf/resolve_btfids/main.c     |  58 +-------------
 tools/lib/bpf/btf.c                 | 114 +++++++++++++++++++---------
 tools/lib/bpf/btf.h                 |   5 +-
 tools/lib/bpf/libbpf.map            |   2 +
 6 files changed, 89 insertions(+), 148 deletions(-)
 create mode 100644 tools/bpf/resolve_btfids/.gitignore
Quoted 3 lines

--
2.24.1

--

- Arnaldo

#7 lore raw
Quoted 8 lines · Andrii Nakryiko
Quoted 8 lines

I haven't checked which of the patches, or some in other series caused this on Clear Linux:

21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1

gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41

Quoted 8 lines
Quoted 5 lines

btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]

    625 |  return err ? ERR_PTR(err) : btf;
        |         ~~~~~~~~~~~~~~~~~~~^~~~~

Yeah, fixed in https://patchwork.ozlabs.org/project/netdev/patch/[email protected]/

Thanks for headsup.
The fix will be pushed to bpf tree when net tree gets ffwd-ed to Linus. Hopefully today.