Add generic and raw BTF parsing APIs to libbpf
Subsystems (5)
cover Add generic and raw BTF parsing APIs to libbpf
It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.
Patches #2 and #3 removes re-implementations of such APIs from bpftool and resolve_btfids tools.
Andrii Nakryiko (3):
libbpf: add btf__parse_raw() and generic btf__parse() APIs tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file tools/resolve_btfids: use libbpf's btf__parse() API
tools/bpf/bpftool/btf.c | 54 +------------ tools/bpf/resolve_btfids/.gitignore | 4 + tools/bpf/resolve_btfids/main.c | 58 +------------- tools/lib/bpf/btf.c | 114 +++++++++++++++++++--------- tools/lib/bpf/btf.h | 5 +- tools/lib/bpf/libbpf.map | 2 + 6 files changed, 89 insertions(+), 148 deletions(-) create mode 100644 tools/bpf/resolve_btfids/.gitignore
Signature · 1 line
2.24.1
PATCH 1/3 libbpf: add btf__parse_raw() and generic btf__parse() APIs
Add public APIs to parse BTF from raw data file (e.g., /sys/kernel/btf/vmlinux), as well as generic btf__parse(), which will try to determine correct format, currently either raw or ELF.
--- tools/lib/bpf/btf.c | 114 ++++++++++++++++++++++++++------------- tools/lib/bpf/btf.h | 5 +- tools/lib/bpf/libbpf.map | 2 + 3 files changed, 83 insertions(+), 38 deletions(-)
tools/lib/bpf/btf.c +78 -36
@@ -562,6 +562,83 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)return btf; } +struct btf *btf__parse_raw (const char *path) +{ +void *data =NULL ; +struct btf *btf; +FILE *f =NULL ; +__u16 magic; +int err =0 ; +long sz; + + f =fopen (path,"rb" ); +if (!f) { + err = -errno; + goto err_out; + } + +/* check BTF magic */ +if (fread (&magic,1 ,sizeof (magic), f) <sizeof (magic)) { + err = -EIO ; + goto err_out; + } +if (magic !=BTF_MAGIC ) { +/* definitely not a raw BTF */ + err = -EPROTO ; + goto err_out; + } + +/* get file size */ +if (fseek (f,0 ,SEEK_END )) { + err = -errno; + goto err_out; + } + sz =ftell (f); +if (sz <0 ) { + err = -errno; + goto err_out; + } +/* rewind to the start */ +if (fseek (f,0 ,SEEK_SET )) { + err = -errno; + goto err_out; + } + +/* pre-alloc memory and read all of BTF data */ + data =malloc (sz); +if (!data) { + err = -ENOMEM ; + goto err_out; + } +if (fread (data,1 , sz, f) < sz) { + err = -EIO ; + goto err_out; + } + +/* finally parse BTF data */ + btf =btf__new (data, sz); + +err_out: +free (data); +if (f) +fclose (f); +return err ?ERR_PTR (err) : btf; +} + +struct btf *btf__parse (const char *path,struct btf_ext **btf_ext) +{ +struct btf *btf; + +if (btf_ext) + *btf_ext =NULL ; + + btf =btf__parse_raw (path); +if (!IS_ERR (btf) ||PTR_ERR (btf) != -EPROTO ) +return btf; + +return btf__parse_elf (path, btf_ext); +} +static int compare_vsi_off (const void *_a,const void *_b) {const struct btf_var_secinfo *a = _a;
@@ -2951,41 +3028,6 @@ static int btf_dedup_remap_types(struct btf_dedup *d)return 0 ; } -static struct btf *btf_load_raw (const char *path) -{ -struct btf *btf; -size_t read_cnt; -struct stat st; -void *data; -FILE *f; - -if (stat (path, &st)) -return ERR_PTR (-errno); - - data =malloc (st.st_size); -if (!data) -return ERR_PTR (-ENOMEM ); - - f =fopen (path,"rb" ); -if (!f) { - btf =ERR_PTR (-errno); - goto cleanup; - } - - read_cnt =fread (data,1 , st.st_size, f); -fclose (f); -if (read_cnt < st.st_size) { - btf =ERR_PTR (-EBADF ); - goto cleanup; - } - - btf =btf__new (data, read_cnt); - -cleanup: -free (data); -return btf; -} -/* * Probe few well-known locations for vmlinux kernel image and try to load BTF * data out of it to use for target BTF.
@@ -3021,7 +3063,7 @@ struct btf *libbpf_find_kernel_btf(void)continue ;if (locations[i].raw_btf) - btf =btf_load_raw (path); + btf =btf__parse_raw (path);else btf =btf__parse_elf (path,NULL );
tools/lib/bpf/btf.h +3 -2
@@ -64,8 +64,9 @@ struct btf_ext_header { LIBBPF_API void btf__free(struct btf *btf); LIBBPF_API struct btf *btf__new(const void *data, __u32 size); -LIBBPF_API struct btf *btf__parse_elf(const char *path, -struct btf_ext **btf_ext); +LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext); +LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext); +LIBBPF_API struct btf *btf__parse_raw(const char *path); LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf); LIBBPF_API int btf__load(struct btf *btf); LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
tools/lib/bpf/libbpf.map +2 -0
@@ -291,5 +291,7 @@ LIBBPF_0.1.0 { bpf_program__is_sk_lookup; bpf_program__set_autoload; bpf_program__set_sk_lookup; + btf__parse; + btf__parse_raw; btf__set_fd; } LIBBPF_0.0.9;
Signature · 1 line
2.24.1
PATCH 2/3 tools/bpftool: use libbpf's btf__parse() API for parsing BTF from file
In reply to Andrii NakryikoUse generic libbpf API to parse BTF data from file, instead of re-implementing it in bpftool.
--- tools/bpf/bpftool/btf.c | 54 +---------------------------------------- 1 file changed, 1 insertion(+), 53 deletions(-)
tools/bpf/bpftool/btf.c +1 -53
@@ -422,54 +422,6 @@ static int dump_btf_c(const struct btf *btf,return err; } -static struct btf *btf__parse_raw (const char *file) -{ -struct btf *btf; -struct stat st; -__u8 *buf; -FILE *f; - -if (stat (file, &st)) -return NULL ; - - f =fopen (file,"rb" ); -if (!f) -return NULL ; - - buf =malloc (st.st_size); -if (!buf) { - btf =ERR_PTR (-ENOMEM ); - goto exit_close; - } - -if ((size_t ) st.st_size !=fread (buf,1 , st.st_size, f)) { - btf =ERR_PTR (-EINVAL ); - goto exit_free; - } - - btf =btf__new (buf, st.st_size); - -exit_free: -free (buf); -exit_close: -fclose (f); -return btf; -} - -static bool is_btf_raw (const char *file) -{ -__u16 magic =0 ; -int fd, nb_read; - - fd =open (file,O_RDONLY ); -if (fd <0 ) -return false; - - nb_read =read (fd, &magic,sizeof (magic)); -close (fd); -return nb_read ==sizeof (magic) && magic ==BTF_MAGIC ; -} -static int do_dump (int argc,char **argv) {struct btf *btf =NULL ;
@@ -547,11 +499,7 @@ static int do_dump(int argc, char **argv) }NEXT_ARG (); }else if (is_prefix (src,"file" )) { -if (is_btf_raw (*argv)) - btf =btf__parse_raw (*argv); -else - btf =btf__parse_elf (*argv,NULL ); - + btf =btf__parse (*argv,NULL );if (IS_ERR (btf)) { err = -PTR_ERR (btf); btf =NULL ;
Signature · 1 line
2.24.1
PATCH 3/3 tools/resolve_btfids: use libbpf's btf__parse() API
In reply to Andrii NakryikoInstead of re-implementing generic BTF parsing logic, use libbpf's API. Also add .gitignore for resolve_btfids's build artifacts.
--- tools/bpf/resolve_btfids/.gitignore | 4 ++ tools/bpf/resolve_btfids/main.c | 58 +---------------------------- 2 files changed, 5 insertions(+), 57 deletions(-) create mode 100644 tools/bpf/resolve_btfids/.gitignore
tools/bpf/resolve_btfids/.gitignore new file +4 -0
@@ -0,0 +1,4 @@ +/FEATURE-DUMP.libbpf +/bpf_helper_defs.h +/fixdep +/resolve_btfids
tools/bpf/resolve_btfids/main.c +1 -57
70 lines not on this page · raw message
Signature · 1 line
2.24.1
Quoted 15 lines · Andrii Nakryiko
It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.
Patches #2 and #3 removes re-implementations of such APIs from bpftool and
resolve_btfidstools.Andrii Nakryiko (3):
libbpf: addbtf__parse_raw()and genericbtf__parse()APIstools/bpftool: use libbpf'sbtf__parse()API for parsing BTF from filetools/resolve_btfids: use libbpf'sbtf__parse()API
Quoted 9 lines
tools/bpf/bpftool/btf.c | 54 +------------ tools/bpf/resolve_btfids/.gitignore | 4 + tools/bpf/resolve_btfids/main.c | 58 +------------- tools/lib/bpf/btf.c | 114 +++++++++++++++++++--------- tools/lib/bpf/btf.h | 5 +- tools/lib/bpf/libbpf.map | 2 + 6 files changed, 89 insertions(+), 148 deletions(-)create mode 100644
tools/bpf/resolve_btfids/.gitignore
Applied, thanks!
Em Sat, Aug 01, 2020 at 06:32:16PM -0700, Andrii Nakryiko escreveu:
Quoted 7 lines
It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.
Quoted 7 lines
Patches #2 and #3 removes re-implementations of such APIs from bpftool and
resolve_btfidstools.Andrii Nakryiko (3):
libbpf: addbtf__parse_raw()and genericbtf__parse()APIstools/bpftool: use libbpf'sbtf__parse()API for parsing BTF from filetools/resolve_btfids: use libbpf'sbtf__parse()API
I haven't checked which of the patches, or some in other series caused this on Clear Linux:
21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1
gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41
btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]
625 | return err ? ERR_PTR(err) : btf;
| ~~~~~~~~~~~~~~~~~~~^~~~~This is what I have:
[acme@quaco perf]$ git log -10 --oneline tools/lib/bpf94a1fedd63ed libbpf: Add btf__parse_raw() and generic btf__parse() APIs 2e49527e5248 libbpf: Add bpf_link detach APIs1acf8f90ea7e libbpf: Fix register in PT_REGS MIPS macros 50450fc716c1 libbpf: Make destructors more robust by handling ERR_PTR(err) cases dc8698cac7aa libbpf: Add support for BPF XDP linkd4b4dd6ce770 libbpf: Print hint when PERF_EVENT_IOC_SET_BPF returns -EPROTO cd31039a7347 tools/libbpf: Add support for bpf map element iterator da7a35062bcc libbpf bpf_helpers: Use __builtin_offsetof for offsetof 499dd29d90bb libbpf: Add support for SK_LOOKUP program type 4be556cf5aef libbpf: Add SEC name for xdp programs attached to CPUMAP [acme@quaco perf]$
Quoted 12 lines
tools/bpf/bpftool/btf.c | 54 +------------ tools/bpf/resolve_btfids/.gitignore | 4 + tools/bpf/resolve_btfids/main.c | 58 +------------- tools/lib/bpf/btf.c | 114 +++++++++++++++++++--------- tools/lib/bpf/btf.h | 5 +- tools/lib/bpf/libbpf.map | 2 + 6 files changed, 89 insertions(+), 148 deletions(-) create mode 100644 tools/bpf/resolve_btfids/.gitignoreSignature · 1 line
2.24.1
Signature · 1 line
- Arnaldo
Quoted 23 lines · Arnaldo Carvalho de Melo
Em Sat, Aug 01, 2020 at 06:32:16PM -0700, Andrii Nakryiko escreveu:
Quoted 7 lines
It's pretty common for applications to want to parse raw (binary) BTF data from file, as opposed to parsing it from ELF sections. It's also pretty common for tools to not care whether given file is ELF or raw BTF format. This patch series exposes internal raw BTF parsing API and adds generic variant of BTF parsing, which will efficiently determine the format of a given fail and will parse BTF appropriately.
Quoted 7 lines
Patches #2 and #3 removes re-implementations of such APIs from bpftool and
resolve_btfidstools.Andrii Nakryiko (3):
libbpf: addbtf__parse_raw()and genericbtf__parse()APIstools/bpftool: use libbpf'sbtf__parse()API for parsing BTF from filetools/resolve_btfids: use libbpf'sbtf__parse()APII haven't checked which of the patches, or some in other series caused this on Clear Linux:
21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1
Quoted 7 lines
gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41
btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]625 | return err ? ERR_PTR(err) : btf; | ~~~~~~~~~~~~~~~~~~~^~~~~
Yeah, fixed in https://patchwork.ozlabs.org/project/netdev/patch/[email protected]/
Quoted 31 lines
This is what I have:
[acme@quaco perf]$ git log -10 --oneline
tools/lib/bpf94a1fedd63edlibbpf: Addbtf__parse_raw()and genericbtf__parse()APIs2e49527e5248libbpf: Addbpf_linkdetach APIs1acf8f90ea7elibbpf: Fix register inPT_REGSMIPS macros50450fc716c1libbpf: Make destructors more robust by handlingERR_PTR(err)casesdc8698cac7aalibbpf: Add support for BPF XDP linkd4b4dd6ce770libbpf: Print hint whenPERF_EVENT_IOC_SET_BPFreturns-EPROTOcd31039a7347tools/libbpf: Add support for bpf map element iteratorda7a35062bcclibbpfbpf_helpers: Use__builtin_offsetoffor offsetof499dd29d90bblibbpf: Add support forSK_LOOKUPprogram type4be556cf5aeflibbpf: Add SEC name for xdp programs attached to CPUMAP [acme@quaco perf]$Quoted 9 lines
tools/bpf/bpftool/btf.c | 54 +------------ tools/bpf/resolve_btfids/.gitignore | 4 + tools/bpf/resolve_btfids/main.c | 58 +------------- tools/lib/bpf/btf.c | 114 +++++++++++++++++++--------- tools/lib/bpf/btf.h | 5 +- tools/lib/bpf/libbpf.map | 2 + 6 files changed, 89 insertions(+), 148 deletions(-) create mode 100644 tools/bpf/resolve_btfids/.gitignoreQuoted 3 lines
--
2.24.1--
- Arnaldo
Quoted 8 lines · Andrii Nakryiko
Quoted 8 lines
I haven't checked which of the patches, or some in other series caused this on Clear Linux:
21 clearlinux:latest : FAIL gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41, clang ver
sion 10.0.1gcc (Clear Linux OS for Intel Architecture) 10.2.1 20200723 releases/gcc-10.2.0-3-g677b80db41
Quoted 8 lines
Quoted 5 lines
btf.c: In function 'btf__parse_raw':
btf.c:625:28: error: 'btf' may be used uninitialized in this function [-Werror=maybe-uninitialized]625 | return err ? ERR_PTR(err) : btf; | ~~~~~~~~~~~~~~~~~~~^~~~~Yeah, fixed in https://patchwork.ozlabs.org/project/netdev/patch/[email protected]/
Thanks for headsup.
The fix will be pushed to bpf tree when net tree gets ffwd-ed to Linus. Hopefully today.