bpf: btf: json print btf info with bpftool map dump
Subsystems (5)
cover bpf: btf: json print btf info with bpftool map dump
Hi,
These patches augment the bpftool's map dump command with BTF info. In particular, when user runs bpftool map dump [-j|-p] id <map-id>, they will see map data formatted and tagged based upon BTF information associated with that map. Here is what each patch does:
Patch 1 exports BTF functions inside libbpf, to be used by patch 2.
Patch 2 adds btf_dumper which uses type info exported in patch 1 along with json_writer to json print or pretty json print map values alongside btf debug info.
Patch 3 uses btf_dumper to json or pretty print map values when -j or -p flag is specified to btf map dump.
Further details are included in patch descriptions.
Thanks,
Okash
PATCH 1/3 bpf: btf: export btf types and name by offset from lib
This patch introduces btf__resolve_type() function and exports two existing functions from libbpf. btf__resolve_type follows modifier types like const and typedef until it hits a type which actually takes up memory, and then returns it. This function follows similar pattern to btf__resolve_size but instead of computing size, it just returns the type.
These functions will be used in the followig patch which parses information inside array of struct btf_type *. btf_name_by_offset is used for printing variable names.
--- tools/lib/bpf/btf.c | 65 ++++++++++++++++++++++++++++++++++++---------------- tools/lib/bpf/btf.h | 3 ++ 2 files changed, 48 insertions(+), 20 deletions(-)
tools/lib/bpf/btf.c +45 -20
@@ -17,6 +17,11 @@#define BTF_MAX_NR_TYPES 65535 +#define IS_MODIFIER (k) (((k) == BTF_KIND_TYPEDEF) || \ + ((k) == BTF_KIND_VOLATILE) || \ + ((k) == BTF_KIND_CONST) || \ + ((k) == BTF_KIND_RESTRICT)) +static struct btf_type btf_void;struct btf {
@@ -33,14 +38,6 @@ struct btf {int fd; }; -static const char *btf_name_by_offset (const struct btf *btf,uint32_t offset) -{ -if (offset < btf->hdr->str_len) -return &btf->strings[offset]; -else -return NULL ; -} -static int btf_add_type (struct btf *btf,struct btf_type *t) {if (btf->types_size - btf->nr_types <2 ) {
@@ -190,15 +187,6 @@ static int btf_parse_type_sec(struct btfreturn 0 ; } -static const struct btf_type *btf_type_by_id (const struct btf *btf, -uint32_t type_id) -{ -if (type_id > btf->nr_types) -return NULL ; - -return btf->types[type_id]; -} -static bool btf_type_is_void (const struct btf_type *t) {return t == &btf_void ||BTF_INFO_KIND (t->info) ==BTF_KIND_FWD ;
@@ -234,7 +222,7 @@ int64_t btf__resolve_size(const struct bint64_t size =-1 ;int i; - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id);for (i =0 ; i <MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null (t); i++) { size =btf_type_size (t);
@@ -259,7 +247,7 @@ int64_t btf__resolve_size(const struct breturn -EINVAL ; } - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id); }if (size <0 )
@@ -271,6 +259,26 @@ int64_t btf__resolve_size(const struct breturn nelems * size; } +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id) +{ +const struct btf_type *t; +int depth =0 ; + + t =btf__type_by_id (btf, type_id); +while (depth <MAX_RESOLVE_DEPTH && + !btf_type_is_void_or_null (t) && +IS_MODIFIER (BTF_INFO_KIND (t->info))) { + type_id = t->type; + t =btf__type_by_id (btf, type_id); + depth++; + } + +if (depth ==MAX_RESOLVE_DEPTH ||btf_type_is_void_or_null (t)) +return -EINVAL ; + +return type_id; +} +int32_t btf__find_by_name (const struct btf *btf,const char *type_name) {uint32_t i;
@@ -280,7 +288,7 @@ int32_t btf__find_by_name(const struct bfor (i =1 ; i <= btf->nr_types; i++) {const struct btf_type *t = btf->types[i]; -const char *name =btf_name_by_offset (btf, t->name_off); +const char *name =btf__name_by_offset (btf, t->name_off);if (name && !strcmp (type_name, name))return i;
@@ -371,3 +379,20 @@ int btf__fd(const struct btf *btf) {return btf->fd; } + +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset) +{ +if (offset < btf->hdr->str_len) +return &btf->strings[offset]; +else +return NULL ; +} + +const struct btf_type *btf__type_by_id (const struct btf *btf, +uint32_t type_id) +{ +if (type_id > btf->nr_types) +return NULL ; + +return btf->types[type_id]; +}
tools/lib/bpf/btf.h +3 -0
@@ -17,6 +17,9 @@ void btf__free(struct btf *btf);struct btf *btf__new (uint8_t *data,uint32_t size,btf_print_fn_t err_log);int32_t btf__find_by_name (const struct btf *btf,const char *type_name);int64_t btf__resolve_size (const struct btf *btf,uint32_t type_id); +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id);int btf__fd (const struct btf *btf); +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset); +const struct btf_type *btf__type_by_id (const struct btf *btf,uint32_t type_id);#endif
Re: [PATCH bpf-next 1/3] bpf: btf: export btf types and name by offset from lib
Quoted 54 lines · Okash Khawaja
This patch introduces
btf__resolve_type()function and exports two existing functions from libbpf.btf__resolve_typefollows modifier types like const and typedef until it hits a type which actually takes up memory, and then returns it. This function follows similar pattern tobtf__resolve_sizebut instead of computing size, it just returns the type.These functions will be used in the followig patch which parses information inside array of
struct btf_type *.btf_name_by_offsetis used for printing variable names.Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/lib/bpf/btf.c| 65 ++++++++++++++++++++++++++++++++++++----------------tools/lib/bpf/btf.h| 3 ++2 files changed, 48 insertions(+), 20 deletions(-)tools/lib/bpf/btf.c +5
-8@@ -17,6 +17,11 @@#define BTF_MAX_NR_TYPES 65535 +#define IS_MODIFIER (k) (((k) == BTF_KIND_TYPEDEF) || \ + ((k) == BTF_KIND_VOLATILE) || \ + ((k) == BTF_KIND_CONST) || \ + ((k) == BTF_KIND_RESTRICT)) +static struct btf_type btf_void;struct btf {@@ -33,14 +38,6 @@ struct btf {int fd; }; -static const char *btf_name_by_offset (const struct btf *btf,uint32_t offset) -{ -if (offset < btf->hdr->str_len) -return &btf->strings[offset]; -else -return NULL ; -} -static int btf_add_type (struct btf *btf,struct btf_type *t) {if (btf->types_size - btf->nr_types <2 ) {@@ -190,15 +187,6 @@ static int btf_parse_type_sec(struct btfreturn 0 ;
tools/lib/bpf/btf.c static int btf_parse_type_sec(struct btf · line 200
Quoted 10 lines
tools/lib/bpf/btf.c +0
-8} -static const struct btf_type *btf_type_by_id (const struct btf *btf, -uint32_t type_id) -{ -if (type_id > btf->nr_types) -return NULL ; - -return btf->types[type_id]; -}
nit: Why do we need to move these two functions to later of the file?
Other than this,
Quoted 92 lines
tools/lib/bpf/btf.c +40
-4-static bool btf_type_is_void (const struct btf_type *t) {return t == &btf_void ||BTF_INFO_KIND (t->info) ==BTF_KIND_FWD ;@@ -234,7 +222,7 @@ int64_t btf__resolve_size(const struct bint64_t size =-1 ;int i; - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id);for (i =0 ; i <MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null (t); i++) { size =btf_type_size (t);@@ -259,7 +247,7 @@ int64_t btf__resolve_size(const struct breturn -EINVAL ; } - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id); }if (size <0 )@@ -271,6 +259,26 @@ int64_t btf__resolve_size(const struct breturn nelems * size; } +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id) +{ +const struct btf_type *t; +int depth =0 ; + + t =btf__type_by_id (btf, type_id); +while (depth <MAX_RESOLVE_DEPTH && + !btf_type_is_void_or_null (t) && +IS_MODIFIER (BTF_INFO_KIND (t->info))) { + type_id = t->type; + t =btf__type_by_id (btf, type_id); + depth++; + } + +if (depth ==MAX_RESOLVE_DEPTH ||btf_type_is_void_or_null (t)) +return -EINVAL ; + +return type_id; +} +int32_t btf__find_by_name (const struct btf *btf,const char *type_name) {uint32_t i;@@ -280,7 +288,7 @@ int32_t btf__find_by_name(const struct bfor (i =1 ; i <= btf->nr_types; i++) {const struct btf_type *t = btf->types[i]; -const char *name =btf_name_by_offset (btf, t->name_off); +const char *name =btf__name_by_offset (btf, t->name_off);if (name && !strcmp (type_name, name))return i;@@ -371,3 +379,20 @@ int btf__fd(const struct btf *btf) {return btf->fd; } + +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset) +{ +if (offset < btf->hdr->str_len) +return &btf->strings[offset]; +else +return NULL ; +} + +const struct btf_type *btf__type_by_id (const struct btf *btf, +uint32_t type_id) +{ +if (type_id > btf->nr_types) +return NULL ; + +return btf->types[type_id]; +}tools/lib/bpf/btf.h +3
-0@@ -17,6 +17,9 @@ void btf__free(struct btf *btf);struct btf *btf__new (uint8_t *data,uint32_t size,btf_print_fn_t err_log);int32_t btf__find_by_name (const struct btf *btf,const char *type_name);int64_t btf__resolve_size (const struct btf *btf,uint32_t type_id); +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id);int btf__fd (const struct btf *btf); +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset); +const struct btf_type *btf__type_by_id (const struct btf *btf,uint32_t type_id);#endif
Re: [PATCH bpf-next 1/3] bpf: btf: export btf types and name by offset from lib
Quoted 56 lines · Song Liu
Quoted 56 lines · Okash Khawaja
This patch introduces
btf__resolve_type()function and exports two existing functions from libbpf.btf__resolve_typefollows modifier types like const and typedef until it hits a type which actually takes up memory, and then returns it. This function follows similar pattern tobtf__resolve_sizebut instead of computing size, it just returns the type.These functions will be used in the followig patch which parses information inside array of
struct btf_type *.btf_name_by_offsetis used for printing variable names.Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/lib/bpf/btf.c| 65 ++++++++++++++++++++++++++++++++++++----------------tools/lib/bpf/btf.h| 3 ++2 files changed, 48 insertions(+), 20 deletions(-)tools/lib/bpf/btf.c +5
-8@@ -17,6 +17,11 @@#define BTF_MAX_NR_TYPES 65535 +#define IS_MODIFIER (k) (((k) == BTF_KIND_TYPEDEF) || \ + ((k) == BTF_KIND_VOLATILE) || \ + ((k) == BTF_KIND_CONST) || \ + ((k) == BTF_KIND_RESTRICT)) +static struct btf_type btf_void;struct btf {@@ -33,14 +38,6 @@ struct btf {int fd; }; -static const char *btf_name_by_offset (const struct btf *btf,uint32_t offset) -{ -if (offset < btf->hdr->str_len) -return &btf->strings[offset]; -else -return NULL ; -} -static int btf_add_type (struct btf *btf,struct btf_type *t) {if (btf->types_size - btf->nr_types <2 ) {@@ -190,15 +187,6 @@ static int btf_parse_type_sec(struct btfreturn 0 ; }
Quoted 10 lines
tools/lib/bpf/btf.c static int btf_parse_type_sec(struct btf · line 200
Quoted 8 lines
tools/lib/bpf/btf.c +0
-8-static const struct btf_type *btf_type_by_id (const struct btf *btf, -uint32_t type_id) -{ -if (type_id > btf->nr_types) -return NULL ; - -return btf->types[type_id]; -}nit: Why do we need to move these two functions to later of the file?
No real reason. It looked like this file was following a convention of keeping statics together. I'll put them back in place if no one objects.
Quoted 4 lines
Other than this,
Acked-by Song Liu <[email protected]>
Thanks
Quoted 94 lines
Quoted 93 lines
tools/lib/bpf/btf.c +40
-4-static bool btf_type_is_void (const struct btf_type *t) {return t == &btf_void ||BTF_INFO_KIND (t->info) ==BTF_KIND_FWD ;@@ -234,7 +222,7 @@ int64_t btf__resolve_size(const struct bint64_t size =-1 ;int i; - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id);for (i =0 ; i <MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null (t); i++) { size =btf_type_size (t);@@ -259,7 +247,7 @@ int64_t btf__resolve_size(const struct breturn -EINVAL ; } - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id); }if (size <0 )@@ -271,6 +259,26 @@ int64_t btf__resolve_size(const struct breturn nelems * size; } +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id) +{ +const struct btf_type *t; +int depth =0 ; + + t =btf__type_by_id (btf, type_id); +while (depth <MAX_RESOLVE_DEPTH && + !btf_type_is_void_or_null (t) && +IS_MODIFIER (BTF_INFO_KIND (t->info))) { + type_id = t->type; + t =btf__type_by_id (btf, type_id); + depth++; + } + +if (depth ==MAX_RESOLVE_DEPTH ||btf_type_is_void_or_null (t)) +return -EINVAL ; + +return type_id; +} +int32_t btf__find_by_name (const struct btf *btf,const char *type_name) {uint32_t i;@@ -280,7 +288,7 @@ int32_t btf__find_by_name(const struct bfor (i =1 ; i <= btf->nr_types; i++) {const struct btf_type *t = btf->types[i]; -const char *name =btf_name_by_offset (btf, t->name_off); +const char *name =btf__name_by_offset (btf, t->name_off);if (name && !strcmp (type_name, name))return i;@@ -371,3 +379,20 @@ int btf__fd(const struct btf *btf) {return btf->fd; } + +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset) +{ +if (offset < btf->hdr->str_len) +return &btf->strings[offset]; +else +return NULL ; +} + +const struct btf_type *btf__type_by_id (const struct btf *btf, +uint32_t type_id) +{ +if (type_id > btf->nr_types) +return NULL ; + +return btf->types[type_id]; +}tools/lib/bpf/btf.h +3
-0@@ -17,6 +17,9 @@ void btf__free(struct btf *btf);struct btf *btf__new (uint8_t *data,uint32_t size,btf_print_fn_t err_log);int32_t btf__find_by_name (const struct btf *btf,const char *type_name);int64_t btf__resolve_size (const struct btf *btf,uint32_t type_id); +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id);int btf__fd (const struct btf *btf); +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset); +const struct btf_type *btf__type_by_id (const struct btf *btf,uint32_t type_id);#endif
Re: [PATCH bpf-next 1/3] bpf: btf: export btf types and name by offset from lib
Quoted 71 lines · Okash Khawaja
Quoted 56 lines · Song Liu
Quoted 56 lines · Okash Khawaja
This patch introduces
btf__resolve_type()function and exports two existing functions from libbpf.btf__resolve_typefollows modifier types like const and typedef until it hits a type which actually takes up memory, and then returns it. This function follows similar pattern tobtf__resolve_sizebut instead of computing size, it just returns the type.These functions will be used in the followig patch which parses information inside array of
struct btf_type *.btf_name_by_offsetis used for printing variable names.Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/lib/bpf/btf.c| 65 ++++++++++++++++++++++++++++++++++++----------------tools/lib/bpf/btf.h| 3 ++2 files changed, 48 insertions(+), 20 deletions(-)tools/lib/bpf/btf.c +5
-8@@ -17,6 +17,11 @@#define BTF_MAX_NR_TYPES 65535 +#define IS_MODIFIER (k) (((k) == BTF_KIND_TYPEDEF) || \ + ((k) == BTF_KIND_VOLATILE) || \ + ((k) == BTF_KIND_CONST) || \ + ((k) == BTF_KIND_RESTRICT)) +static struct btf_type btf_void;struct btf {@@ -33,14 +38,6 @@ struct btf {int fd; }; -static const char *btf_name_by_offset (const struct btf *btf,uint32_t offset) -{ -if (offset < btf->hdr->str_len) -return &btf->strings[offset]; -else -return NULL ; -} -static int btf_add_type (struct btf *btf,struct btf_type *t) {if (btf->types_size - btf->nr_types <2 ) {@@ -190,15 +187,6 @@ static int btf_parse_type_sec(struct btfreturn 0 ; }Quoted 10 lines
tools/lib/bpf/btf.c static int btf_parse_type_sec(struct btf · line 200
Quoted 8 lines
tools/lib/bpf/btf.c +0
-8-static const struct btf_type *btf_type_by_id (const struct btf *btf, -uint32_t type_id) -{ -if (type_id > btf->nr_types) -return NULL ; - -return btf->types[type_id]; -}nit: Why do we need to move these two functions to later of the file?
Quoted 2 lines
No real reason. It looked like this file was following a convention of keeping statics together. I'll put them back in place if no one objects.
I see. Let's keep this patch as-is then.
Thanks,
Song
Quoted 99 lines
Quoted 5 lines
Other than this,
Acked-by Song Liu <[email protected]>Thanks
Quoted 92 lines
Quoted 92 lines
tools/lib/bpf/btf.c +40
-4-static bool btf_type_is_void (const struct btf_type *t) {return t == &btf_void ||BTF_INFO_KIND (t->info) ==BTF_KIND_FWD ;@@ -234,7 +222,7 @@ int64_t btf__resolve_size(const struct bint64_t size =-1 ;int i; - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id);for (i =0 ; i <MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null (t); i++) { size =btf_type_size (t);@@ -259,7 +247,7 @@ int64_t btf__resolve_size(const struct breturn -EINVAL ; } - t =btf_type_by_id (btf, type_id); + t =btf__type_by_id (btf, type_id); }if (size <0 )@@ -271,6 +259,26 @@ int64_t btf__resolve_size(const struct breturn nelems * size; } +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id) +{ +const struct btf_type *t; +int depth =0 ; + + t =btf__type_by_id (btf, type_id); +while (depth <MAX_RESOLVE_DEPTH && + !btf_type_is_void_or_null (t) && +IS_MODIFIER (BTF_INFO_KIND (t->info))) { + type_id = t->type; + t =btf__type_by_id (btf, type_id); + depth++; + } + +if (depth ==MAX_RESOLVE_DEPTH ||btf_type_is_void_or_null (t)) +return -EINVAL ; + +return type_id; +} +int32_t btf__find_by_name (const struct btf *btf,const char *type_name) {uint32_t i;@@ -280,7 +288,7 @@ int32_t btf__find_by_name(const struct bfor (i =1 ; i <= btf->nr_types; i++) {const struct btf_type *t = btf->types[i]; -const char *name =btf_name_by_offset (btf, t->name_off); +const char *name =btf__name_by_offset (btf, t->name_off);if (name && !strcmp (type_name, name))return i;@@ -371,3 +379,20 @@ int btf__fd(const struct btf *btf) {return btf->fd; } + +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset) +{ +if (offset < btf->hdr->str_len) +return &btf->strings[offset]; +else +return NULL ; +} + +const struct btf_type *btf__type_by_id (const struct btf *btf, +uint32_t type_id) +{ +if (type_id > btf->nr_types) +return NULL ; + +return btf->types[type_id]; +}tools/lib/bpf/btf.h +3
-0@@ -17,6 +17,9 @@ void btf__free(struct btf *btf);struct btf *btf__new (uint8_t *data,uint32_t size,btf_print_fn_t err_log);int32_t btf__find_by_name (const struct btf *btf,const char *type_name);int64_t btf__resolve_size (const struct btf *btf,uint32_t type_id); +int32_t btf__resolve_type (const struct btf *btf,uint32_t type_id);int btf__fd (const struct btf *btf); +const char *btf__name_by_offset (const struct btf *btf,uint32_t offset); +const struct btf_type *btf__type_by_id (const struct btf *btf,uint32_t type_id);#endif
PATCH 2/3 bpf: btf: add btf json print functionality
In reply to Okash KhawajaThis consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on json_writer to do json printing. Below is sample output where map keys are ints and values are of type struct A:
typedef int int_type;
enum E {
E0,
E1,
};struct B {
int x;
int y;
};struct A {
int m;
unsigned long long n;
char o;
int p[8];
int q[4][8];
enum E r;
void *s;
struct B t;
const int u;
int_type v;
unsigned int w1: 3;
unsigned int w2: 3;
};$ sudo bpftool map dump -p id 14
[{
"key": 0
},{
"value": {
"m": 1,
"n": 2,
"o": "c",
"p": [15,16,17,18,15,16,17,18
],
"q": [[25,26,27,28,25,26,27,28
],[35,36,37,38,35,36,37,38
],[45,46,47,48,45,46,47,48
],[55,56,57,58,55,56,57,58
]
],
"r": 1,
"s": 0x7ffff6f70568,
"t": {
"x": 5,
"y": 10
},
"u": 100,
"v": 20,
"w1": 0x7,
"w2": 0x3
}
}
]This patch uses json's {} and [] to imply struct/union and array. More explicit information can be added later. For example, a command line option can be introduced to print whether a key or value is struct or union, name of a struct etc. This will however come at the expense of duplicating info when, for example, printing an array of structs. enums are printed as ints without their names.
--- tools/bpf/bpftool/btf_dumper.c | 247 +++++++++++++++++++++++++++++++++++++++++ tools/bpf/bpftool/btf_dumper.h | 18 ++ 2 files changed, 265 insertions(+)
tools/bpf/bpftool/btf_dumper.c new file +247 -0
@@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#include <linux/btf.h> +#include <linux/err.h> +#include <stdio.h> /* for (FILE *) used by json_writer */ +#include <linux/bitops.h> +#include <string.h> +#include <ctype.h> + +#include "btf.h" +#include "json_writer.h" + +#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) +#define BITS_PER_BYTE_MASKED (bits) ((bits) & BITS_PER_BYTE_MASK) +#define BITS_ROUNDDOWN_BYTES (bits) ((bits) >> 3) +#define BITS_ROUNDUP_BYTES (bits) \ + (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw); + +static void btf_dumper_ptr (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%p" , *((uintptr_t *)data)); +} + +static int btf_dumper_modifier (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +int32_t actual_type_id =btf__resolve_type (btf, type_id); +int ret; + +if (actual_type_id <0 ) +return actual_type_id; + + ret =btf_dumper_do_type (btf, actual_type_id,0 , data, jw); + +return ret; +} + +static void btf_dumper_enum (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%d" , *((int32_t *)data)); +} + +static int btf_dumper_array (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_array *arr = (struct btf_array *)(t +1 ); +int64_t elem_size; +uint32_t i; +int ret; + + elem_size =btf__resolve_size (btf, arr->type); +if (elem_size <0 ) +return elem_size; + +jsonw_start_array (jw); +for (i =0 ; i < arr->nelems; i++) { + ret =btf_dumper_do_type (btf, arr->type,0 , + data + (i * elem_size), jw); +if (ret) +return ret; + } + +jsonw_end_array (jw); + +return 0 ; +} + +static void btf_dumper_int_bits (uint32_t int_type,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint16_t total_bits_offset; +uint32_t bits =BTF_INT_BITS (int_type); +uint16_t bytes_to_copy; +uint16_t bits_to_copy; +uint8_t upper_bits; +union { +uint64_t u64_num; +uint8_t u8_nums[8 ]; + } print_num; + + total_bits_offset = bit_offset +BTF_INT_OFFSET (int_type); + data +=BITS_ROUNDDOWN_BYTES (total_bits_offset); + bit_offset =BITS_PER_BYTE_MASKED (total_bits_offset); + bits_to_copy = bits + bit_offset; + bytes_to_copy =BITS_ROUNDUP_BYTES (bits_to_copy); + + print_num.u64_num =0 ; +memcpy (&print_num.u64_num, data, bytes_to_copy); + + upper_bits =BITS_PER_BYTE_MASKED (bits_to_copy); +if (upper_bits) { +uint8_t mask = (1 << upper_bits) -1 ; + + print_num.u8_nums[bytes_to_copy -1 ] &= mask; + } + + print_num.u64_num >>= bit_offset; + +jsonw_printf (jw,"0x%llx" , print_num.u64_num); +} + +static int btf_dumper_int (const struct btf_type *t,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint32_t *int_type = (uint32_t *)(t +1 ); +uint32_t bits =BTF_INT_BITS (*int_type); +int ret =0 ; + +/* if this is bit field */ +if (bit_offset ||BTF_INT_OFFSET (*int_type) || +BITS_PER_BYTE_MASKED (bits)) { +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +return ret; + } + +switch (BTF_INT_ENCODING (*int_type)) { +case 0 : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%lu" , *((uint64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%u" , *((uint32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hu" , *((uint16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhu" , *((uint8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_SIGNED : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%ld" , *((int64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%d" , *((int32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hd" , *((int16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhd" , *((int8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_CHAR : +if (*((char *)data) =='\0' ) +jsonw_null (jw); +else if (isprint (*((char *)data))) +jsonw_printf (jw,"\"%c\"" , *((char *)data)); +else +jsonw_printf (jw,"%hhx" , *((char *)data)); +break ; +case BTF_INT_BOOL : +jsonw_bool (jw, *((int *)data)); +break ; +default : +/* shouldn't happen */ + ret = -EINVAL ; +break ; + } + +return ret; +} + +static int btf_dumper_struct (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_member *m; +int ret =0 ; +int i, vlen; + +if (t ==NULL ) +return -EINVAL ; + + vlen =BTF_INFO_VLEN (t->info); +jsonw_start_object (jw); + m = (struct btf_member *)(t +1 ); + +for (i =0 ; i < vlen; i++) { +jsonw_name (jw,btf__name_by_offset (btf, m[i].name_off)); + ret =btf_dumper_do_type (btf, m[i].type, +BITS_PER_BYTE_MASKED (m[i].offset), + data +BITS_ROUNDDOWN_BYTES (m[i].offset), jw); +if (ret) +return ret; + } + +jsonw_end_object (jw); + +return 0 ; +} + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +int ret =0 ; + +switch (BTF_INFO_KIND (t->info)) { +case BTF_KIND_INT : + ret =btf_dumper_int (t, bit_offset, data, jw); +break ; +case BTF_KIND_STRUCT : +case BTF_KIND_UNION : + ret =btf_dumper_struct (btf, type_id, data, jw); +break ; +case BTF_KIND_ARRAY : + ret =btf_dumper_array (btf, type_id, data, jw); +break ; +case BTF_KIND_ENUM : +btf_dumper_enum (data, jw); +break ; +case BTF_KIND_PTR : +btf_dumper_ptr (data, jw); +break ; +case BTF_KIND_UNKN : +jsonw_printf (jw,"(unknown)" ); +break ; +case BTF_KIND_FWD : +/* map key or value can't be forward */ + ret = -EINVAL ; +break ; +case BTF_KIND_TYPEDEF : +case BTF_KIND_VOLATILE : +case BTF_KIND_CONST : +case BTF_KIND_RESTRICT : + ret =btf_dumper_modifier (btf, type_id, data, jw); +break ; +default : +jsonw_printf (jw,"(unsupported-kind" ); + ret = -EINVAL ; +break ; + } + +return ret; +} + +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,const void *data) +{ +if (!jw) +return -EINVAL ; + +return btf_dumper_do_type (btf, type_id,0 , data, jw); +}
tools/bpf/bpftool/btf_dumper.h new file +18 -0
@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#ifndef BTF_DUMPER_H +#define BTF_DUMPER_H + +/* btf_dumper_type - json print data along with type information +* @btf: btf instance initialised via btf__new() +* @jw: json writer used for printing +* @type_id: index in btf->types array. this points to the type to be dumped +* @data: pointer the actual data, i.e. the values to be printed +* +* Returns zero on success and negative error code otherwise +*/ +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,void *data); + +#endif
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 139 lines · Okash Khawaja
This consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on
json_writerto do json printing. Below is sample output where map keys are ints and values are of type struct A:typedef int
int_type;enum E { E0, E1, };struct B { int x; int y; };struct A { int m; unsigned long long n; char o; int p[8]; int q[4][8]; enum E r; void *s; struct B t; const int u; int_type v; unsigned int w1: 3; unsigned int w2: 3; };$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10 }, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]This patch uses json's {} and [] to imply struct/union and array. More explicit information can be added later. For example, a command line option can be introduced to print whether a key or value is struct or union, name of a struct etc. This will however come at the expense of duplicating info when, for example, printing an array of structs. enums are printed as ints without their names.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/bpf/bpftool/btf_dumper.c| 247 +++++++++++++++++++++++++++++++++++++++++tools/bpf/bpftool/btf_dumper.h| 18 ++2 files changed, 265 insertions(+)tools/bpf/bpftool/btf_dumper.c new file +55
-0@@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#include <linux/btf.h> +#include <linux/err.h> +#include <stdio.h> /* for (FILE *) used by json_writer */ +#include <linux/bitops.h> +#include <string.h> +#include <ctype.h> + +#include "btf.h" +#include "json_writer.h" + +#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) +#define BITS_PER_BYTE_MASKED (bits) ((bits) & BITS_PER_BYTE_MASK) +#define BITS_ROUNDDOWN_BYTES (bits) ((bits) >> 3) +#define BITS_ROUNDUP_BYTES (bits) \ + (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw); + +static void btf_dumper_ptr (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%p" , *((uintptr_t *)data)); +} + +static int btf_dumper_modifier (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +int32_t actual_type_id =btf__resolve_type (btf, type_id); +int ret; + +if (actual_type_id <0 ) +return actual_type_id; + + ret =btf_dumper_do_type (btf, actual_type_id,0 , data, jw); + +return ret; +} + +static void btf_dumper_enum (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%d" , *((int32_t *)data)); +} + +static int btf_dumper_array (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_array *arr = (struct btf_array *)(t +1 ); +int64_t elem_size; +uint32_t i; +int ret; +
tools/bpf/bpftool/btf_dumper.c line 65
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+ elem_size =btf__resolve_size (btf, arr->type); +if (elem_size <0 ) +return elem_size; + +jsonw_start_array (jw); +for (i =0 ; i < arr->nelems; i++) { + ret =btf_dumper_do_type (btf, arr->type,0 , + data + (i * elem_size), jw); +if (ret) +return ret;
Shall we call jsonw_end_array() before return here?
Quoted 112 lines
tools/bpf/bpftool/btf_dumper.c new file +112
-0+ } + +jsonw_end_array (jw); + +return 0 ; +} + +static void btf_dumper_int_bits (uint32_t int_type,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint16_t total_bits_offset; +uint32_t bits =BTF_INT_BITS (int_type); +uint16_t bytes_to_copy; +uint16_t bits_to_copy; +uint8_t upper_bits; +union { +uint64_t u64_num; +uint8_t u8_nums[8 ]; + } print_num; + + total_bits_offset = bit_offset +BTF_INT_OFFSET (int_type); + data +=BITS_ROUNDDOWN_BYTES (total_bits_offset); + bit_offset =BITS_PER_BYTE_MASKED (total_bits_offset); + bits_to_copy = bits + bit_offset; + bytes_to_copy =BITS_ROUNDUP_BYTES (bits_to_copy); + + print_num.u64_num =0 ; +memcpy (&print_num.u64_num, data, bytes_to_copy); + + upper_bits =BITS_PER_BYTE_MASKED (bits_to_copy); +if (upper_bits) { +uint8_t mask = (1 << upper_bits) -1 ; + + print_num.u8_nums[bytes_to_copy -1 ] &= mask; + } + + print_num.u64_num >>= bit_offset; + +jsonw_printf (jw,"0x%llx" , print_num.u64_num); +} + +static int btf_dumper_int (const struct btf_type *t,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint32_t *int_type = (uint32_t *)(t +1 ); +uint32_t bits =BTF_INT_BITS (*int_type); +int ret =0 ; + +/* if this is bit field */ +if (bit_offset ||BTF_INT_OFFSET (*int_type) || +BITS_PER_BYTE_MASKED (bits)) { +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +return ret; + } + +switch (BTF_INT_ENCODING (*int_type)) { +case 0 : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%lu" , *((uint64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%u" , *((uint32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hu" , *((uint16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhu" , *((uint8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_SIGNED : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%ld" , *((int64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%d" , *((int32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hd" , *((int16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhd" , *((int8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_CHAR : +if (*((char *)data) =='\0' ) +jsonw_null (jw); +else if (isprint (*((char *)data))) +jsonw_printf (jw,"\"%c\"" , *((char *)data)); +else +jsonw_printf (jw,"%hhx" , *((char *)data)); +break ; +case BTF_INT_BOOL : +jsonw_bool (jw, *((int *)data)); +break ; +default : +/* shouldn't happen */ + ret = -EINVAL ; +break ; + } + +return ret; +} + +static int btf_dumper_struct (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_member *m; +int ret =0 ; +int i, vlen; + +if (t ==NULL ) +return -EINVAL ; + + vlen =BTF_INFO_VLEN (t->info);
tools/bpf/bpftool/btf_dumper.c line 187
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+jsonw_start_object (jw); + m = (struct btf_member *)(t +1 ); + +for (i =0 ; i < vlen; i++) { +jsonw_name (jw,btf__name_by_offset (btf, m[i].name_off)); + ret =btf_dumper_do_type (btf, m[i].type, +BITS_PER_BYTE_MASKED (m[i].offset), + data +BITS_ROUNDDOWN_BYTES (m[i].offset), jw); +if (ret) +return ret;
Shall we call jsonw_end_object() before return here?
Quoted 35 lines
tools/bpf/bpftool/btf_dumper.c new file +35
-0+ } + +jsonw_end_object (jw); + +return 0 ; +} + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +int ret =0 ; + +switch (BTF_INFO_KIND (t->info)) { +case BTF_KIND_INT : + ret =btf_dumper_int (t, bit_offset, data, jw); +break ; +case BTF_KIND_STRUCT : +case BTF_KIND_UNION : + ret =btf_dumper_struct (btf, type_id, data, jw); +break ; +case BTF_KIND_ARRAY : + ret =btf_dumper_array (btf, type_id, data, jw); +break ; +case BTF_KIND_ENUM : +btf_dumper_enum (data, jw); +break ; +case BTF_KIND_PTR : +btf_dumper_ptr (data, jw); +break ; +case BTF_KIND_UNKN : +jsonw_printf (jw,"(unknown)" ); +break ; +case BTF_KIND_FWD : +/* map key or value can't be forward */
tools/bpf/bpftool/btf_dumper.c line 232
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+ ret = -EINVAL ; +break ; +case BTF_KIND_TYPEDEF : +case BTF_KIND_VOLATILE : +case BTF_KIND_CONST : +case BTF_KIND_RESTRICT : + ret =btf_dumper_modifier (btf, type_id, data, jw); +break ; +default : +jsonw_printf (jw,"(unsupported-kind" );
"(unsupported-kind)" (missing ')').
Quoted 4 lines
tools/bpf/bpftool/btf_dumper.c new file +4
-0+ ret = -EINVAL ; +break ; + } +
tools/bpf/bpftool/btf_dumper.c line 246
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+return ret; +} + +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,const void *data) +{ +if (!jw) +return -EINVAL ; + +return btf_dumper_do_type (btf, type_id,0 , data, jw);
nit: btf_dumper_do_type() returns int, while btf_dumper_type() returns int32_t.
This should not matter though.
Quoted 23 lines
tools/bpf/bpftool/btf_dumper.c new file +1
-0+}tools/bpf/bpftool/btf_dumper.h new file +18
-0@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#ifndef BTF_DUMPER_H +#define BTF_DUMPER_H + +/* btf_dumper_type - json print data along with type information +* @btf: btf instance initialised via btf__new() +* @jw: json writer used for printing +* @type_id: index in btf->types array. this points to the type to be dumped +* @data: pointer the actual data, i.e. the values to be printed +* +* Returns zero on success and negative error code otherwise +*/ +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,void *data); + +#endif
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 143 lines · Song Liu
Quoted 143 lines · Okash Khawaja
This consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on
json_writerto do json printing. Below is sample output where map keys are ints and values are of type struct A:typedef int
int_type;enum E { E0, E1, };struct B { int x; int y; };struct A { int m; unsigned long long n; char o; int p[8]; int q[4][8]; enum E r; void *s; struct B t; const int u; int_type v; unsigned int w1: 3; unsigned int w2: 3; };$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10 }, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]This patch uses json's {} and [] to imply struct/union and array. More explicit information can be added later. For example, a command line option can be introduced to print whether a key or value is struct or union, name of a struct etc. This will however come at the expense of duplicating info when, for example, printing an array of structs. enums are printed as ints without their names.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/bpf/bpftool/btf_dumper.c| 247 +++++++++++++++++++++++++++++++++++++++++tools/bpf/bpftool/btf_dumper.h| 18 ++2 files changed, 265 insertions(+)tools/bpf/bpftool/btf_dumper.c new file +59
-0@@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#include <linux/btf.h> +#include <linux/err.h> +#include <stdio.h> /* for (FILE *) used by json_writer */ +#include <linux/bitops.h> +#include <string.h> +#include <ctype.h> + +#include "btf.h" +#include "json_writer.h" + +#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) +#define BITS_PER_BYTE_MASKED (bits) ((bits) & BITS_PER_BYTE_MASK) +#define BITS_ROUNDDOWN_BYTES (bits) ((bits) >> 3) +#define BITS_ROUNDUP_BYTES (bits) \ + (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw); + +static void btf_dumper_ptr (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%p" , *((uintptr_t *)data)); +} + +static int btf_dumper_modifier (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +int32_t actual_type_id =btf__resolve_type (btf, type_id); +int ret; + +if (actual_type_id <0 ) +return actual_type_id; + + ret =btf_dumper_do_type (btf, actual_type_id,0 , data, jw); + +return ret; +} + +static void btf_dumper_enum (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%d" , *((int32_t *)data)); +} + +static int btf_dumper_array (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_array *arr = (struct btf_array *)(t +1 ); +int64_t elem_size; +uint32_t i; +int ret; + + elem_size =btf__resolve_size (btf, arr->type); +if (elem_size <0 ) +return elem_size; +
Quoted 8 lines
tools/bpf/bpftool/btf_dumper.c line 65
Quoted 6 lines
tools/bpf/bpftool/btf_dumper.c new file +6
-0+jsonw_start_array (jw); +for (i =0 ; i < arr->nelems; i++) { + ret =btf_dumper_do_type (btf, arr->type,0 , + data + (i * elem_size), jw); +if (ret) +return ret;Shall we call
jsonw_end_array()before return here?
Thanks. I'll update the patch.
Quoted 165 lines
Quoted 113 lines
tools/bpf/bpftool/btf_dumper.c new file +112
-0+ } + +jsonw_end_array (jw); + +return 0 ; +} + +static void btf_dumper_int_bits (uint32_t int_type,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint16_t total_bits_offset; +uint32_t bits =BTF_INT_BITS (int_type); +uint16_t bytes_to_copy; +uint16_t bits_to_copy; +uint8_t upper_bits; +union { +uint64_t u64_num; +uint8_t u8_nums[8 ]; + } print_num; + + total_bits_offset = bit_offset +BTF_INT_OFFSET (int_type); + data +=BITS_ROUNDDOWN_BYTES (total_bits_offset); + bit_offset =BITS_PER_BYTE_MASKED (total_bits_offset); + bits_to_copy = bits + bit_offset; + bytes_to_copy =BITS_ROUNDUP_BYTES (bits_to_copy); + + print_num.u64_num =0 ; +memcpy (&print_num.u64_num, data, bytes_to_copy); + + upper_bits =BITS_PER_BYTE_MASKED (bits_to_copy); +if (upper_bits) { +uint8_t mask = (1 << upper_bits) -1 ; + + print_num.u8_nums[bytes_to_copy -1 ] &= mask; + } + + print_num.u64_num >>= bit_offset; + +jsonw_printf (jw,"0x%llx" , print_num.u64_num); +} + +static int btf_dumper_int (const struct btf_type *t,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint32_t *int_type = (uint32_t *)(t +1 ); +uint32_t bits =BTF_INT_BITS (*int_type); +int ret =0 ; + +/* if this is bit field */ +if (bit_offset ||BTF_INT_OFFSET (*int_type) || +BITS_PER_BYTE_MASKED (bits)) { +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +return ret; + } + +switch (BTF_INT_ENCODING (*int_type)) { +case 0 : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%lu" , *((uint64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%u" , *((uint32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hu" , *((uint16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhu" , *((uint8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_SIGNED : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%ld" , *((int64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%d" , *((int32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hd" , *((int16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhd" , *((int8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_CHAR : +if (*((char *)data) =='\0' ) +jsonw_null (jw); +else if (isprint (*((char *)data))) +jsonw_printf (jw,"\"%c\"" , *((char *)data)); +else +jsonw_printf (jw,"%hhx" , *((char *)data)); +break ; +case BTF_INT_BOOL : +jsonw_bool (jw, *((int *)data)); +break ; +default : +/* shouldn't happen */ + ret = -EINVAL ; +break ; + } + +return ret; +} + +static int btf_dumper_struct (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_member *m; +int ret =0 ; +int i, vlen; + +if (t ==NULL ) +return -EINVAL ; + + vlen =BTF_INFO_VLEN (t->info);tools/bpf/bpftool/btf_dumper.c line 187
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+jsonw_start_object (jw); + m = (struct btf_member *)(t +1 ); + +for (i =0 ; i < vlen; i++) { +jsonw_name (jw,btf__name_by_offset (btf, m[i].name_off)); + ret =btf_dumper_do_type (btf, m[i].type, +BITS_PER_BYTE_MASKED (m[i].offset), + data +BITS_ROUNDDOWN_BYTES (m[i].offset), jw); +if (ret) +return ret;Shall we call
jsonw_end_object()before return here?Quoted 39 lines
tools/bpf/bpftool/btf_dumper.c new file +39
-0+ } + +jsonw_end_object (jw); + +return 0 ; +} + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +int ret =0 ; + +switch (BTF_INFO_KIND (t->info)) { +case BTF_KIND_INT : + ret =btf_dumper_int (t, bit_offset, data, jw); +break ; +case BTF_KIND_STRUCT : +case BTF_KIND_UNION : + ret =btf_dumper_struct (btf, type_id, data, jw); +break ; +case BTF_KIND_ARRAY : + ret =btf_dumper_array (btf, type_id, data, jw); +break ; +case BTF_KIND_ENUM : +btf_dumper_enum (data, jw); +break ; +case BTF_KIND_PTR : +btf_dumper_ptr (data, jw); +break ; +case BTF_KIND_UNKN : +jsonw_printf (jw,"(unknown)" ); +break ; +case BTF_KIND_FWD : +/* map key or value can't be forward */ + ret = -EINVAL ; +break ; +case BTF_KIND_TYPEDEF : +case BTF_KIND_VOLATILE :
Quoted 8 lines
tools/bpf/bpftool/btf_dumper.c line 232
Quoted 6 lines
tools/bpf/bpftool/btf_dumper.c new file +6
-0+case BTF_KIND_CONST : +case BTF_KIND_RESTRICT : + ret =btf_dumper_modifier (btf, type_id, data, jw); +break ; +default : +jsonw_printf (jw,"(unsupported-kind" );"(unsupported-kind)" (missing ')').
Thank you, will update the patch.
Quoted 17 lines
Quoted 5 lines
tools/bpf/bpftool/btf_dumper.c new file +4
-0+ ret = -EINVAL ; +break ; + } +tools/bpf/bpftool/btf_dumper.c line 246
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+return ret; +} + +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,const void *data) +{ +if (!jw) +return -EINVAL ; + +return btf_dumper_do_type (btf, type_id,0 , data, jw);nit:
btf_dumper_do_type()returns int, whilebtf_dumper_type()returnsint32_t.
This should not matter though.
Probably an oversight from me. I'll make them consistent. Thank you.
Quoted 25 lines
Quoted 24 lines
tools/bpf/bpftool/btf_dumper.c new file +1
-0+}tools/bpf/bpftool/btf_dumper.h new file +18
-0@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#ifndef BTF_DUMPER_H +#define BTF_DUMPER_H + +/* btf_dumper_type - json print data along with type information +* @btf: btf instance initialised via btf__new() +* @jw: json writer used for printing +* @type_id: index in btf->types array. this points to the type to be dumped +* @data: pointer the actual data, i.e. the values to be printed +* +* Returns zero on success and negative error code otherwise +*/ +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,void *data); + +#endif
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
In reply to Okash KhawajaHi Okash,
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
Quoted 42 lines
This consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on
json_writerto do json printing. Below is sample output where map keys are ints and values are of type struct A:typedef int
int_type;enum E { E0, E1, };struct B { int x; int y; };struct A { int m; unsigned long long n; char o; int p[8]; int q[4][8]; enum E r; void *s; struct B t; const int u; int_type v; unsigned int w1: 3; unsigned int w2: 3; };$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ],
Quoted 8 lines
"q": [[25,26,27,28,25,26,27,28
],[35,36,37,38,35,36,37,38
],[45,46,47,48,45,46,47,48
],[55,56,57,58,55,56,57,58
]
],
"r": 1,
"s": 0x7ffff6f70568,Hexadecimal values, without quotes, are not valid JSON. Please stick to decimal values.
Quoted 97 lines
"t": { "x": 5, "y": 10 }, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } }]
This patch uses json's {} and [] to imply struct/union and array. More explicit information can be added later. For example, a command line option can be introduced to print whether a key or value is struct or union, name of a struct etc. This will however come at the expense of duplicating info when, for example, printing an array of structs. enums are printed as ints without their names.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>--- tools/bpf/bpftool/btf_dumper.c | 247 +++++++++++++++++++++++++++++++++++++++++ tools/bpf/bpftool/btf_dumper.h | 18 ++ 2 files changed, 265 insertions(+)tools/bpf/bpftool/btf_dumper.c new file +67
-0@@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#include <linux/btf.h> +#include <linux/err.h> +#include <stdio.h> /* for (FILE *) used by json_writer */ +#include <linux/bitops.h> +#include <string.h> +#include <ctype.h> + +#include "btf.h" +#include "json_writer.h" + +#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) +#define BITS_PER_BYTE_MASKED (bits) ((bits) & BITS_PER_BYTE_MASK) +#define BITS_ROUNDDOWN_BYTES (bits) ((bits) >> 3) +#define BITS_ROUNDUP_BYTES (bits) \ + (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw); + +static void btf_dumper_ptr (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%p" , *((uintptr_t *)data)); +} + +static int btf_dumper_modifier (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +int32_t actual_type_id =btf__resolve_type (btf, type_id); +int ret; + +if (actual_type_id <0 ) +return actual_type_id; + + ret =btf_dumper_do_type (btf, actual_type_id,0 , data, jw); + +return ret; +} + +static void btf_dumper_enum (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%d" , *((int32_t *)data)); +} + +static int btf_dumper_array (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_array *arr = (struct btf_array *)(t +1 ); +int64_t elem_size; +uint32_t i; +int ret; + + elem_size =btf__resolve_size (btf, arr->type); +if (elem_size <0 ) +return elem_size; + +jsonw_start_array (jw); +for (i =0 ; i < arr->nelems; i++) { + ret =btf_dumper_do_type (btf, arr->type,0 , + data + (i * elem_size), jw); +if (ret) +return ret; + } +
tools/bpf/bpftool/btf_dumper.c line 77
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+jsonw_end_array (jw); + +return 0 ; +} + +static void btf_dumper_int_bits (uint32_t int_type,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint16_t total_bits_offset; +uint32_t bits =BTF_INT_BITS (int_type);
Nit: Please use reverse-Christmas-tree here.
As for patch 3 you also have a number of continuation lines not aligned on the opening parenthesis from the first line, throughout the patch (please consider running checkpatch in "--strict" mode for the list).
Quoted 173 lines
tools/bpf/bpftool/btf_dumper.c new file +170
-0+uint16_t bytes_to_copy; +uint16_t bits_to_copy; +uint8_t upper_bits; +union { +uint64_t u64_num; +uint8_t u8_nums[8 ]; + } print_num; + + total_bits_offset = bit_offset +BTF_INT_OFFSET (int_type); + data +=BITS_ROUNDDOWN_BYTES (total_bits_offset); + bit_offset =BITS_PER_BYTE_MASKED (total_bits_offset); + bits_to_copy = bits + bit_offset; + bytes_to_copy =BITS_ROUNDUP_BYTES (bits_to_copy); + + print_num.u64_num =0 ; +memcpy (&print_num.u64_num, data, bytes_to_copy); + + upper_bits =BITS_PER_BYTE_MASKED (bits_to_copy); +if (upper_bits) { +uint8_t mask = (1 << upper_bits) -1 ; + + print_num.u8_nums[bytes_to_copy -1 ] &= mask; + } + + print_num.u64_num >>= bit_offset; + +jsonw_printf (jw,"0x%llx" , print_num.u64_num); +} + +static int btf_dumper_int (const struct btf_type *t,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint32_t *int_type = (uint32_t *)(t +1 ); +uint32_t bits =BTF_INT_BITS (*int_type); +int ret =0 ; + +/* if this is bit field */ +if (bit_offset ||BTF_INT_OFFSET (*int_type) || +BITS_PER_BYTE_MASKED (bits)) { +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +return ret; + } + +switch (BTF_INT_ENCODING (*int_type)) { +case 0 : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%lu" , *((uint64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%u" , *((uint32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hu" , *((uint16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhu" , *((uint8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_SIGNED : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%ld" , *((int64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%d" , *((int32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hd" , *((int16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhd" , *((int8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_CHAR : +if (*((char *)data) =='\0' ) +jsonw_null (jw); +else if (isprint (*((char *)data))) +jsonw_printf (jw,"\"%c\"" , *((char *)data)); +else +jsonw_printf (jw,"%hhx" , *((char *)data)); +break ; +case BTF_INT_BOOL : +jsonw_bool (jw, *((int *)data)); +break ; +default : +/* shouldn't happen */ + ret = -EINVAL ; +break ; + } + +return ret; +} + +static int btf_dumper_struct (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_member *m; +int ret =0 ; +int i, vlen; + +if (t ==NULL ) +return -EINVAL ; + + vlen =BTF_INFO_VLEN (t->info); +jsonw_start_object (jw); + m = (struct btf_member *)(t +1 ); + +for (i =0 ; i < vlen; i++) { +jsonw_name (jw,btf__name_by_offset (btf, m[i].name_off)); + ret =btf_dumper_do_type (btf, m[i].type, +BITS_PER_BYTE_MASKED (m[i].offset), + data +BITS_ROUNDDOWN_BYTES (m[i].offset), jw); +if (ret) +return ret; + } + +jsonw_end_object (jw); + +return 0 ; +} + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +int ret =0 ; + +switch (BTF_INFO_KIND (t->info)) { +case BTF_KIND_INT : + ret =btf_dumper_int (t, bit_offset, data, jw); +break ; +case BTF_KIND_STRUCT : +case BTF_KIND_UNION : + ret =btf_dumper_struct (btf, type_id, data, jw); +break ; +case BTF_KIND_ARRAY : + ret =btf_dumper_array (btf, type_id, data, jw); +break ; +case BTF_KIND_ENUM : +btf_dumper_enum (data, jw); +break ; +case BTF_KIND_PTR : +btf_dumper_ptr (data, jw); +break ; +case BTF_KIND_UNKN : +jsonw_printf (jw,"(unknown)" ); +break ; +case BTF_KIND_FWD : +/* map key or value can't be forward */ + ret = -EINVAL ; +break ; +case BTF_KIND_TYPEDEF : +case BTF_KIND_VOLATILE : +case BTF_KIND_CONST : +case BTF_KIND_RESTRICT : + ret =btf_dumper_modifier (btf, type_id, data, jw); +break ; +default : +jsonw_printf (jw,"(unsupported-kind" ); + ret = -EINVAL ; +break ; + } + +return ret; +} + +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,const void *data) +{ +if (!jw) +return -EINVAL ; + +return btf_dumper_do_type (btf, type_id,0 , data, jw); +}
tools/bpf/bpftool/btf_dumper.h line 1
Quoted 1 line
tools/bpf/bpftool/btf_dumper.h new file +1
-0@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */
I believe SPDX tag in header files should use C++ style comments (//)?
Quoted 8 lines
tools/bpf/bpftool/btf_dumper.h new file +7
-0+/* Copyright (c) 2018 Facebook */ + +#ifndef BTF_DUMPER_H +#define BTF_DUMPER_H + +/* btf_dumper_type - json print data along with type information +* @btf: btf instance initialised via btf__new()
tools/bpf/bpftool/btf_dumper.h line 18
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.h new file +10
-0+* @jw: json writer used for printing +* @type_id: index in btf->types array. this points to the type to be dumped +* @data: pointer the actual data, i.e. the values to be printed +* +* Returns zero on success and negative error code otherwise +*/ +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,void *data); + +#endif
Thanks,
Quentin
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 1 line · Quentin Monnet
Hi Okash,
hi and sorry about delay in responding. the email got routed to incorrect folder.
Quoted 53 lines
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
Quoted 42 lines
This consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on
json_writerto do json printing. Below is sample output where map keys are ints and values are of type struct A:typedef int
int_type;enum E { E0, E1, };struct B { int x; int y; };struct A { int m; unsigned long long n; char o; int p[8]; int q[4][8]; enum E r; void *s; struct B t; const int u; int_type v; unsigned int w1: 3; unsigned int w2: 3; };$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ],Quoted 8 lines
"q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568,
Quoted 2 lines
Hexadecimal values, without quotes, are not valid JSON. Please stick to decimal values.
ah sorry, i used a buggy json validator. this should be a quick fix. which would be better: pointers be output hex strings or integers?
Quoted 111 lines
Quoted 98 lines
"t": { "x": 5, "y": 10 }, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } }]
This patch uses json's {} and [] to imply struct/union and array. More explicit information can be added later. For example, a command line option can be introduced to print whether a key or value is struct or union, name of a struct etc. This will however come at the expense of duplicating info when, for example, printing an array of structs. enums are printed as ints without their names.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>--- tools/bpf/bpftool/btf_dumper.c | 247 +++++++++++++++++++++++++++++++++++++++++ tools/bpf/bpftool/btf_dumper.h | 18 ++ 2 files changed, 265 insertions(+)tools/bpf/bpftool/btf_dumper.c new file +67
-0@@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2018 Facebook */ + +#include <linux/btf.h> +#include <linux/err.h> +#include <stdio.h> /* for (FILE *) used by json_writer */ +#include <linux/bitops.h> +#include <string.h> +#include <ctype.h> + +#include "btf.h" +#include "json_writer.h" + +#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) +#define BITS_PER_BYTE_MASKED (bits) ((bits) & BITS_PER_BYTE_MASK) +#define BITS_ROUNDDOWN_BYTES (bits) ((bits) >> 3) +#define BITS_ROUNDUP_BYTES (bits) \ + (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw); + +static void btf_dumper_ptr (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%p" , *((uintptr_t *)data)); +} + +static int btf_dumper_modifier (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +int32_t actual_type_id =btf__resolve_type (btf, type_id); +int ret; + +if (actual_type_id <0 ) +return actual_type_id; + + ret =btf_dumper_do_type (btf, actual_type_id,0 , data, jw); + +return ret; +} + +static void btf_dumper_enum (const void *data,json_writer_t *jw) +{ +jsonw_printf (jw,"%d" , *((int32_t *)data)); +} + +static int btf_dumper_array (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_array *arr = (struct btf_array *)(t +1 ); +int64_t elem_size; +uint32_t i; +int ret; + + elem_size =btf__resolve_size (btf, arr->type); +if (elem_size <0 ) +return elem_size; + +jsonw_start_array (jw); +for (i =0 ; i < arr->nelems; i++) { + ret =btf_dumper_do_type (btf, arr->type,0 , + data + (i * elem_size), jw); +if (ret) +return ret; + } +tools/bpf/bpftool/btf_dumper.c line 77
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.c new file +10
-0+jsonw_end_array (jw); + +return 0 ; +} + +static void btf_dumper_int_bits (uint32_t int_type,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint16_t total_bits_offset; +uint32_t bits =BTF_INT_BITS (int_type);Nit: Please use reverse-Christmas-tree here.
Quoted 3 lines
As for patch 3 you also have a number of continuation lines not aligned on the opening parenthesis from the first line, throughout the patch (please consider running checkpatch in "--strict" mode for the list).
i didn't use "--strict". that would explain style mismatch. will fix that in v2.
Quoted 169 lines
Quoted 169 lines
tools/bpf/bpftool/btf_dumper.c new file +168
-0+uint16_t bytes_to_copy; +uint16_t bits_to_copy; +uint8_t upper_bits; +union { +uint64_t u64_num; +uint8_t u8_nums[8 ]; + } print_num; + + total_bits_offset = bit_offset +BTF_INT_OFFSET (int_type); + data +=BITS_ROUNDDOWN_BYTES (total_bits_offset); + bit_offset =BITS_PER_BYTE_MASKED (total_bits_offset); + bits_to_copy = bits + bit_offset; + bytes_to_copy =BITS_ROUNDUP_BYTES (bits_to_copy); + + print_num.u64_num =0 ; +memcpy (&print_num.u64_num, data, bytes_to_copy); + + upper_bits =BITS_PER_BYTE_MASKED (bits_to_copy); +if (upper_bits) { +uint8_t mask = (1 << upper_bits) -1 ; + + print_num.u8_nums[bytes_to_copy -1 ] &= mask; + } + + print_num.u64_num >>= bit_offset; + +jsonw_printf (jw,"0x%llx" , print_num.u64_num); +} + +static int btf_dumper_int (const struct btf_type *t,uint8_t bit_offset, +const void *data,json_writer_t *jw) +{ +uint32_t *int_type = (uint32_t *)(t +1 ); +uint32_t bits =BTF_INT_BITS (*int_type); +int ret =0 ; + +/* if this is bit field */ +if (bit_offset ||BTF_INT_OFFSET (*int_type) || +BITS_PER_BYTE_MASKED (bits)) { +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +return ret; + } + +switch (BTF_INT_ENCODING (*int_type)) { +case 0 : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%lu" , *((uint64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%u" , *((uint32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hu" , *((uint16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhu" , *((uint8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_SIGNED : +if (BTF_INT_BITS (*int_type) ==64 ) +jsonw_printf (jw,"%ld" , *((int64_t *)data)); +else if (BTF_INT_BITS (*int_type) ==32 ) +jsonw_printf (jw,"%d" , *((int32_t *)data)); +else if (BTF_INT_BITS (*int_type) ==16 ) +jsonw_printf (jw,"%hd" , *((int16_t *)data)); +else if (BTF_INT_BITS (*int_type) ==8 ) +jsonw_printf (jw,"%hhd" , *((int8_t *)data)); +else +btf_dumper_int_bits (*int_type, bit_offset, data, jw); +break ; +case BTF_INT_CHAR : +if (*((char *)data) =='\0' ) +jsonw_null (jw); +else if (isprint (*((char *)data))) +jsonw_printf (jw,"\"%c\"" , *((char *)data)); +else +jsonw_printf (jw,"%hhx" , *((char *)data)); +break ; +case BTF_INT_BOOL : +jsonw_bool (jw, *((int *)data)); +break ; +default : +/* shouldn't happen */ + ret = -EINVAL ; +break ; + } + +return ret; +} + +static int btf_dumper_struct (const struct btf *btf,uint32_t type_id, +const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +struct btf_member *m; +int ret =0 ; +int i, vlen; + +if (t ==NULL ) +return -EINVAL ; + + vlen =BTF_INFO_VLEN (t->info); +jsonw_start_object (jw); + m = (struct btf_member *)(t +1 ); + +for (i =0 ; i < vlen; i++) { +jsonw_name (jw,btf__name_by_offset (btf, m[i].name_off)); + ret =btf_dumper_do_type (btf, m[i].type, +BITS_PER_BYTE_MASKED (m[i].offset), + data +BITS_ROUNDDOWN_BYTES (m[i].offset), jw); +if (ret) +return ret; + } + +jsonw_end_object (jw); + +return 0 ; +} + +static int btf_dumper_do_type (const struct btf *btf,uint32_t type_id, +uint8_t bit_offset,const void *data,json_writer_t *jw) +{ +const struct btf_type *t =btf__type_by_id (btf, type_id); +int ret =0 ; + +switch (BTF_INFO_KIND (t->info)) { +case BTF_KIND_INT : + ret =btf_dumper_int (t, bit_offset, data, jw); +break ; +case BTF_KIND_STRUCT : +case BTF_KIND_UNION : + ret =btf_dumper_struct (btf, type_id, data, jw); +break ; +case BTF_KIND_ARRAY : + ret =btf_dumper_array (btf, type_id, data, jw); +break ; +case BTF_KIND_ENUM : +btf_dumper_enum (data, jw); +break ; +case BTF_KIND_PTR : +btf_dumper_ptr (data, jw); +break ; +case BTF_KIND_UNKN : +jsonw_printf (jw,"(unknown)" ); +break ; +case BTF_KIND_FWD : +/* map key or value can't be forward */ + ret = -EINVAL ; +break ; +case BTF_KIND_TYPEDEF : +case BTF_KIND_VOLATILE : +case BTF_KIND_CONST : +case BTF_KIND_RESTRICT : + ret =btf_dumper_modifier (btf, type_id, data, jw); +break ; +default : +jsonw_printf (jw,"(unsupported-kind" ); + ret = -EINVAL ; +break ; + } + +return ret; +} + +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,const void *data) +{ +if (!jw) +return -EINVAL ; +
Quoted 8 lines
tools/bpf/bpftool/btf_dumper.h line 1
Quoted 6 lines
tools/bpf/bpftool/btf_dumper.c new file +2
-0+return btf_dumper_do_type (btf, type_id,0 , data, jw); +}tools/bpf/bpftool/btf_dumper.h new file +1
-0@@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ I believe SPDX tag in header files should use C++ style comments (//)?
okay i will verify and fix that.
Quoted 22 lines
Quoted 9 lines
tools/bpf/bpftool/btf_dumper.h new file +7
-0+/* Copyright (c) 2018 Facebook */ + +#ifndef BTF_DUMPER_H +#define BTF_DUMPER_H + +/* btf_dumper_type - json print data along with type information +* @btf: btf instance initialised via btf__new() tools/bpf/bpftool/btf_dumper.h line 18
Quoted 10 lines
tools/bpf/bpftool/btf_dumper.h new file +10
-0+* @jw: json writer used for printing +* @type_id: index in btf->types array. this points to the type to be dumped +* @data: pointer the actual data, i.e. the values to be printed +* +* Returns zero on success and negative error code otherwise +*/ +int32_t btf_dumper_type (const struct btf *btf,json_writer_t *jw, +uint32_t type_id,void *data); + +#endif Thanks,
Quentin
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
2018-06-22 11:24 UTC+0100 ~ Okash Khawaja <[email protected]>
Quoted 57 lines · Quentin Monnet
Quoted 1 line
Hi Okash,
hi and sorry about delay in responding. the email got routed to incorrect folder.
Quoted 53 lines
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
Quoted 50 lines
This consumes functionality exported in the previous patch. It does the main job of printing with BTF data. This is used in the following patch to provide a more readable output of a map's dump. It relies on
json_writerto do json printing. Below is sample output where map keys are ints and values are of type struct A:typedef int
int_type;enum E { E0, E1, };struct B { int x; int y; };struct A { int m; unsigned long long n; char o; int p[8]; int q[4][8]; enum E r; void *s; struct B t; const int u; int_type v; unsigned int w1: 3; unsigned int w2: 3; };$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568,
Quoted 4 lines
Quoted 2 lines
Hexadecimal values, without quotes, are not valid JSON. Please stick to decimal values.
ah sorry, i used a buggy json validator. this should be a quick fix. which would be better: pointers be output hex strings or integers?
I would go for integers. Although this is harder to read for humans, it is easier to process for machines, which remain the primary targets for JSON output.
Quentin
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 9 lines · Quentin Monnet
2018-06-22 11:24 UTC+0100 ~ Okash Khawaja <[email protected]>
Quoted 6 lines · Quentin Monnet
Quoted 1 line
Hi Okash,
hi and sorry about delay in responding. the email got routed to incorrect folder.
Quoted 2 lines
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
[...]
Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Hexadecimal values, without quotes, are not valid JSON. Please stick to decimal values.
ah sorry, i used a buggy json validator. this should be a quick fix. which would be better: pointers be output hex strings or integers?
I would go for integers. Although this is harder to read for humans, it is easier to process for machines, which remain the primary targets for JSON output.
+1
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
In reply to Okash KhawajaQuoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14
[{
"key": 0
},{
"value": {
"m": 1,
"n": 2,
"o": "c",
"p": [15,16,17,18,15,16,17,18
],
"q": [[25,26,27,28,25,26,27,28
],[35,36,37,38,35,36,37,38
],[45,46,47,48,45,46,47,48
],[55,56,57,58,55,56,57,58
]
],
"r": 1,
"s": 0x7ffff6f70568,
"t": {
"x": 5,
"y": 10Quoted 8 lines
},
"u": 100,
"v": 20,
"w1": 0x7,
"w2": 0x3
}
}
]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{
"key":["0x00","0x00","0x00","0x00",
],
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
]
}This format must remain, you may only augment it with new fields. E.g.:
{
"key":["0x00","0x00","0x00","0x00",
],
"key_struct":{
"index":0
},
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"value_struct":{
"src_ip":2,
"dst_ip:0
}
}The name XYZ_struct may not be the best, perhaps you can come up with a better one?
Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 46 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
Quoted 13 lines
{
"key":["0x00","0x00","0x00","0x00",
],
"key_struct":{
"index":0
},
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"value_struct":{
"src_ip":2,
"dst_ip:0
}
}I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
The existing json can be kept as is.
Quoted 10 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 60 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
Quoted 13 lines
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 }, "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2, "dst_ip:0 } }
Quoted 3 lines
I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 6 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or key_fields and value_fields?
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 73 lines · Jakub Kicinski
Quoted 60 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
Quoted 13 lines
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 }, "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2, "dst_ip:0 } }Quoted 3 lines
I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?
I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json. key_fields and value_fields may hint it has >1 fields though. May be "formatted_key" and "formatted_value"?
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 90 lines · Martin KaFai Lau
Quoted 73 lines · Jakub Kicinski
Quoted 60 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
Quoted 13 lines
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 }, "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2, "dst_ip:0 } }Quoted 3 lines
I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(
Quoted 5 lines
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json.
key_fieldsandvalue_fieldsmay hint it has >1 fields though. May be "formatted_key" and "formatted_value"?
SGTM! Or even maybe as a "formatted" object?:
{
"key":["0x00","0x00","0x00","0x00",
],
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"formatted":{
"key":{
"index":0
},
"value":{
"src_ip":2,
"dst_ip:0
}
}
} Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 48 lines · Jakub Kicinski
Quoted 47 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 45 lines · Martin KaFai Lau
Quoted 44 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },
Got a few questions.
When we support hashtab later, the key could be int
but reusing the name as "index" is weird.
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "key_struct" will look like?
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,
If for the same map the user changes the "src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
struct five_tuples {
- int src_ip;
+ int src_ip[4];
/* ... */
};
Quoted 51 lines
Quoted 31 lines
Quoted 15 lines
Quoted 6 lines
Quoted 3 lines
"dst_ip:0 } }I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(Quoted 5 lines
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json.
key_fieldsandvalue_fieldsmay hint it has >1 fields though. May be "formatted_key" and "formatted_value"?SGTM! Or even maybe as a "formatted" object?:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "formatted":{ "key":{ "index":0 }, "value":{(quoted diff) +0
-0"src_ip":2, "dst_ip:0
Quoted 2 lines
} }
hmm... that is an extra indentation (keep in mind that the "value" could already have a few nested structs which itself consumes a few indentations) but I guess adding another one may be ok-ish.
Quoted 12 lines
}
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 131 lines · Martin KaFai Lau
Quoted 48 lines · Jakub Kicinski
Quoted 47 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 45 lines · Martin KaFai Lau
Quoted 44 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },Got a few questions.
When we support hashtab later, the key could be int
but reusing the name as "index" is weird.
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "key_struct" will look like?Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Quoted 51 lines
Quoted 31 lines
Quoted 15 lines
Quoted 6 lines
Quoted 3 lines
"dst_ip:0 } }I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(Quoted 5 lines
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json.
key_fieldsandvalue_fieldsmay hint it has >1 fields though. May be "formatted_key" and "formatted_value"?SGTM! Or even maybe as a "formatted" object?:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "formatted":{ "key":{ "index":0 }, "value":{(quoted diff) +0
-0"src_ip":2, "dst_ip:0Quoted 2 lines
} }
Quoted 16 lines
hmm... that is an extra indentation (keep in mind that the "value" could already have a few nested structs which itself consumes a few indentations) but I guess adding another one may be ok-ish.
Quoted 12 lines
}
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Hi,
While I agree on the point of backward compatibility, I think printing two overlapping pieces of information side-by-side will make the interface less clear. Having separate outputs for the two will keep the interface clear and readable.
Is there a major downside to adding a new flag for BTF output?
Thanks,
Okash
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 11 lines · Okash Khawaja
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Quoted 8 lines
Hi,
While I agree on the point of backward compatibility, I think printing two overlapping pieces of information side-by-side will make the interface less clear. Having separate outputs for the two will keep the interface clear and readable.
Is there a major downside to adding a new flag for BTF output?
plain output and JSON should be more or less equivalent, just formatted differently. If we hide the BTF-ed JSON under some flag most users will just run bpftool -jp map dump id X, see BTF is not there and move on, without ever looking into the man page to discover there is a magic switch...
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
In reply to Martin KaFai LauQuoted 57 lines · Martin KaFai Lau
Quoted 48 lines · Jakub Kicinski
Quoted 47 lines · Martin KaFai Lau
Quoted 46 lines · Jakub Kicinski
Quoted 45 lines · Martin KaFai Lau
Quoted 44 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },
Quoted 4 lines
Got a few questions.
When we support hashtab later, the key could be int but reusing the name as "index" is weird.
Ugh, yes, naturally. I just typed that out without thinking, so for array maps there is usually no BTF info?... For hashes obviously we should just use the BTF, I'm not sure we should format indexes for arrays nicely or not :S
Quoted 2 lines
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "
key_struct" will look like?
Hm. I think in my mind it has only been a struct but that's not true :S So the struct in the name makes very limited sense now.
Should we do:
"formatted" : {
"value" : XXX
}
Where
XXX == plain int for integers, e.g. "value":0
XXX == array for arrays, e.g. "value":[1,2,3,4]
XXX == object for objects, e.g. "value":{"field":XXX, "field2":XXX}
Quoted 11 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];
Quoted 2 lines
/* ... */
};
Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 53 lines
Quoted 51 lines
Quoted 31 lines
Quoted 15 lines
Quoted 6 lines
Quoted 3 lines
"dst_ip:0 } }I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(Quoted 5 lines
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json.
key_fieldsandvalue_fieldsmay hint it has >1 fields though. May be "formatted_key" and "formatted_value"?SGTM! Or even maybe as a "formatted" object?:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "formatted":{ "key":{ "index":0 }, "value":{(quoted diff) +0
-0"src_ip":2, "dst_ip:0Quoted 2 lines
} }
Quoted 3 lines
hmm... that is an extra indentation (keep in mind that the "value" could already have a few nested structs which itself consumes a few indentations) but I guess adding another one may be ok-ish.
I'm not fussed about this, whatever JSON writer does by default is fine with me, really.
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 58 lines · Jakub Kicinski
Quoted 57 lines · Martin KaFai Lau
Quoted 56 lines · Jakub Kicinski
Quoted 55 lines · Martin KaFai Lau
Quoted 54 lines · Jakub Kicinski
Quoted 53 lines · Martin KaFai Lau
Quoted 52 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },
Quoted 7 lines
Quoted 4 lines
Got a few questions.
When we support hashtab later, the key could be int but reusing the name as "index" is weird.
Ugh, yes, naturally. I just typed that out without thinking, so for array maps there is usually no BTF info?... For hashes obviously we
The key of array map also has BTF info which must be an int.
Quoted 14 lines
should just use the BTF, I'm not sure we should format indexes for arrays nicely or not :S
Quoted 2 lines
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "
key_struct" will look like?Hm. I think in my mind it has only been a struct but that's not true :S So the struct in the name makes very limited sense now.
Should we do:
"formatted" : {
"value" : XXX
}
Quoted 4 lines
Where
XXX == plain int for integers, e.g. "value":0
XXX == array for arrays, e.g. "value":[1,2,3,4]
XXX == object for objects, e.g. "value":{"field":XXX, "field2":XXX}
It is exactly how this patch is using json's {}, [] and int. ;) but other than that, it does not have to be json.
In the next spin, lets stop calling this output json to avoid wrong user's expection and I also don't want the future readability improvements to be limited by that. Lets call it something like plain text output with BTF.
How about:
When "bpftool map dump id 1" is used, it will print the BTF plaintext output if a map has BTF available. If not, it will print the existing plaintext output. That should solve the concern about the user may not know BTF is available.
This ascii output is for human. The script should not parse the ascii output because it is silly not to use the binary ABI (like what this patch is using) which does not suffer backward compat issue.
The existing "-j" can be used to dump the map's binary data
for remote debugging purpose. The BTF is in one of the elf section of the bpf_prog.o.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,
Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
Quoted 72 lines
Quoted 54 lines
Quoted 52 lines
Quoted 32 lines
Quoted 16 lines
Quoted 7 lines
Quoted 4 lines
"dst_ip:0 } }I am not sure how useful to have both "key|value" and "(key|value)_struct" while most people would prefer "key_struct"/"value_struct" if it is available.
Agreed, it's not that useful, especially with the string-hex debacle :( It's just about the backwards compat.
Quoted 5 lines
How about introducing a new option, like "-b", to print the map with BTF (if available) such that it won't break the existing one (-j or -p) while the "-b" output can keep using the "key" and "value".
Quoted 8 lines
Quoted 1 line
The existing json can be kept as is.
That was my knee jerk reaction too, but on reflection it doesn't sound that great. We expect people with new-enough bpftool to use btf, so it should be available in the default output, without hiding it behind a switch. We could add a switch to hide the old output, but that doesn't give us back the names... What about Key and Value or k and v? Or
key_fieldsandvalue_fields?I thought the current default output is "plain" ;) Having said that, yes, the btf is currently printed in json.
Ideally, the default json output should do what most people want:
print btf and btf only (if it is available).
but I don't see a way out without new option if we need to be backward compat :(Quoted 5 lines
Agree that showing the btf in the existing json output will be useful (e.g. to hint people that BTF is available). If btf is showing in old json, also agree that the names should be the same with the new json.
key_fieldsandvalue_fieldsmay hint it has >1 fields though. May be "formatted_key" and "formatted_value"?SGTM! Or even maybe as a "formatted" object?:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "formatted":{ "key":{ "index":0 }, "value":{(quoted diff) +0
-0"src_ip":2, "dst_ip:0Quoted 2 lines
} }Quoted 3 lines
hmm... that is an extra indentation (keep in mind that the "value" could already have a few nested structs which itself consumes a few indentations) but I guess adding another one may be ok-ish.
I'm not fussed about this, whatever JSON writer does by default is fine with me, really.
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
The name
XYZ_structmay not be the best, perhaps you can come up with a better one?Does that make sense? Am I missing what you're doing here?
One process note - please make sure you run checkpatch.pl --strict on bpftool patches before posting.
Thanks for working on this!
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 59 lines · Martin KaFai Lau
Quoted 58 lines · Jakub Kicinski
Quoted 57 lines · Martin KaFai Lau
Quoted 56 lines · Jakub Kicinski
Quoted 55 lines · Martin KaFai Lau
Quoted 54 lines · Jakub Kicinski
Quoted 53 lines · Martin KaFai Lau
Quoted 52 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },
Quoted 8 lines
Quoted 7 lines
Quoted 4 lines
Got a few questions.
When we support hashtab later, the key could be int but reusing the name as "index" is weird.
Ugh, yes, naturally. I just typed that out without thinking, so for array maps there is usually no BTF info?... For hashes obviously we
The key of array map also has BTF info which must be an int.
Perfect.
Quoted 14 lines
Quoted 14 lines
should just use the BTF, I'm not sure we should format indexes for arrays nicely or not :S
Quoted 2 lines
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "
key_struct" will look like?Hm. I think in my mind it has only been a struct but that's not true :S So the struct in the name makes very limited sense now.
Should we do:
"formatted" : {
"value" : XXX
}
Quoted 5 lines
Quoted 4 lines
Where
XXX == plain int for integers, e.g. "value":0
XXX == array for arrays, e.g. "value":[1,2,3,4]
XXX == object for objects, e.g. "value":{"field":XXX, "field2":XXX}It is exactly how this patch is using json's {}, [] and int. ;)
Great, then just wrap that in a "formatted" object instead of redefining fields and we're good?
Quoted 5 lines
but other than that, it does not have to be json.
In the next spin, lets stop calling this output json to avoid wrong user's expection and I also don't want the future readability improvements to be limited by that. Lets call it something like plain text output with BTF.
I don't understand. We are discussing JSON output here. The example we are commenting on is output of:
$ sudo bpftool map dump -p id 14
That -p means JSON. Nobody objects to plain test output changes. I actually didn't realize you haven't implemented plain text in this series, we should have both.
Quoted 9 lines
How about:
When "bpftool map dump id 1" is used, it will print the BTF plaintext output if a map has BTF available. If not, it will print the existing plaintext output. That should solve the concern about the user may not know BTF is available.This ascii output is for human. The script should not parse the ascii output because it is silly not to use the binary ABI (like what this patch is using) which does not suffer backward compat issue.
What binary ABI? I'm also not 100% sure what this patch is doing as it adds bunch of code in new files that never gets called:
tools/bpf/bpftool/btf_dumper.c | 247 +++++++++++++++++++++++++++++++++++++++++ tools/bpf/bpftool/btf_dumper.h | 18 ++ 2 files changed, 265 insertions(+)
Quoted 20 lines
The existing "-j" can be used to dump the map's binary data for remote debugging purpose. The BTF is in one of the elf section of the bpf_prog.o.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 5 lines · Jakub Kicinski
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
Ohh, maybe that's the misunderstanding, you only implemented JSON so you wanted it to be as readable and clean as possible. Hence the hex output and cutting out the old cruft! That perspective makes sense! But I think we should keep JSON for machines (but including BTF formatted values) and let's make the plain text output nice and clean, agreed.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 7 lines · Jakub Kicinski
Quoted 5 lines · Jakub Kicinski
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
Quoted 6 lines
Ohh, maybe that's the misunderstanding, you only implemented JSON so you wanted it to be as readable and clean as possible. Hence the hex output and cutting out the old cruft! That perspective makes sense! But I think we should keep JSON for machines (but including BTF formatted values) and let's make the plain text output nice and clean, agreed.
Right, it is what my earlier comment meant on "this ascii output is for human". We merely call it json because we are reusing the json's meaning on {}, [] and int since it fits nicely on what we want to achieve, readability. Other than that, it does not have to follow other json's requirements.
We can call it whatever except json to avoid wrong
user expectation. Putting it under "-j"/"-p" was a mistake. Hence, I said this patch belongs to the 'plaintext" output.
If we really needed a similar output under "-j" or "-p", things had to be changed and it belongs to a separate patch.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 14 lines · Martin KaFai Lau
Quoted 7 lines · Jakub Kicinski
Quoted 5 lines · Jakub Kicinski
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
Quoted 6 lines
Ohh, maybe that's the misunderstanding, you only implemented JSON so you wanted it to be as readable and clean as possible. Hence the hex output and cutting out the old cruft! That perspective makes sense! But I think we should keep JSON for machines (but including BTF formatted values) and let's make the plain text output nice and clean, agreed.
Quoted 8 lines
Right, it is what my earlier comment meant on "this ascii output is for human". We merely call it json because we are reusing the json's meaning on {}, [] and int since it fits nicely on what we want to achieve, readability. Other than that, it does not have to follow other json's requirements.
We can call it whatever except json to avoid wrong
user expectation. Putting it under "-j"/"-p" was a mistake. Hence, I said this patch belongs to the 'plaintext" output.
Yes, that were the confusion came from, right. I'm personally not sold on JSON as "human readable" but I'm very far from a UI guru so up to you :)
I think it may be good to intentionally do non-JSON things, like use hex integers, don't put quotes around strings, or always add a comma after value, so people can't use it as JSON even if they try.
Basic printer is trivial to write, I'm concerned that the reuse of JSON writer to create a user-readable output will bite us on the posterior later on...
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 24 lines · Jakub Kicinski
Quoted 14 lines · Martin KaFai Lau
Quoted 7 lines · Jakub Kicinski
Quoted 5 lines · Jakub Kicinski
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
Quoted 6 lines
Ohh, maybe that's the misunderstanding, you only implemented JSON so you wanted it to be as readable and clean as possible. Hence the hex output and cutting out the old cruft! That perspective makes sense! But I think we should keep JSON for machines (but including BTF formatted values) and let's make the plain text output nice and clean, agreed.
Quoted 8 lines
Right, it is what my earlier comment meant on "this ascii output is for human". We merely call it json because we are reusing the json's meaning on {}, [] and int since it fits nicely on what we want to achieve, readability. Other than that, it does not have to follow other json's requirements.
We can call it whatever except json to avoid wrong
user expectation. Putting it under "-j"/"-p" was a mistake. Hence, I said this patch belongs to the 'plaintext" output.
Quoted 3 lines
Yes, that were the confusion came from, right. I'm personally not sold on JSON as "human readable" but I'm very far from a UI guru so up to you :)
We have thought of a few options but nothing else shine in term of saving the verbosity on spelling out the word like "it is a struct" , "it is an array"... and at the same time trying to use some short form that people are already familiar with.
Quoted 4 lines
I think it may be good to intentionally do non-JSON things, like use hex integers, don't put quotes around strings, or always add a comma after value, so people can't use it as JSON even if they try.
Good point.
Quoted 4 lines
Basic printer is trivial to write, I'm concerned that the reuse of JSON writer to create a user-readable output will bite us on the posterior later on...
If we can think of some better way later, we can also provide another plaintext output. I don't want to over design it at this point. Let see how it goes.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
In reply to Jakub KicinskiOn Fri, Jun 22, 2018 at 02:27:43PM -0700, Jakub Kicinski wrote:
[...]
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
should just use the BTF, I'm not sure we should format indexes for arrays nicely or not :S
Quoted 2 lines
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "
key_struct" will look like?Hm. I think in my mind it has only been a struct but that's not true :S So the struct in the name makes very limited sense now.
Should we do:
"formatted" : {
"value" : XXX
}
Quoted 8 lines
Quoted 5 lines
Quoted 4 lines
Where
XXX == plain int for integers, e.g. "value":0
XXX == array for arrays, e.g. "value":[1,2,3,4]
XXX == object for objects, e.g. "value":{"field":XXX, "field2":XXX}It is exactly how this patch is using json's {}, [] and int. ;)
Great, then just wrap that in a "formatted" object instead of redefining fields and we're good?
Please don't let two formats of same data in a single ouput. Overloading same output with multple formats suggests a confused design, IMO. It will confuse users too.
Quoted 22 lines
Quoted 6 lines
but other than that, it does not have to be json.
In the next spin, lets stop calling this output json to avoid wrong user's expection and I also don't want the future readability improvements to be limited by that. Lets call it something like plain text output with BTF.I don't understand. We are discussing JSON output here. The example we are commenting on is output of:
$ sudo bpftool map dump -p id 14That -p means JSON. Nobody objects to plain test output changes. I actually didn't realize you haven't implemented plain text in this series, we should have both.
Quoted 6 lines
How about:
When "bpftool map dump id 1" is used, it will print the BTF plaintext output if a map has BTF available. If not, it will print the existing plaintext output. That should solve the concern about the user may not know BTF is available.
Quoted 5 lines
Quoted 3 lines
This ascii output is for human. The script should not parse the ascii output because it is silly not to use the binary ABI (like what this patch is using) which does not suffer backward compat issue.
What binary ABI?
Reading binary data and linking it with BTF information.
Quoted 2 lines
I'm also not 100% sure what this patch is doing as it adds bunch of code in new files that never gets called:
Please take a look at the patch then :) Code in these files does get called.
We seem to be conflating two different - and conflicting - intents here. First is progam-readability of the output and second is human readability of the output. I believe both are important. Let's leave existing output for programs to read. That way we can try to keep it backward compatible as well as JSON compatible. Let's dedicate the new BTF format for humans to read. That way, we don't have to worry about backward compatibility or JSON compatibility.
Let me know if this is not clear. If we agree on above then I think we can move towards a solution.
Thanks,
Okash
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
In reply to Jakub KicinskiQuoted 115 lines · Jakub Kicinski
Quoted 59 lines · Martin KaFai Lau
Quoted 58 lines · Jakub Kicinski
Quoted 57 lines · Martin KaFai Lau
Quoted 56 lines · Jakub Kicinski
Quoted 55 lines · Martin KaFai Lau
Quoted 54 lines · Jakub Kicinski
Quoted 53 lines · Martin KaFai Lau
Quoted 52 lines · Jakub Kicinski
Quoted 21 lines · Okash Khawaja
$ sudo bpftool map dump -p id 14 [{ "key": 0 },{ "value": { "m": 1, "n": 2, "o": "c", "p": [15,16,17,18,15,16,17,18 ], "q": [[25,26,27,28,25,26,27,28 ],[35,36,37,38,35,36,37,38 ],[45,46,47,48,45,46,47,48 ],[55,56,57,58,55,56,57,58 ] ], "r": 1, "s": 0x7ffff6f70568, "t": { "x": 5, "y": 10Quoted 8 lines
}, "u": 100, "v": 20, "w1": 0x7, "w2": 0x3 } } ]I don't think this format is okay, JSON output is an API you shouldn't break. You can change the non-JSON output whatever way you like, but JSON must remain backwards compatible.
The dump today has object per entry, e.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ] }This format must remain, you may only augment it with new fields. E.g.:
{ "key":["0x00","0x00","0x00","0x00", ], "key_struct":{ "index":0 },Quoted 8 lines
Quoted 7 lines
Quoted 4 lines
Got a few questions.
When we support hashtab later, the key could be int but reusing the name as "index" is weird.
Ugh, yes, naturally. I just typed that out without thinking, so for array maps there is usually no BTF info?... For hashes obviously we
The key of array map also has BTF info which must be an int.
Perfect.
Quoted 14 lines
Quoted 14 lines
should just use the BTF, I'm not sure we should format indexes for arrays nicely or not :S
Quoted 2 lines
The key could also be a struct (e.g. a struct to describe ip:port). Can you suggest how the "
key_struct" will look like?Hm. I think in my mind it has only been a struct but that's not true :S So the struct in the name makes very limited sense now.
Should we do:
"formatted" : {
"value" : XXX
}Quoted 5 lines
Quoted 4 lines
Where
XXX == plain int for integers, e.g. "value":0
XXX == array for arrays, e.g. "value":[1,2,3,4]
XXX == object for objects, e.g. "value":{"field":XXX, "field2":XXX}It is exactly how this patch is using json's {}, [] and int. ;)
Great, then just wrap that in a "formatted" object instead of redefining fields and we're good?
Quoted 5 lines
but other than that, it does not have to be json.
In the next spin, lets stop calling this output json to avoid wrong user's expection and I also don't want the future readability improvements to be limited by that. Lets call it something like plain text output with BTF.I don't understand. We are discussing JSON output here. The example we are commenting on is output of:
$ sudo bpftool map dump -p id 14That -p means JSON. Nobody objects to plain test output changes. I actually didn't realize you haven't implemented plain text in this series, we should have both.
Quoted 6 lines
How about:
When "bpftool map dump id 1" is used, it will print the BTF plaintext output if a map has BTF available. If not, it will print the existing plaintext output. That should solve the concern about the user may not know BTF is available.
Quoted 6 lines
Quoted 3 lines
This ascii output is for human. The script should not parse the ascii output because it is silly not to use the binary ABI (like what this patch is using) which does not suffer backward compat issue.
What binary ABI? I'm also not 100% sure what this patch is doing as it adds bunch of code in new files that never gets called:
I meant the BTF format, the new kernel API to get BTF and how it is used on map data are stable.
Yes, there is new codes to get and consume the new BTF format but so does any new kernel API. and it should not drive everybody to parse ascii.
Quoted 33 lines
tools/bpf/bpftool/btf_dumper.c | 247 +++++++++++++++++++++++++++++++++++++++++ tools/bpf/bpftool/btf_dumper.h | 18 ++ 2 files changed, 265 insertions(+)Quoted 20 lines
The existing "-j" can be used to dump the map's binary data for remote debugging purpose. The BTF is in one of the elf section of the bpf_prog.o.
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Quoted 3 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Quoted 3 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = []
for i in strtab:
inttab.append(int(i, 16))
ba = bytearray(inttab)
if len(strtab) == 4:
fmt = "I"
elif len(strtab) == 8:
fmt = "Q"
else:
raise Exception("String array of len %d can't be unpacked to an int" %
(len(strtab)))
return struct.unpack(fmt, ba)[0]def convert(elems, idx):
val = []
for i in range(3):
part = elems[idx]["value"][i * length:(i + 1) * length]
val.append(str2int(part))
return valWith BTF it would be:
elems[idx]["formatted"]["value"]
Which is fairly awesome.
Quoted 9 lines
Quoted 3 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 76 lines · Jakub Kicinski
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = [] for i in strtab: inttab.append(int(i, 16)) ba = bytearray(inttab) if len(strtab) == 4: fmt = "I" elif len(strtab) == 8: fmt = "Q" else: raise Exception("String array of len %d can't be unpacked to an int" % (len(strtab))) return struct.unpack(fmt, ba)[0]def
convert(elems, idx):val = [] for i in range(3): part = elems[idx]["value"][i * length:(i + 1) * length] val.append(str2int(part)) return val
Quoted 5 lines
With BTF it would be:
elems[idx]["formatted"]["value"]Which is fairly awesome.
Thanks for the example. Agree that with BTF, things are easier in general.
btw, what more awesome is,
#> bpftool map find id 100 key 1
{
"counter_x": 1,
"counter_y": 10
}Quoted 27 lines
Quoted 10 lines
Quoted 4 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 82 lines · Martin KaFai Lau
Quoted 76 lines · Jakub Kicinski
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00" ], "value_struct":{ "src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = [] for i in strtab: inttab.append(int(i, 16)) ba = bytearray(inttab) if len(strtab) == 4: fmt = "I" elif len(strtab) == 8: fmt = "Q" else: raise Exception("String array of len %d can't be unpacked to an int" % (len(strtab))) return struct.unpack(fmt, ba)[0]def
convert(elems, idx):val = [] for i in range(3): part = elems[idx]["value"][i * length:(i + 1) * length] val.append(str2int(part)) return valQuoted 5 lines
With BTF it would be:
elems[idx]["formatted"]["value"]Which is fairly awesome.
Quoted 36 lines
Thanks for the example. Agree that with BTF, things are easier in general.
btw, what more awesome is,
#> bpftool map find id 100 key 1{ "counter_x": 1, "counter_y": 10 }Quoted 27 lines
Quoted 10 lines
Quoted 4 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.
hi,
so i guess following is what we want:
1. a "formatted" object nested inside -p and -j switches for bpf map dump. this will be JSON and backward compatible
2. an output for humans - which is like the current output. this will not be JSON. this won't have to be backward compatible. this output will be shown when neither of -j and -p are supplied and btf info is available.
i can update the patches to v2 which covers 2 above + all other comments on the patchset. later we can follow up with a patch for 1.
thanks for valuable feedback :)
okash
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 131 lines · Okash Khawaja
Quoted 82 lines · Martin KaFai Lau
Quoted 76 lines · Jakub Kicinski
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"value_struct":{
"src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = [] for i in strtab: inttab.append(int(i, 16)) ba = bytearray(inttab) if len(strtab) == 4: fmt = "I" elif len(strtab) == 8: fmt = "Q" else: raise Exception("String array of len %d can't be unpacked to an int" % (len(strtab))) return struct.unpack(fmt, ba)[0]def
convert(elems, idx):val = [] for i in range(3): part = elems[idx]["value"][i * length:(i + 1) * length] val.append(str2int(part)) return valQuoted 5 lines
With BTF it would be:
elems[idx]["formatted"]["value"]Which is fairly awesome.
Quoted 36 lines
Thanks for the example. Agree that with BTF, things are easier in general.
btw, what more awesome is,
#> bpftool map find id 100 key 1{ "counter_x": 1, "counter_y": 10 }Quoted 27 lines
Quoted 10 lines
Quoted 4 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.hi,
so i guess following is what we want:
1. a "formatted" object nested inside -p and -j switches for bpf map dump. this will be JSON and backward compatible
2. an output for humans - which is like the current output. this will not be JSON. this won't have to be backward compatible. this output will be shown when neither of -j and -p are supplied and btf info is available.
Quoted 2 lines
i can update the patches to v2 which covers 2 above + all other comments on the patchset. later we can follow up with a patch for 1.
Please do both at the same time. I've learnt not to trust people when they say things like "we can follow up with xyz" :( In my experience it _always_ backfires.
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 135 lines · Jakub Kicinski
Quoted 131 lines · Okash Khawaja
Quoted 82 lines · Martin KaFai Lau
Quoted 76 lines · Jakub Kicinski
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"value_struct":{
"src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = [] for i in strtab: inttab.append(int(i, 16)) ba = bytearray(inttab) if len(strtab) == 4: fmt = "I" elif len(strtab) == 8: fmt = "Q" else: raise Exception("String array of len %d can't be unpacked to an int" % (len(strtab))) return struct.unpack(fmt, ba)[0]def
convert(elems, idx):val = [] for i in range(3): part = elems[idx]["value"][i * length:(i + 1) * length] val.append(str2int(part)) return valQuoted 5 lines
With BTF it would be:
elems[idx]["formatted"]["value"]Which is fairly awesome.
Quoted 36 lines
Thanks for the example. Agree that with BTF, things are easier in general.
btw, what more awesome is,
#> bpftool map find id 100 key 1{ "counter_x": 1, "counter_y": 10 }Quoted 27 lines
Quoted 10 lines
Quoted 4 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.hi,
so i guess following is what we want:
1. a "formatted" object nested inside -p and -j switches for bpf map dump. this will be JSON and backward compatible
2. an output for humans - which is like the current output. this will not be JSON. this won't have to be backward compatible. this output will be shown when neither of -j and -p are supplied and btf info is available.Quoted 2 lines
i can update the patches to v2 which covers 2 above + all other comments on the patchset. later we can follow up with a patch for 1.
Quoted 6 lines
Please do both at the same time. I've learnt not to trust people when they say things like "we can follow up with xyz" :( In my experience it _always_ backfires.
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 146 lines · Martin KaFai Lau
Quoted 135 lines · Jakub Kicinski
Quoted 131 lines · Okash Khawaja
Quoted 82 lines · Martin KaFai Lau
Quoted 76 lines · Jakub Kicinski
Quoted 29 lines · Martin KaFai Lau
Quoted 24 lines
Quoted 16 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
"value": ["0x02","0x00","0x00","0x00","0x00","0x00","0x00","0x00"
],
"value_struct":{
"src_ip":2,Quoted 12 lines
Quoted 9 lines
If for the same map the user changes the "
src_ip" to an array of int[4] later (e.g. to support ipv6), it will become "src_ip": [1, 2, 3, 4]. Is it breaking backward compat?
i.e.
structfive_tuples{(quoted diff) +1
-1- int src_ip; + int src_ip[4];/* ... */
};Well, it is breaking backward compat, but it's the program doing it, not bpftool :) BTF changes so does the output.
Quoted 3 lines
As we see, the key/value's btf-output is inherently not backward compat. Hence, "-j" and "-p" will stay as is. The whole existing json will be backward compat instead of only partly backward compat.
No. There is a difference between user of a facility changing their input and kernel/libraries providing different output in response to that, and the libraries suddenly changing the output on their own.
Quoted 5 lines
Your example is like saying if user started using IPv6 addresses instead of IPv4 the netlink attributes in dumps will be different so kernel didn't keep backwards compat. While what you're doing is more equivalent to dropping support for old ioctl interfaces because there is a better mechanism now.
Quoted 6 lines
Sorry, I don't follow this. I don't see netlink suffer json issue like the one on "key" and "value".
All I can grasp is, the json should normally be backward compat but now we are saying anything added by btf-output is an exception because the script parsing it will treat it differently than "key" and "value"
Backward compatibility means that if I run *the same* program against different kernels/libraries it continues to work. If someone decides to upgrade their program to work with IPv6 (which was your example) obviously there is no way system as a whole will look 1:1 the same.
Quoted 4 lines
Quoted 2 lines
BTF in JSON is very useful, and will help people who writes simple orchestration/scripts based on bpftool *a* *lot*. I really appreciate
Can you share what the script will do? I want to understand why it cannot directly use the BTF format and the map data.
Think about a python script which wants to read a counter in a map. Right now it would have to get the BTF, find out which bytes are the counter, then convert the bytes into a larger int. With JSON BTF if just does entry["formatted"]["value"]["counter"].
Real life example from my test code (conversion of 3 element counter array):
def str2int(strtab):
inttab = [] for i in strtab: inttab.append(int(i, 16)) ba = bytearray(inttab) if len(strtab) == 4: fmt = "I" elif len(strtab) == 8: fmt = "Q" else: raise Exception("String array of len %d can't be unpacked to an int" % (len(strtab))) return struct.unpack(fmt, ba)[0]def
convert(elems, idx):val = [] for i in range(3): part = elems[idx]["value"][i * length:(i + 1) * length] val.append(str2int(part)) return valQuoted 5 lines
With BTF it would be:
elems[idx]["formatted"]["value"]Which is fairly awesome.
Quoted 36 lines
Thanks for the example. Agree that with BTF, things are easier in general.
btw, what more awesome is,
#> bpftool map find id 100 key 1{ "counter_x": 1, "counter_y": 10 }Quoted 27 lines
Quoted 10 lines
Quoted 4 lines
this addition to bpftool and will start using it myself as soon as it lands. I'm not sure why the reluctance to slightly change the output format?
The initial change argument is because the json has to be backward compat.
Then we show that btf-output is inherently not backward compat, so printing it in json does not make sense at all.
However, now it is saying part of it does not have to be backward compat.
Compatibility of "formatted" member is defined as -> fields broken out according to BTF. So it is backward compatible. The definition of "value" member is -> an array of unfortunately formatted array of ugly hex strings :(
Quoted 10 lines
I am fine putting it under "formatted" for "-j" or "-p" if that is the case, other than the double output is still confusing. Lets wait for Okash's input.
At the same time, the same output will be used as the default plaintext output when BTF is available. Then the plaintext BTF output will not be limited by the json restrictions when we want
to improve human readability later. Apparently, the improvements on plaintext will not be always applicable
to json output.hi,
so i guess following is what we want:
1. a "formatted" object nested inside -p and -j switches for bpf map dump. this will be JSON and backward compatible
2. an output for humans - which is like the current output. this will not be JSON. this won't have to be backward compatible. this output will be shown when neither of -j and -p are supplied and btf info is available.Quoted 2 lines
i can update the patches to v2 which covers 2 above + all other comments on the patchset. later we can follow up with a patch for 1.
Quoted 6 lines
Please do both at the same time. I've learnt not to trust people when they say things like "we can follow up with xyz" :( In my experience it _always_ backfires.
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Quoted 6 lines
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Parity between JSON and plain text output is non negotiable.
Daniel Borkmann
maintainerRe: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 2 lines · Jakub Kicinski
Quoted 1 line · Martin KaFai Lau
On Tue, Jun 26, 2018 at 01:31:33PM -0700, Jakub Kicinski wrote:
[...]
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Quoted 8 lines
Quoted 6 lines
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Parity between JSON and plain text output is non negotiable.
Longish discussion and some confusion in this thread. :-) First of all thanks a lot for working on it, very useful! My $0.02 on it is that so far great care has been taken in bpftool to indeed have feature parity between JSON and plain text, so it would be highly desirable to keep continuing this practice if the consensus is that it indeed is feasible and makes sense wrt BTF data. There has been mentioned that given BTF data can be dynamic depending on what the user loads via bpf(2) so a potential JSON output may look different/break each time anyway. This however could all be embedded under a container object that has a fixed key like 'formatted' where tools like jq(1) can query into it. I think this would be fine since the rest of the (non-dynamic) output is still retained as-is and then wouldn't confuse or collide with existing users, and anyone programmatically parsing deeper into the BTF data under such JSON container object needs to have awareness of what specific data it wants to query from it; so there's no conflict wrt breaking anything here. Imho, both outputs would be very valuable.
Thanks,
Daniel
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 17 lines · Daniel Borkmann
Quoted 2 lines · Jakub Kicinski
Quoted 1 line · Martin KaFai Lau
On Tue, Jun 26, 2018 at 01:31:33PM -0700, Jakub Kicinski wrote:
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 2 lines
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Quoted 4 lines
Quoted 1 line
Parity between JSON and plain text output is non negotiable.
Longish discussion and some confusion in this thread. :-) First of all thanks a lot for working on it, very useful!
Thanks :)
Quoted 7 lines
My $0.02 on it is that so far
great care has been taken in bpftool to indeed have feature parity between JSON and plain text, so it would be highly desirable to keep continuing this practice if the consensus is that it indeed is feasible and makes sense wrt BTF data. There has been mentioned that given BTF data can be dynamic depending on what the user loads via bpf(2) so a potential JSON output may look different/break each time anyway. This however could all be
Quoted 8 lines
embedded under a container object that has a fixed key like 'formatted' where tools like jq(1) can query into it. I think this would be fine since the rest of the (non-dynamic) output is still retained as-is and then wouldn't confuse or collide with existing users, and anyone programmatically parsing deeper into the BTF data under such JSON container object needs to have awareness of what specific data it wants to query from it; so there's no conflict wrt breaking anything here. Imho, both outputs would be very valuable.
Okay I can add "formatted" object under json output.
One thing to note here is that the fixed output will change if the map itself changes. So someone writing a program that consumes that fixed output will have to account for his program breaking in future, thus breaking backward compatibility anyway as far as the developer is concerned :)
I will go ahead with work on "formatted" object.
Thanks,
Okash
Quoted 3 lines
Thanks,
Daniel
Daniel Borkmann
maintainerRe: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 39 lines · Okash Khawaja
Quoted 17 lines · Daniel Borkmann
Quoted 2 lines · Jakub Kicinski
Quoted 1 line · Martin KaFai Lau
On Tue, Jun 26, 2018 at 01:31:33PM -0700, Jakub Kicinski wrote:
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 2 lines
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Quoted 4 lines
Quoted 1 line
Parity between JSON and plain text output is non negotiable.
Longish discussion and some confusion in this thread. :-) First of all thanks a lot for working on it, very useful!
Thanks :)
Quoted 7 lines
My $0.02 on it is that so far
great care has been taken in bpftool to indeed have feature parity between JSON and plain text, so it would be highly desirable to keep continuing this practice if the consensus is that it indeed is feasible and makes sense wrt BTF data. There has been mentioned that given BTF data can be dynamic depending on what the user loads via bpf(2) so a potential JSON output may look different/break each time anyway. This however could all beQuoted 8 lines
embedded under a container object that has a fixed key like 'formatted' where tools like jq(1) can query into it. I think this would be fine since the rest of the (non-dynamic) output is still retained as-is and then wouldn't confuse or collide with existing users, and anyone programmatically parsing deeper into the BTF data under such JSON container object needs to have awareness of what specific data it wants to query from it; so there's no conflict wrt breaking anything here. Imho, both outputs would be very valuable.
Quoted 5 lines
Okay I can add "formatted" object under json output.
One thing to note here is that the fixed output will change if the map itself changes. So someone writing a program that consumes that fixed output will have to account for his program breaking in future, thus
Yes, that aspect is fine though, any program/script parsing this would need to be aware of the underlying map type to make sense of it (e.g. per-cpu vs non per-cpu maps to name one). But that info it could query/verify already beforehand via bpftool as well (via normal map info dump for a given id).
Quoted 4 lines
breaking backward compatibility anyway as far as the developer is concerned :)
I will go ahead with work on "formatted" object.
Cool, thanks,
Daniel
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 51 lines · Daniel Borkmann
Quoted 39 lines · Okash Khawaja
Quoted 17 lines · Daniel Borkmann
Quoted 2 lines · Jakub Kicinski
Quoted 1 line · Martin KaFai Lau
On Tue, Jun 26, 2018 at 01:31:33PM -0700, Jakub Kicinski wrote:
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 2 lines
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Quoted 4 lines
Quoted 1 line
Parity between JSON and plain text output is non negotiable.
Longish discussion and some confusion in this thread. :-) First of all thanks a lot for working on it, very useful!
Thanks :)
Quoted 7 lines
My $0.02 on it is that so far
great care has been taken in bpftool to indeed have feature parity between JSON and plain text, so it would be highly desirable to keep continuing this practice if the consensus is that it indeed is feasible and makes sense wrt BTF data. There has been mentioned that given BTF data can be dynamic depending on what the user loads via bpf(2) so a potential JSON output may look different/break each time anyway. This however could all beQuoted 8 lines
embedded under a container object that has a fixed key like 'formatted' where tools like jq(1) can query into it. I think this would be fine since the rest of the (non-dynamic) output is still retained as-is and then wouldn't confuse or collide with existing users, and anyone programmatically parsing deeper into the BTF data under such JSON container object needs to have awareness of what specific data it wants to query from it; so there's no conflict wrt breaking anything here. Imho, both outputs would be very valuable.
Quoted 5 lines
Okay I can add "formatted" object under json output.
One thing to note here is that the fixed output will change if the map itself changes. So someone writing a program that consumes that fixed output will have to account for his program breaking in future, thus
Yes, that aspect is fine though, any program/script parsing this would need to be aware of the underlying map type to make sense of it (e.g. per-cpu vs non per-cpu maps to name one). But that info it could query/verify already beforehand via bpftool as well (via normal map info dump for a given id).
Quoted 7 lines
Quoted 4 lines
breaking backward compatibility anyway as far as the developer is concerned :)
I will go ahead with work on "formatted" object.
Cool, thanks,
Daniel
hi,
couple of questions:
1. just to be sure, formatted section will be on the same level as "key" and "value"? so something like following:
$ bpftool map dump -p id 8
[{
"key": ["0x00","0x00","0x00","0x00"
],
"value": [...
],
"formatted": {
"key": 0,
"value": {
"int_field": 3,
"pointerfield": 2152930552,
...
}
}
}]2. i noticed that the ouput in v1 has all the keys and values on the same level. in v2, i'll change them so that each key-value pair is a separate object. let me know what you think.
finally, i noticed there is a map lookup command which also prints map entries. do want that to also be btf-printed in this patchset?
thanks,
okash
Re: [PATCH bpf-next 2/3] bpf: btf: add btf json print functionality
Quoted 69 lines · Okash Khawaja
Quoted 51 lines · Daniel Borkmann
Quoted 39 lines · Okash Khawaja
Quoted 17 lines · Daniel Borkmann
Quoted 2 lines · Jakub Kicinski
Quoted 1 line · Martin KaFai Lau
On Tue, Jun 26, 2018 at 01:31:33PM -0700, Jakub Kicinski wrote:
[...]
Quoted 13 lines
Quoted 12 lines
Quoted 2 lines
Implementing both outputs in one series will help you structure your code to best suit both of the formats up front.
hex and "formatted" are the only things missing? As always, things can be refactored when new use case comes up. Lets wait for Okash input.
Regardless, plaintext is our current use case. Having the current patchset in does not stop us or others from contributing other use cases (json, "bpftool map find"...etc), and IMO it is actually the opposite. Others may help us get there faster than us alone. We should not stop making forward progress and take this patch as hostage because "abc" and "xyz" are not done together.
Quoted 4 lines
Quoted 1 line
Parity between JSON and plain text output is non negotiable.
Longish discussion and some confusion in this thread. :-) First of all thanks a lot for working on it, very useful!
Thanks :)
Quoted 7 lines
My $0.02 on it is that so far
great care has been taken in bpftool to indeed have feature parity between JSON and plain text, so it would be highly desirable to keep continuing this practice if the consensus is that it indeed is feasible and makes sense wrt BTF data. There has been mentioned that given BTF data can be dynamic depending on what the user loads via bpf(2) so a potential JSON output may look different/break each time anyway. This however could all beQuoted 8 lines
embedded under a container object that has a fixed key like 'formatted' where tools like jq(1) can query into it. I think this would be fine since the rest of the (non-dynamic) output is still retained as-is and then wouldn't confuse or collide with existing users, and anyone programmatically parsing deeper into the BTF data under such JSON container object needs to have awareness of what specific data it wants to query from it; so there's no conflict wrt breaking anything here. Imho, both outputs would be very valuable.
Quoted 5 lines
Okay I can add "formatted" object under json output.
One thing to note here is that the fixed output will change if the map itself changes. So someone writing a program that consumes that fixed output will have to account for his program breaking in future, thus
Yes, that aspect is fine though, any program/script parsing this would need to be aware of the underlying map type to make sense of it (e.g. per-cpu vs non per-cpu maps to name one). But that info it could query/verify already beforehand via bpftool as well (via normal map info dump for a given id).
Quoted 7 lines
Quoted 4 lines
breaking backward compatibility anyway as far as the developer is concerned :)
I will go ahead with work on "formatted" object.
Cool, thanks,
Danielhi,
couple of questions:
1. just to be sure, formatted section will be on the same level as "key" and "value"? so something like following:
Quoted 15 lines
$ bpftool map dump -p id 8
[{
"key": ["0x00","0x00","0x00","0x00"
],
"value": [...
],
"formatted": {
"key": 0,
"value": {
"int_field": 3,
"pointerfield": 2152930552,
...
}
}
}]Looks good, yes!
Quoted 3 lines
2. i noticed that the ouput in v1 has all the keys and values on the same level. in v2, i'll change them so that each key-value pair is a separate object. let me know what you think.
For non-JSON output? No preference, whatever looks better :) Empty line between key/value pairs to visually separate them could also work. But up to you.
Quoted 2 lines
finally, i noticed there is a map lookup command which also prints map entries. do want that to also be btf-printed in this patchset?
It would be nice to share the printing code for the two, yes.
PATCH 3/3 bpf: btf: json print map dump with btf info
In reply to Okash KhawajaThis patch modifies bpftool map dump [-j|-p] id <map-id> to json- print and pretty-json-print map dump. It calls btf_dumper introduced in previous patch to accomplish this.
The patch only prints debug info when -j or -p flags are supplied. Then too, if the map has associated btf data loaded. Otherwise the usual debug-less output is printed.
--- tools/bpf/bpftool/map.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-)
tools/bpf/bpftool/map.c +91 -3
@@ -43,9 +43,13 @@#include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <linux/err.h> #include <bpf.h> +#include "json_writer.h" +#include "btf.h" +#include "btf_dumper.h" #include "main.h" static const char *const map_type_name[] = {
@@ -508,6 +512,83 @@ static int do_show(int argc, char **argvreturn errno ==ENOENT ?0 :-1 ; } + +static int do_dump_btf (struct btf *btf,struct bpf_map_info *map_info, +void *key,void *value) +{ +int ret; + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"key" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_key_type_id, key); +if (ret) + goto out; + +jsonw_end_object (json_wtr); + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"value" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_value_type_id, + value); + +out: +/* end of root object */ +jsonw_end_object (json_wtr); + +return ret; +} + +static struct btf *get_btf (struct bpf_map_info *map_info) +{ +int btf_fd =bpf_btf_get_fd_by_id (map_info->btf_id); +struct bpf_btf_info btf_info = {0 }; +__u32 len =sizeof (btf_info); +uint32_t last_size; +int err; +struct btf *btf =NULL ; +void *ptr =NULL , *temp_ptr; + +if (btf_fd <0 ) +return NULL ; + + btf_info.btf_size =4096 ; +do { + last_size = btf_info.btf_size; + temp_ptr =realloc (ptr, last_size); +if (!temp_ptr) { +p_err ("unable allocate memory for debug info." ); + goto exit_free; + } + + ptr = temp_ptr; +bzero (ptr, last_size); + btf_info.btf =ptr_to_u64 (ptr); + err =bpf_obj_get_info_by_fd (btf_fd, &btf_info, &len); + }while (!err && btf_info.btf_size > last_size && last_size ==4096 ); + +if (err || btf_info.btf_size > last_size) { +p_info ("can't get btf info. debug info won't be displayed. error: %s" , + err ?strerror (errno) :"exceeds size retry" ); + goto exit_free; + } + + btf =btf__new ((uint8_t *) btf_info.btf, + btf_info.btf_size,NULL ); +if (IS_ERR (btf)) { +printf ("error when initialising btf: %s\n" , +strerror (PTR_ERR (btf))); + btf =NULL ; + } + +exit_free: +close (btf_fd); +free (ptr); + +return btf; +} +static int do_dump (int argc,char **argv) {void *key, *value, *prev_key;
@@ -516,6 +597,7 @@ static int do_dump(int argc, char **argv__u32 len =sizeof (info);int err;int fd; +struct btf *btf =NULL ;if (argc !=2 )usage ();
@@ -538,6 +620,8 @@ static int do_dump(int argc, char **argv goto exit_free; } + btf =get_btf (&info); + prev_key =NULL ;if (json_output)jsonw_start_array (json_wtr);
@@ -550,9 +634,12 @@ static int do_dump(int argc, char **argv }if (!bpf_map_lookup_elem (fd, key, value)) { -if (json_output) -print_entry_json (&info, key, value); -else +if (json_output) { +if (btf) +do_dump_btf (btf, &info, key, value); +else +print_entry_json (&info, key, value); + }else print_entry_plain (&info, key, value); }else {if (json_output) {
@@ -584,6 +671,7 @@ exit_free:free (key);free (value);close (fd); +btf__free (btf);return err; }
Re: [PATCH bpf-next 3/3] bpf: btf: json print map dump with btf info
Quoted 56 lines · Okash Khawaja
This patch modifies
bpftool map dump [-j|-p] id <map-id>to json- print and pretty-json-print map dump. It callsbtf_dumperintroduced in previous patch to accomplish this.The patch only prints debug info when -j or -p flags are supplied. Then too, if the map has associated btf data loaded. Otherwise the usual debug-less output is printed.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/bpf/bpftool/map.c| 94 ++++++++++++++++++++++++++++++++++++++++++++++--1 file changed, 91 insertions(+), 3 deletions(-)tools/bpf/bpftool/map.c +21
-0@@ -43,9 +43,13 @@#include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <linux/err.h> #include <bpf.h> +#include "json_writer.h" +#include "btf.h" +#include "btf_dumper.h" #include "main.h" static const char *const map_type_name[] = {@@ -508,6 +512,83 @@ static int do_show(int argc, char **argvreturn errno ==ENOENT ?0 :-1 ; } + +static int do_dump_btf (struct btf *btf,struct bpf_map_info *map_info, +void *key,void *value) +{ +int ret; + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"key" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_key_type_id, key); +if (ret) + goto out; + +jsonw_end_object (json_wtr); + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"value" );
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 541
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+ + ret =btf_dumper_type (btf, json_wtr, map_info->btf_value_type_id, + value); + +out: +/* end of root object */ +jsonw_end_object (json_wtr); + +return ret; +}
Please add some tests for do_dump_btf(), including some invalid data type/kind.
Quoted 5 lines
tools/bpf/bpftool/map.c +5
-0+ +static struct btf *get_btf (struct bpf_map_info *map_info) +{ +int btf_fd =bpf_btf_get_fd_by_id (map_info->btf_id); +struct bpf_btf_info btf_info = {0 };
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 556
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+__u32 len =sizeof (btf_info); +uint32_t last_size; +int err; +struct btf *btf =NULL ; +void *ptr =NULL , *temp_ptr; + +if (btf_fd <0 ) +return NULL ; + + btf_info.btf_size =4096 ;
Is btf_size always a constant of 4096?
Quoted 80 lines
tools/bpf/bpftool/map.c +45
-3+do { + last_size = btf_info.btf_size; + temp_ptr =realloc (ptr, last_size); +if (!temp_ptr) { +p_err ("unable allocate memory for debug info." ); + goto exit_free; + } + + ptr = temp_ptr; +bzero (ptr, last_size); + btf_info.btf =ptr_to_u64 (ptr); + err =bpf_obj_get_info_by_fd (btf_fd, &btf_info, &len); + }while (!err && btf_info.btf_size > last_size && last_size ==4096 ); + +if (err || btf_info.btf_size > last_size) { +p_info ("can't get btf info. debug info won't be displayed. error: %s" , + err ?strerror (errno) :"exceeds size retry" ); + goto exit_free; + } + + btf =btf__new ((uint8_t *) btf_info.btf, + btf_info.btf_size,NULL ); +if (IS_ERR (btf)) { +printf ("error when initialising btf: %s\n" , +strerror (PTR_ERR (btf))); + btf =NULL ; + } + +exit_free: +close (btf_fd); +free (ptr); + +return btf; +} +static int do_dump (int argc,char **argv) {void *key, *value, *prev_key;@@ -516,6 +597,7 @@ static int do_dump(int argc, char **argv__u32 len =sizeof (info);int err;int fd; +struct btf *btf =NULL ;if (argc !=2 )usage ();@@ -538,6 +620,8 @@ static int do_dump(int argc, char **argv goto exit_free; } + btf =get_btf (&info); + prev_key =NULL ;if (json_output)jsonw_start_array (json_wtr);@@ -550,9 +634,12 @@ static int do_dump(int argc, char **argv }if (!bpf_map_lookup_elem (fd, key, value)) { -if (json_output) -print_entry_json (&info, key, value); -else +if (json_output) { +if (btf) +do_dump_btf (btf, &info, key, value); +else +print_entry_json (&info, key, value); + }else print_entry_plain (&info, key, value); }else {if (json_output) {@@ -584,6 +671,7 @@ exit_free:free (key);free (value);close (fd); +btf__free (btf);return err; }
Re: [PATCH bpf-next 3/3] bpf: btf: json print map dump with btf info
Quoted 60 lines · Song Liu
Quoted 60 lines · Okash Khawaja
This patch modifies
bpftool map dump [-j|-p] id <map-id>to json- print and pretty-json-print map dump. It callsbtf_dumperintroduced in previous patch to accomplish this.The patch only prints debug info when -j or -p flags are supplied. Then too, if the map has associated btf data loaded. Otherwise the usual debug-less output is printed.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>---
tools/bpf/bpftool/map.c| 94 ++++++++++++++++++++++++++++++++++++++++++++++--1 file changed, 91 insertions(+), 3 deletions(-)tools/bpf/bpftool/map.c +25
-0@@ -43,9 +43,13 @@#include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <linux/err.h> #include <bpf.h> +#include "json_writer.h" +#include "btf.h" +#include "btf_dumper.h" #include "main.h" static const char *const map_type_name[] = {@@ -508,6 +512,83 @@ static int do_show(int argc, char **argvreturn errno ==ENOENT ?0 :-1 ; } + +static int do_dump_btf (struct btf *btf,struct bpf_map_info *map_info, +void *key,void *value) +{ +int ret; + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"key" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_key_type_id, key); +if (ret) + goto out; + +jsonw_end_object (json_wtr); + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"value" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_value_type_id, + value); +
Quoted 8 lines
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 541
Quoted 6 lines
tools/bpf/bpftool/map.c +6
-0+out: +/* end of root object */ +jsonw_end_object (json_wtr); + +return ret; +}Please add some tests for
do_dump_btf(), including some invalid data type/kind.
Sure. I was wondering if we can follow this up with tests afterwards. However if they are essential now, I can add them.
Quoted 10 lines
Quoted 10 lines
tools/bpf/bpftool/map.c +9
-0+ +static struct btf *get_btf (struct bpf_map_info *map_info) +{ +int btf_fd =bpf_btf_get_fd_by_id (map_info->btf_id); +struct bpf_btf_info btf_info = {0 }; +__u32 len =sizeof (btf_info); +uint32_t last_size; +int err; +struct btf *btf =NULL ;
Quoted 8 lines
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 556
Quoted 6 lines
tools/bpf/bpftool/map.c +6
-0+void *ptr =NULL , *temp_ptr; + +if (btf_fd <0 ) +return NULL ; + + btf_info.btf_size =4096 ;Is
btf_sizealways a constant of 4096?
We start out with 4096 and in the loop below we increase the size if needed. It seems like a sane valeu to start with. However if there are better ideas for start value, we can discuss them.
Quoted 82 lines
Quoted 81 lines
tools/bpf/bpftool/map.c +45
-3+do { + last_size = btf_info.btf_size; + temp_ptr =realloc (ptr, last_size); +if (!temp_ptr) { +p_err ("unable allocate memory for debug info." ); + goto exit_free; + } + + ptr = temp_ptr; +bzero (ptr, last_size); + btf_info.btf =ptr_to_u64 (ptr); + err =bpf_obj_get_info_by_fd (btf_fd, &btf_info, &len); + }while (!err && btf_info.btf_size > last_size && last_size ==4096 ); + +if (err || btf_info.btf_size > last_size) { +p_info ("can't get btf info. debug info won't be displayed. error: %s" , + err ?strerror (errno) :"exceeds size retry" ); + goto exit_free; + } + + btf =btf__new ((uint8_t *) btf_info.btf, + btf_info.btf_size,NULL ); +if (IS_ERR (btf)) { +printf ("error when initialising btf: %s\n" , +strerror (PTR_ERR (btf))); + btf =NULL ; + } + +exit_free: +close (btf_fd); +free (ptr); + +return btf; +} +static int do_dump (int argc,char **argv) {void *key, *value, *prev_key;@@ -516,6 +597,7 @@ static int do_dump(int argc, char **argv__u32 len =sizeof (info);int err;int fd; +struct btf *btf =NULL ;if (argc !=2 )usage ();@@ -538,6 +620,8 @@ static int do_dump(int argc, char **argv goto exit_free; } + btf =get_btf (&info); + prev_key =NULL ;if (json_output)jsonw_start_array (json_wtr);@@ -550,9 +634,12 @@ static int do_dump(int argc, char **argv }if (!bpf_map_lookup_elem (fd, key, value)) { -if (json_output) -print_entry_json (&info, key, value); -else +if (json_output) { +if (btf) +do_dump_btf (btf, &info, key, value); +else +print_entry_json (&info, key, value); + }else print_entry_plain (&info, key, value); }else {if (json_output) {@@ -584,6 +671,7 @@ exit_free:free (key);free (value);close (fd); +btf__free (btf);return err; }
Re: [PATCH bpf-next 3/3] bpf: btf: json print map dump with btf info
In reply to Okash KhawajaHi Okash,
Thanks for the patch! Please find some nitpicks inline below.
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
Quoted 32 lines
This patch modifies
bpftool map dump [-j|-p] id <map-id>to json- print and pretty-json-print map dump. It callsbtf_dumperintroduced in previous patch to accomplish this.The patch only prints debug info when -j or -p flags are supplied. Then too, if the map has associated btf data loaded. Otherwise the usual debug-less output is printed.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>--- tools/bpf/bpftool/map.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-)tools/bpf/bpftool/map.c +4
-0@@ -43,9 +43,13 @@#include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <linux/err.h> #include <bpf.h> +#include "json_writer.h" +#include "btf.h" +#include "btf_dumper.h" #include "main.h" static const char *const map_type_name[] = {
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 517
Quoted 6 lines
tools/bpf/bpftool/map.c +3
-0@@ -508,6 +512,83 @@ static int do_show(int argc, char **argvreturn errno ==ENOENT ?0 :-1 ; } + +static int do_dump_btf (struct btf *btf,struct bpf_map_info *map_info, +void *key,void *value)
Nit: Please align the second line on the opening parenthesis.
Quoted 7 lines
tools/bpf/bpftool/map.c +7
-0+{ +int ret; + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"key" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_key_type_id, key);
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 534
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+if (ret) + goto out; + +jsonw_end_object (json_wtr); + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"value" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_value_type_id, + value);
Same comment.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 538
Quoted 4 lines
tools/bpf/bpftool/map.c +4
-0+ +out: +/* end of root object */ +jsonw_end_object (json_wtr);
This is not the root JSON object, which is not produced in that function, so I find the comment misleading.
I also find it confusing that it closes the first JSON object of this function if there is an error, but the second if "btf_dumper_type()" succeeds. What about the following: closing the first object in all cases, before evaluating the value of "ret", and if "ret" is non-null returning immediately; and completely removing the "goto" from this function?
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 551
Quoted 13 lines
tools/bpf/bpftool/map.c +13
-0+ +return ret; +} + +static struct btf *get_btf (struct bpf_map_info *map_info) +{ +int btf_fd =bpf_btf_get_fd_by_id (map_info->btf_id); +struct bpf_btf_info btf_info = {0 }; +__u32 len =sizeof (btf_info); +uint32_t last_size; +int err; +struct btf *btf =NULL ; +void *ptr =NULL , *temp_ptr;
Nit: please sort declarations in reverse-Christmas-tree order.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 561
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+ +if (btf_fd <0 ) +return NULL ; + + btf_info.btf_size =4096 ; + do { + last_size = btf_info.btf_size; + temp_ptr = realloc(ptr, last_size); + if (!temp_ptr) { + p_err("unable allocate memory for debug info.");
"unable *to* allocate"?
(Also most other error messages do not end with a period, but here this is just me being fussy.)
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 569
Quoted 8 lines
tools/bpf/bpftool/map.c +8
-0+ goto exit_free; + } + + ptr = temp_ptr; +bzero (ptr, last_size); + btf_info.btf =ptr_to_u64 (ptr); + err =bpf_obj_get_info_by_fd (btf_fd, &btf_info, &len); + }while (!err && btf_info.btf_size > last_size && last_size ==4096 );
If I understand correctly, the first time you try to retrieve up to 4096 bytes, and if the btf_info is larger than this, you try a second time with the size returned in btf_info.btf_size instead. I don't find it intuitive (but maybe this is just me?), do you think you could add a comment above this bloc maybe?
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 573
Quoted 4 lines
tools/bpf/bpftool/map.c +4
-0+ +if (err || btf_info.btf_size > last_size) { +p_info ("can't get btf info. debug info won't be displayed. error: %s" , + err ?strerror (errno) :"exceeds size retry" );
Nit: Please align the second line on the opening parenthesis.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 577
Quoted 4 lines
tools/bpf/bpftool/map.c +4
-0+ goto exit_free; + } + + btf = btf__new((uint8_t *) btf_info.btf,
Nit: No space between the cast and the name of the variable.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 578
Quoted 1 line
tools/bpf/bpftool/map.c +1
-0+ btf_info.btf_size, NULL);
Same remark on parenthesis here...
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 581
Quoted 3 lines
tools/bpf/bpftool/map.c +3
-0+if (IS_ERR (btf)) { +printf ("error when initialising btf: %s\n" , +strerror (PTR_ERR (btf)));
... and here.
Quoted 14 lines
tools/bpf/bpftool/map.c +10
-0+ btf =NULL ; + } + +exit_free: +close (btf_fd); +free (ptr); + +return btf; +} +static int do_dump (int argc,char **argv) {void *key, *value, *prev_key;
tools/bpf/bpftool/map.c static int do_dump(int argc, char **argv · line 600
Quoted 4 lines
tools/bpf/bpftool/map.c +1
-0@@ -516,6 +597,7 @@ static int do_dump(int argc, char **argv__u32 len =sizeof (info);int err;int fd; +struct btf *btf =NULL ;
Reverse-Christmas-tree order, please.
Quoted 16 lines
tools/bpf/bpftool/map.c +2
-0if (argc !=2 )usage ();@@ -538,6 +620,8 @@ static int do_dump(int argc, char **argv goto exit_free; } + btf =get_btf (&info); + prev_key =NULL ;if (json_output)jsonw_start_array (json_wtr);@@ -550,9 +634,12 @@ static int do_dump(int argc, char **argv }if (!bpf_map_lookup_elem (fd, key, value)) {
tools/bpf/bpftool/map.c static int do_dump(int argc, char **argv · line 643
Quoted 10 lines
tools/bpf/bpftool/map.c +6
-3-if (json_output) -print_entry_json (&info, key, value); -else +if (json_output) { +if (btf) +do_dump_btf (btf, &info, key, value); +else +print_entry_json (&info, key, value); + }else print_entry_plain (&info, key, value);
Please add brackets around "print_entry_plain()" (to harmonise with the "if" of the same bloc).
tools/bpf/bpftool/map.c exit_free: · line 677
Quoted 11 lines
tools/bpf/bpftool/map.c +1
-0}else {if (json_output) {@@ -584,6 +671,7 @@ exit_free:free (key);free (value);close (fd); +btf__free (btf);return err; }
Thanks,
Quentin
Re: [PATCH bpf-next 3/3] bpf: btf: json print map dump with btf info
Hi Quentin,
Quoted 3 lines · Quentin Monnet
Hi Okash,
Thanks for the patch! Please find some nitpicks inline below.
Thanks for your feedback. All of it makes sense so I'll send v2 with those changes. Couple of responses are inlined below.
Quoted 71 lines
2018-06-20 13:30 UTC-0700 ~ Okash Khawaja <[email protected]>
Quoted 32 lines
This patch modifies
bpftool map dump [-j|-p] id <map-id>to json- print and pretty-json-print map dump. It callsbtf_dumperintroduced in previous patch to accomplish this.The patch only prints debug info when -j or -p flags are supplied. Then too, if the map has associated btf data loaded. Otherwise the usual debug-less output is printed.
Signed-off-by Okash Khawaja <[email protected]>Acked-by Martin KaFai Lau <[email protected]>--- tools/bpf/bpftool/map.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-)tools/bpf/bpftool/map.c +4
-0@@ -43,9 +43,13 @@#include <unistd.h> #include <sys/types.h> #include <sys/stat.h> +#include <linux/err.h> #include <bpf.h> +#include "json_writer.h" +#include "btf.h" +#include "btf_dumper.h" #include "main.h" static const char *const map_type_name[] = {tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 517
Quoted 6 lines
tools/bpf/bpftool/map.c +3
-0@@ -508,6 +512,83 @@ static int do_show(int argc, char **argvreturn errno ==ENOENT ?0 :-1 ; } + +static int do_dump_btf (struct btf *btf,struct bpf_map_info *map_info, +void *key,void *value)Nit: Please align the second line on the opening parenthesis.
Quoted 7 lines
tools/bpf/bpftool/map.c +7
-0+{ +int ret; + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"key" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_key_type_id, key);tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 534
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+if (ret) + goto out; + +jsonw_end_object (json_wtr); + +jsonw_start_object (json_wtr); +jsonw_name (json_wtr,"value" ); + + ret =btf_dumper_type (btf, json_wtr, map_info->btf_value_type_id, + value);Same comment.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 538
Quoted 4 lines
tools/bpf/bpftool/map.c +4
-0+ +out: +/* end of root object */ +jsonw_end_object (json_wtr);This is not the root JSON object, which is not produced in that function, so I find the comment misleading.
Quoted 6 lines
I also find it confusing that it closes the first JSON object of this function if there is an error, but the second if "
btf_dumper_type()" succeeds. What about the following: closing the first object in all cases, before evaluating the value of "ret", and if "ret" is non-null returning immediately; and completely removing the "goto" from this function?
Code will be more intuitive that way so I'll re-organise it accordingly.
Quoted 17 lines
Quoted 4 lines
tools/bpf/bpftool/map.c +3
-0+ +return ret; +}tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 551
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+ +static struct btf *get_btf (struct bpf_map_info *map_info) +{ +int btf_fd =bpf_btf_get_fd_by_id (map_info->btf_id); +struct bpf_btf_info btf_info = {0 }; +__u32 len =sizeof (btf_info); +uint32_t last_size; +int err; +struct btf *btf =NULL ; +void *ptr =NULL , *temp_ptr;Nit: please sort declarations in reverse-Christmas-tree order.
Quoted 14 lines
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 561
Quoted 10 lines
tools/bpf/bpftool/map.c +10
-0+ +if (btf_fd <0 ) +return NULL ; + + btf_info.btf_size =4096 ; + do { + last_size = btf_info.btf_size; + temp_ptr = realloc(ptr, last_size); + if (!temp_ptr) { + p_err("unable allocate memory for debug info.");"unable *to* allocate"?
(Also most other error messages do not end with a period, but here this is just me being fussy.)
I think it makes sense to be consistent. I'll remove the full stop.
Quoted 15 lines
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 569
Quoted 9 lines
tools/bpf/bpftool/map.c +8
-0+ goto exit_free; + } + + ptr = temp_ptr; +bzero (ptr, last_size); + btf_info.btf =ptr_to_u64 (ptr); + err =bpf_obj_get_info_by_fd (btf_fd, &btf_info, &len); + }while (!err && btf_info.btf_size > last_size && last_size ==4096 );If I understand correctly, the first time you try to retrieve up to 4096 bytes, and if the
btf_infois larger than this, you try a second time with the size returned in btf_info.btf_size instead. I don't find it intuitive (but maybe this is just me?), do you think you could add a comment above this bloc maybe?
Yes that is what this code is doing. I'll add comments explaining it.
Quoted 90 lines
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 573
Quoted 5 lines
tools/bpf/bpftool/map.c +4
-0+ +if (err || btf_info.btf_size > last_size) { +p_info ("can't get btf info. debug info won't be displayed. error: %s" , + err ?strerror (errno) :"exceeds size retry" );Nit: Please align the second line on the opening parenthesis.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 577
Quoted 4 lines
tools/bpf/bpftool/map.c +4
-0+ goto exit_free; + } + + btf = btf__new((uint8_t *) btf_info.btf,Nit: No space between the cast and the name of the variable.
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 578
Quoted 1 line
tools/bpf/bpftool/map.c +1
-0+ btf_info.btf_size, NULL);Same remark on parenthesis here...
tools/bpf/bpftool/map.c static int do_show(int argc, char **argv · line 581
Quoted 3 lines
tools/bpf/bpftool/map.c +3
-0+if (IS_ERR (btf)) { +printf ("error when initialising btf: %s\n" , +strerror (PTR_ERR (btf)));... and here.
Quoted 14 lines
tools/bpf/bpftool/map.c +10
-0+ btf =NULL ; + } + +exit_free: +close (btf_fd); +free (ptr); + +return btf; +} +static int do_dump (int argc,char **argv) {void *key, *value, *prev_key;tools/bpf/bpftool/map.c static int do_dump(int argc, char **argv · line 600
Quoted 4 lines
tools/bpf/bpftool/map.c +1
-0@@ -516,6 +597,7 @@ static int do_dump(int argc, char **argv__u32 len =sizeof (info);int err;int fd; +struct btf *btf =NULL ;Reverse-Christmas-tree order, please.
Quoted 16 lines
tools/bpf/bpftool/map.c +2
-0if (argc !=2 )usage ();@@ -538,6 +620,8 @@ static int do_dump(int argc, char **argv goto exit_free; } + btf =get_btf (&info); + prev_key =NULL ;if (json_output)jsonw_start_array (json_wtr);@@ -550,9 +634,12 @@ static int do_dump(int argc, char **argv }if (!bpf_map_lookup_elem (fd, key, value)) {tools/bpf/bpftool/map.c static int do_dump(int argc, char **argv · line 643
Quoted 10 lines
tools/bpf/bpftool/map.c +6
-3-if (json_output) -print_entry_json (&info, key, value); -else +if (json_output) { +if (btf) +do_dump_btf (btf, &info, key, value); +else +print_entry_json (&info, key, value); + }else print_entry_plain (&info, key, value);Please add brackets around "
print_entry_plain()" (to harmonise with the "if" of the same bloc).tools/bpf/bpftool/map.c exit_free: · line 677
Quoted 11 lines
tools/bpf/bpftool/map.c +1
-0}else {if (json_output) {@@ -584,6 +671,7 @@ exit_free:free (key);free (value);close (fd); +btf__free (btf);return err; }Thanks,
Quentin