Rework CTF event description storage
cover Rework CTF event description storage
Following recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.
While at it, I tweaked the metadata file output.
I consider this -rc2 material.
Signature · 14 lines
David Marchand David Marchand (3): trace: fixup CTF event description at registration trace: remove size limit on CTF event description trace: make CTF metadata prettier lib/librte_eal/common/eal_common_trace.c | 44 ++--- lib/librte_eal/common/eal_common_trace_ctf.c | 164 +++++-------------- lib/librte_eal/common/eal_trace.h | 4 +- 3 files changed, 67 insertions(+), 145 deletions(-) -- 2.23.0
PATCH 1/3 trace: fixup CTF event description at registration
CTF event description is currently built by appending all fields in a single string at trace point registration.
When dumping the metadata, this string is split again and inspected to fixup reserved keywords and special tokens like "." or "->".
Move this fixup per field at trace point registration time so that there is no need for inspecting / string parsing when dumping metadata. Use dynamic allocations to remove an artificial size limit on the CTF event description manipulations.
--- lib/librte_eal/common/eal_common_trace.c | 5 + lib/librte_eal/common/eal_common_trace_ctf.c | 159 +++++-------------- lib/librte_eal/common/eal_trace.h | 1 + 3 files changed, 46 insertions(+), 119 deletions(-)
lib/librte_eal/common/eal_common_trace.c +5 -0
@@ -432,11 +432,16 @@ __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype)char *field =RTE_PER_LCORE (ctf_field);int count =RTE_PER_LCORE (ctf_count);size_t size; +char *fixup;int rc; size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count);RTE_PER_LCORE (trace_point_sz) += sz; + fixup =trace_metadata_fixup_field (in); +if (fixup !=NULL ) + in = fixup; rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); +free (fixup);if (rc <=0 || (size_t )rc >= size) {RTE_PER_LCORE (trace_point_sz) =0 ;trace_crit ("CTF field is too long" );
lib/librte_eal/common/eal_common_trace_ctf.c +40 -119
174 lines not on this page · raw message
lib/librte_eal/common/eal_trace.h +1 -0
@@ -104,6 +104,7 @@ bool trace_has_duplicate_entry(void);void trace_uuid_generate (void );int trace_metadata_create (void );void trace_metadata_destroy (void ); +char *trace_metadata_fixup_field (const char *field);int trace_mkdir (void );int trace_epoch_time_save (void );void trace_mem_free (void );
Signature · 1 line
2.23.0
PATCH 2/3 trace: remove size limit on CTF event description
In reply to David MarchandRework registration so that it uses dynamic allocations and has no size limit.
--- lib/librte_eal/common/eal_common_trace.c | 39 +++++++++----------- lib/librte_eal/common/eal_common_trace_ctf.c | 3 +- lib/librte_eal/common/eal_trace.h | 3 +- 3 files changed, 20 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_trace.c +17 -22
@@ -17,8 +17,7 @@ RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz); RTE_DEFINE_PER_LCORE(void *, trace_mem); -static RTE_DEFINE_PER_LCORE(char, ctf_field[TRACE_CTF_FIELD_SIZE]); -static RTE_DEFINE_PER_LCORE(int, ctf_count); +static RTE_DEFINE_PER_LCORE(char *, ctf_field);static struct trace_point_head tp_list =STAILQ_HEAD_INITIALIZER (tp_list);static struct trace trace = { .args =STAILQ_HEAD_INITIALIZER (trace.args), };
@@ -429,32 +428,33 @@ trace_mem_free(void)void __rte_trace_point_emit_field (size_t sz,const char *in,const char *datatype) { -char *field =RTE_PER_LCORE (ctf_field); -int count =RTE_PER_LCORE (ctf_count); -size_t size; +char *field;char *fixup;int rc; - size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count); -RTE_PER_LCORE (trace_point_sz) += sz; fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); + rc =asprintf (&field,"%s%s %s;" , +RTE_PER_LCORE (ctf_field) !=NULL ? +RTE_PER_LCORE (ctf_field) :"" , + datatype, in); +free (RTE_PER_LCORE (ctf_field));free (fixup); -if (rc <=0 || (size_t )rc >= size) { +if (rc ==-1 ) {RTE_PER_LCORE (trace_point_sz) =0 ; -trace_crit ("CTF field is too long" ); +RTE_PER_LCORE (ctf_field) =NULL ; +trace_crit ("could not allocate CTF field" );return ; } -RTE_PER_LCORE (ctf_count) += rc; +RTE_PER_LCORE (trace_point_sz) += sz; +RTE_PER_LCORE (ctf_field) = field; }int __rte_trace_point_register (rte_trace_point_t *handle,const char *name,void (*register_fn)(void )) { -char *field =RTE_PER_LCORE (ctf_field);struct trace_point *tp;uint16_t sz;
@@ -467,7 +467,6 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name,/* Check the size of the trace point object */ RTE_PER_LCORE (trace_point_sz) =0 ; -RTE_PER_LCORE (ctf_count) =0 ;register_fn ();if (RTE_PER_LCORE (trace_point_sz) ==0 ) {trace_err ("missing rte_trace_emit_header() in register fn" );
@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ; - goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE ); +/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;/* Form the trace handle */ *handle = sz;
lib/librte_eal/common/eal_common_trace_ctf.c +2 -1
@@ -233,7 +233,8 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" fields := struct {\n" " %s\n" " };\n" - "};\n\n", trace_id_get(tp->handle), tp->name, tp->ctf_field); +"};\n\n" ,trace_id_get (tp->handle), tp->name, + tp->ctf_field != NULL ? tp->ctf_field : "");return meta_copy (meta, offset, str, rc); }
lib/librte_eal/common/eal_trace.h +1 -2
@@ -24,7 +24,6 @@#define TRACE_PREFIX_LEN 12#define TRACE_DIR_STR_LEN (sizeof("YYYY-mm-dd-AM-HH-MM-SS") + TRACE_PREFIX_LEN) -#define TRACE_CTF_FIELD_SIZE 448#define TRACE_POINT_NAME_SIZE 64#define TRACE_CTF_MAGIC 0xC1FC1FC1#define TRACE_MAX_ARGS 32
@@ -33,7 +32,7 @@ struct trace_point {STAILQ_ENTRY (trace_point) next;rte_trace_point_t *handle;char name[TRACE_POINT_NAME_SIZE ]; -char ctf_field[TRACE_CTF_FIELD_SIZE ]; +char *ctf_field; };enum trace_area_e {
Signature · 1 line
2.23.0
Re: [dpdk-dev] [PATCH 2/3] trace: remove size limit on CTF event description
Quoted 5 lines · David Marchand
Rework registration so that it uses dynamic allocations and has no size limit.
Signed-off-by David Marchand <[email protected]>
Quoted 131 lines
--- lib/librte_eal/common/eal_common_trace.c | 39 +++++++++----------- lib/librte_eal/common/eal_common_trace_ctf.c | 3 +- lib/librte_eal/common/eal_trace.h | 3 +- 3 files changed, 20 insertions(+), 25 deletions(-)lib/librte_eal/common/eal_common_trace.c +17
-22@@ -17,8 +17,7 @@ RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz); RTE_DEFINE_PER_LCORE(void *, trace_mem); -static RTE_DEFINE_PER_LCORE(char, ctf_field[TRACE_CTF_FIELD_SIZE]); -static RTE_DEFINE_PER_LCORE(int, ctf_count); +static RTE_DEFINE_PER_LCORE(char *, ctf_field);static struct trace_point_head tp_list =STAILQ_HEAD_INITIALIZER (tp_list);static struct trace trace = { .args =STAILQ_HEAD_INITIALIZER (trace.args), };@@ -429,32 +428,33 @@ trace_mem_free(void)void __rte_trace_point_emit_field (size_t sz,const char *in,const char *datatype) { -char *field =RTE_PER_LCORE (ctf_field); -int count =RTE_PER_LCORE (ctf_count); -size_t size; +char *field;char *fixup;int rc; - size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count); -RTE_PER_LCORE (trace_point_sz) += sz; fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); + rc =asprintf (&field,"%s%s %s;" , +RTE_PER_LCORE (ctf_field) !=NULL ? +RTE_PER_LCORE (ctf_field) :"" , + datatype, in); +free (RTE_PER_LCORE (ctf_field));free (fixup); -if (rc <=0 || (size_t )rc >= size) { +if (rc ==-1 ) {RTE_PER_LCORE (trace_point_sz) =0 ; -trace_crit ("CTF field is too long" ); +RTE_PER_LCORE (ctf_field) =NULL ; +trace_crit ("could not allocate CTF field" );return ; } -RTE_PER_LCORE (ctf_count) += rc; +RTE_PER_LCORE (trace_point_sz) += sz; +RTE_PER_LCORE (ctf_field) = field; }int __rte_trace_point_register (rte_trace_point_t *handle,const char *name,void (*register_fn)(void )) { -char *field =RTE_PER_LCORE (ctf_field);struct trace_point *tp;uint16_t sz;@@ -467,7 +467,6 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name,/* Check the size of the trace point object */ RTE_PER_LCORE (trace_point_sz) =0 ; -RTE_PER_LCORE (ctf_count) =0 ;register_fn ();if (RTE_PER_LCORE (trace_point_sz) ==0 ) {trace_err ("missing rte_trace_emit_header() in register fn" );@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ; - goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE ); +/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;/* Form the trace handle */ *handle = sz;lib/librte_eal/common/eal_common_trace_ctf.c +2
-1@@ -233,7 +233,8 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" fields := struct {\n" " %s\n" " };\n" - "};\n\n", trace_id_get(tp->handle), tp->name, tp->ctf_field); +"};\n\n" ,trace_id_get (tp->handle), tp->name, + tp->ctf_field != NULL ? tp->ctf_field : "");return meta_copy (meta, offset, str, rc); }lib/librte_eal/common/eal_trace.h +1
-3@@ -24,7 +24,6 @@#define TRACE_PREFIX_LEN 12#define TRACE_DIR_STR_LEN (sizeof("YYYY-mm-dd-AM-HH-MM-SS") + TRACE_PREFIX_LEN) -#define TRACE_CTF_FIELD_SIZE 448#define TRACE_POINT_NAME_SIZE 64#define TRACE_CTF_MAGIC 0xC1FC1FC1#define TRACE_MAX_ARGS 32@@ -33,7 +32,7 @@ struct trace_point {STAILQ_ENTRY (trace_point) next;rte_trace_point_t *handle;char name[TRACE_POINT_NAME_SIZE ]; -char ctf_field[TRACE_CTF_FIELD_SIZE ]; +char *ctf_field; };enum trace_area_e { --2.23.0
PATCH 3/3 trace: make CTF metadata prettier
In reply to David MarchandThis is simply a cosmetic change.
Before:
event {
id = 17;
name = "lib.eal.alarm.set";
fields := struct {
uint64_t us;uintptr_t cb_fn;uintptr_t cb_arg;int32_t rc;
};
};After:
event {
id = 17;
name = "lib.eal.alarm.set";
fields := struct {
uint64_t us;
uintptr_t cb_fn;
uintptr_t cb_arg;
int32_t rc;
};
};--- lib/librte_eal/common/eal_common_trace.c | 2 +- lib/librte_eal/common/eal_common_trace_ctf.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_trace.c +1 -1
@@ -435,7 +435,7 @@ __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype) fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =asprintf (&field,"%s%s %s;" , + rc =asprintf (&field,"%s %s %s;\n" ,RTE_PER_LCORE (ctf_field) !=NULL ?RTE_PER_LCORE (ctf_field) :"" , datatype, in);
lib/librte_eal/common/eal_common_trace_ctf.c +1 -1
@@ -231,7 +231,7 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" id = %d;\n" " name = \"%s\";\n" " fields := struct {\n" -" %s\n" +"%s" " };\n" "};\n\n" ,trace_id_get (tp->handle), tp->name, tp->ctf_field != NULL ? tp->ctf_field : "");
Signature · 1 line
2.23.0
Quoted 9 lines · David Marchand
Following recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.While at it, I tweaked the metadata file output.
I consider this -rc2 material.
Comments?
Thanks.
Signature · 1 line
David Marchand
Quoted 9 lines · David Marchand
Quoted 6 lines · David Marchand
Following recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.
Quoted 6 lines
Quoted 3 lines
While at it, I tweaked the metadata file output.
I consider this -rc2 material.
Comments?
Thanks.
The Generated metadata has issues. Please check
Reproducer:
echo "trace_autotest" | ./build/app/test/dpdk-test -c 0x3 --trace=.* --no-huge --trace=.*
[main][dpdk.org] $ babeltrace /home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37/ [error] at line 1303: invalid character '0x11'
[error] at line 1303: token "": syntax error, unexpected ERROR [error] Error creating AST
[warning] Unable to open trace metadata for path "/home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37". [warning] [Context] Cannot open_trace of format ctf at path /home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37. [warning] [Context] cannot open trace "/home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37" from /home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37/ for reading. [error] Cannot open any trace for reading.
[error] opening trace "/home/jerin/dpdk-traces/rte-2020-10-28-PM-02-02-37/" for reading. [error] none of the specified trace paths could be opened.
Quoted 4 lines
--
David Marchand
Quoted 19 lines · Jerin Jacob
Quoted 9 lines · David Marchand
Quoted 6 lines · David Marchand
Following recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.Quoted 6 lines
Quoted 3 lines
While at it, I tweaked the metadata file output.
I consider this -rc2 material.
Comments?
Thanks.
Quoted 6 lines
The Generated metadata has issues. Please check
Reproducer:
echo "
trace_autotest" | ./build/app/test/dpdk-test -c0x3--trace=.* --no-huge --trace=.*
Err, indeed, thanks for catching.
I did some diff on metadata files, but did not notice this trailing character.
This is an issue with the metadata string manipulations, that appears with the last patch...
I ended up rewriting most of _ctf.c (removing intermediate buffers allocations) and it works but I'll see if I can pinpoint the issue.
Signature · 1 line
David Marchand
Quoted 10 lines · David Marchand
Quoted 2 lines
echo "
trace_autotest" | ./build/app/test/dpdk-test -c0x3--trace=.* --no-huge --trace=.*Err, indeed, thanks for catching.
I did some diff on metadata files, but did not notice this trailing character.This is an issue with the metadata string manipulations, that appears with the last patch...
I ended up rewriting most of_ctf.c(removing intermediate buffers allocations) and it works but I'll see if I can pinpoint the issue.
The problem is in the current HEAD, but it is revealed by my series, maybe because the dynamicity around allocations changed. I see no check on the length of trace->ctf_meta when writing to the metadata file.
Did I miss something?
intrte_trace_metadata_dump(FILE *f)
{
...
rc = fprintf(f, "%s", ctf_meta);
...
}
Breakpoint 1, trace_mkdir () at ../lib/librte_eal/common/eal_common_trace_utils.c:317
317 {
(gdb) p strlen(trace_obj_get()->ctf_meta)
$2 = 21865
(gdb) p trace_obj_get()->ctf_meta[21865]
$3 = 0 '\000'
(gdb) set trace_obj_get()->ctf_meta[21865] = 'A'
...
$ babeltrace $(ls -1rtd $HOME/dpdk-traces/* |tail -1) [error] at line 1008: token "A": syntax error, unexpected IDENTIFIER [error] Error creating AST ...
This fixes it:
@@ -37,11 +37,12 @@ meta_copy(char **meta, int *offset, char *str, int rc)
if (rc < 0)
return rc;- ptr = realloc(ptr, count + rc);
+ ptr = realloc(ptr, count + rc + 1);
if (ptr == NULL)
goto free_str;
memcpy(RTE_PTR_ADD(ptr, count), str, rc);
+ ptr[count + rc] = '\0';
count += rc;
free(str);Signature · 1 line
David Marchand
Quoted 5 lines · David Marchand
This fixes it:
(quoted diff) +0
-0@@ -37,11 +37,12 @@ meta_copy(char **meta, int *offset, char *str, int rc) if (rc < 0) return rc;
Review meta_copy(char **meta, int *offset, char *str, int rc) · line 48
Quoted 10 lines
(quoted diff) +2
-1- ptr = realloc(ptr, count + rc); + ptr = realloc(ptr, count + rc + 1); if (ptr == NULL) goto free_str; memcpy(RTE_PTR_ADD(ptr, count), str, rc); + ptr[count + rc] = '\0'; count += rc; free(str);
The other alternative is to prefer libc string formatting functions rather than plain memory alloc + copy + manual null termination:
https://github.com/david-marchand/dpdk/commit/traces
Signature · 1 line
David Marchand
v2 cover Rework CTF event description storage
In reply to David MarchandFollowing recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.
Signature · 18 lines
David Marchand Changelog since v1: - fix metadata dump, David Marchand (4): trace: fixup CTF event description at registration trace: remove size limit on CTF event description trace: fix metadata dump trace: make CTF metadata prettier lib/librte_eal/common/eal_common_trace.c | 44 ++--- lib/librte_eal/common/eal_common_trace_ctf.c | 167 +++++-------------- lib/librte_eal/common/eal_trace.h | 4 +- 3 files changed, 69 insertions(+), 146 deletions(-) -- 2.23.0
PATCH v2 1/4 trace: fixup CTF event description at registration
CTF event description is currently built by appending all fields in a single string at trace point registration.
When dumping the metadata, this string is split again and inspected to fixup reserved keywords and special tokens like "." or "->".
Move this fixup per field at trace point registration time so that there is no need for inspecting / string parsing when dumping metadata. Use dynamic allocations to remove an artificial size limit on the CTF event description manipulations.
--- lib/librte_eal/common/eal_common_trace.c | 5 + lib/librte_eal/common/eal_common_trace_ctf.c | 159 +++++-------------- lib/librte_eal/common/eal_trace.h | 1 + 3 files changed, 46 insertions(+), 119 deletions(-)
lib/librte_eal/common/eal_common_trace.c +5 -0
@@ -432,11 +432,16 @@ __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype)char *field =RTE_PER_LCORE (ctf_field);int count =RTE_PER_LCORE (ctf_count);size_t size; +char *fixup;int rc; size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count);RTE_PER_LCORE (trace_point_sz) += sz; + fixup =trace_metadata_fixup_field (in); +if (fixup !=NULL ) + in = fixup; rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); +free (fixup);if (rc <=0 || (size_t )rc >= size) {RTE_PER_LCORE (trace_point_sz) =0 ;trace_crit ("CTF field is too long" );
lib/librte_eal/common/eal_common_trace_ctf.c +40 -119
174 lines not on this page · raw message
lib/librte_eal/common/eal_trace.h +1 -0
@@ -104,6 +104,7 @@ bool trace_has_duplicate_entry(void);void trace_uuid_generate (void );int trace_metadata_create (void );void trace_metadata_destroy (void ); +char *trace_metadata_fixup_field (const char *field);int trace_mkdir (void );int trace_epoch_time_save (void );void trace_mem_free (void );
Signature · 1 line
2.23.0
Re: [dpdk-dev] [EXT] [PATCH v2 1/4] trace: fixup CTF event description at registration
Quoted 16 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 2:33 AM
To: [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; Sunil Kumar Kori <[email protected]>
Subject: [EXT] [PATCH v2 1/4] trace: fixup CTF event description at registrationExternal Email
----------------------------------------------------------------------
CTF event description is currently built by appending all fields in a single string at trace point registration.
When dumping the metadata, this string is split again and inspected to fixup reserved keywords and special tokens like "." or "->".
Quoted 6 lines
Move this fixup per field at trace point registration time so that there is no need for inspecting / string parsing when dumping metadata.
Use dynamic allocations to remove an artificial size limit on the CTF event description manipulations.Signed-off-by David Marchand <[email protected]>
[snipped]
Quoted 2 lines
--
2.23.0
PATCH v2 2/4 trace: remove size limit on CTF event description
In reply to David MarchandRework registration so that it uses dynamic allocations and has no size limit.
--- lib/librte_eal/common/eal_common_trace.c | 39 +++++++++----------- lib/librte_eal/common/eal_common_trace_ctf.c | 3 +- lib/librte_eal/common/eal_trace.h | 3 +- 3 files changed, 20 insertions(+), 25 deletions(-)
lib/librte_eal/common/eal_common_trace.c +17 -22
@@ -17,8 +17,7 @@ RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz); RTE_DEFINE_PER_LCORE(void *, trace_mem); -static RTE_DEFINE_PER_LCORE(char, ctf_field[TRACE_CTF_FIELD_SIZE]); -static RTE_DEFINE_PER_LCORE(int, ctf_count); +static RTE_DEFINE_PER_LCORE(char *, ctf_field);static struct trace_point_head tp_list =STAILQ_HEAD_INITIALIZER (tp_list);static struct trace trace = { .args =STAILQ_HEAD_INITIALIZER (trace.args), };
@@ -429,32 +428,33 @@ trace_mem_free(void)void __rte_trace_point_emit_field (size_t sz,const char *in,const char *datatype) { -char *field =RTE_PER_LCORE (ctf_field); -int count =RTE_PER_LCORE (ctf_count); -size_t size; +char *field;char *fixup;int rc; - size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count); -RTE_PER_LCORE (trace_point_sz) += sz; fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); + rc =asprintf (&field,"%s%s %s;" , +RTE_PER_LCORE (ctf_field) !=NULL ? +RTE_PER_LCORE (ctf_field) :"" , + datatype, in); +free (RTE_PER_LCORE (ctf_field));free (fixup); -if (rc <=0 || (size_t )rc >= size) { +if (rc ==-1 ) {RTE_PER_LCORE (trace_point_sz) =0 ; -trace_crit ("CTF field is too long" ); +RTE_PER_LCORE (ctf_field) =NULL ; +trace_crit ("could not allocate CTF field" );return ; } -RTE_PER_LCORE (ctf_count) += rc; +RTE_PER_LCORE (trace_point_sz) += sz; +RTE_PER_LCORE (ctf_field) = field; }int __rte_trace_point_register (rte_trace_point_t *handle,const char *name,void (*register_fn)(void )) { -char *field =RTE_PER_LCORE (ctf_field);struct trace_point *tp;uint16_t sz;
@@ -467,7 +467,6 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name,/* Check the size of the trace point object */ RTE_PER_LCORE (trace_point_sz) =0 ; -RTE_PER_LCORE (ctf_count) =0 ;register_fn ();if (RTE_PER_LCORE (trace_point_sz) ==0 ) {trace_err ("missing rte_trace_emit_header() in register fn" );
@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ; - goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE ); +/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;/* Form the trace handle */ *handle = sz;
lib/librte_eal/common/eal_common_trace_ctf.c +2 -1
@@ -233,7 +233,8 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" fields := struct {\n" " %s\n" " };\n" - "};\n\n", trace_id_get(tp->handle), tp->name, tp->ctf_field); +"};\n\n" ,trace_id_get (tp->handle), tp->name, + tp->ctf_field != NULL ? tp->ctf_field : "");return meta_copy (meta, offset, str, rc); }
lib/librte_eal/common/eal_trace.h +1 -2
@@ -24,7 +24,6 @@#define TRACE_PREFIX_LEN 12#define TRACE_DIR_STR_LEN (sizeof("YYYY-mm-dd-AM-HH-MM-SS") + TRACE_PREFIX_LEN) -#define TRACE_CTF_FIELD_SIZE 448#define TRACE_POINT_NAME_SIZE 64#define TRACE_CTF_MAGIC 0xC1FC1FC1#define TRACE_MAX_ARGS 32
@@ -33,7 +32,7 @@ struct trace_point {STAILQ_ENTRY (trace_point) next;rte_trace_point_t *handle;char name[TRACE_POINT_NAME_SIZE ]; -char ctf_field[TRACE_CTF_FIELD_SIZE ]; +char *ctf_field; };enum trace_area_e {
Signature · 1 line
2.23.0
Re: [dpdk-dev] [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event description
Quoted 96 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 2:33 AM
To: [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; Sunil Kumar Kori <[email protected]>
Subject: [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event descriptionExternal Email
----------------------------------------------------------------------
Rework registration so that it uses dynamic allocations and has no size limit.Signed-off-by David Marchand <[email protected]>Reviewed-by Jerin Jacob <[email protected]>---
lib/librte_eal/common/eal_common_trace.c | 39 +++++++++----------- lib/librte_eal/common/eal_common_trace_ctf.c | 3 +- lib/librte_eal/common/eal_trace.h | 3 +- 3 files changed, 20 insertions(+), 25 deletions(-)lib/librte_eal/common/eal_common_trace.c +0
-0@@ -17,8 +17,7 @@ RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz);
RTE_DEFINE_PER_LCORE(void *, trace_mem); -staticRTE_DEFINE_PER_LCORE(char, ctf_field[TRACE_CTF_FIELD_SIZE]); -staticRTE_DEFINE_PER_LCORE(int, ctf_count);
+staticRTE_DEFINE_PER_LCORE(char *, ctf_field);static struct
trace_point_headtp_list=STAILQ_HEAD_INITIALIZER(tp_list); static struct trace trace = { .args =STAILQ_HEAD_INITIALIZER(trace.args), };lib/librte_eal/common/eal_common_trace.c +11
-9@@ -429,32 +428,33 @@ trace_mem_free(void) void __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype) { - char *field = RTE_PER_LCORE(ctf_field); -int count =RTE_PER_LCORE (ctf_count); -size_t size; + char *field;char *fixup;int rc; - size =RTE_MAX (0 ,TRACE_CTF_FIELD_SIZE -1 - count); -RTE_PER_LCORE (trace_point_sz) += sz; fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =snprintf (RTE_PTR_ADD (field, count), size,"%s %s;" , datatype, in); + rc =asprintf (&field,"%s%s %s;" , +RTE_PER_LCORE (ctf_field) !=NULL ? +RTE_PER_LCORE (ctf_field) :"" , + datatype, in); +free (RTE_PER_LCORE (ctf_field));free (fixup); -if (rc <=0 || (size_t )rc >= size) { +if (rc ==-1 ) {RTE_PER_LCORE (trace_point_sz) =0 ; -trace_crit ("CTF field is too long" ); +RTE_PER_LCORE (ctf_field) =NULL ; +trace_crit ("could not allocate CTF field" );return ; } -RTE_PER_LCORE (ctf_count) += rc; +RTE_PER_LCORE (trace_point_sz) += sz; +RTE_PER_LCORE (ctf_field) = field; }int
__rte_trace_point_register(rte_trace_point_t *handle, const char *name,void (*register_fn)(void)){
lib/librte_eal/common/eal_common_trace.c +0
-1-char *field =RTE_PER_LCORE (ctf_field);struct trace_point *tp; uint16_t sz;lib/librte_eal/common/eal_common_trace.c +0
-1@@ -467,7 +467,6 @@ __rte_trace_point_register(rte_trace_point_t *handle,const char *name,/* Check the size of the trace point object */ RTE_PER_LCORE (trace_point_sz ) =0 ; -RTE_PER_LCORE (ctf_count) =0 ;register_fn ();if (RTE_PER_LCORE (trace_point_sz) ==0 ) {trace_err("missing rte_trace_emit_header() in register fn");lib/librte_eal/common/eal_common_trace.c +0
-4@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ;
lib/librte_eal/common/eal_common_trace.c __rte_trace_point_register(rte_trace_point_t · line 512
Quoted 10 lines
lib/librte_eal/common/eal_common_trace.c +5
-5- goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE ); +/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;
Although patch looks okay but I have one that how "tp->ctf_field" is populated because during registration time RTE_PER_LCORE(ctf_field) will be NULL. So "tp->ctf_field" will always be NULL.
Quoted 41 lines
lib/librte_eal/common/eal_common_trace.c +0
-0/* Form the trace handle */ *handle = sz;lib/librte_eal/common/eal_common_trace_ctf.c +2
-1@@ -233,7 +233,8 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" fields := struct {\n" " %s\n" " };\n" - "};\n\n", trace_id_get(tp->handle), tp->name, tp->ctf_field); +"};\n\n" ,trace_id_get (tp->handle), tp->name, + tp->ctf_field != NULL ? tp->ctf_field : "");return meta_copy (meta, offset, str, rc); }lib/librte_eal/common/eal_trace.h +0
-0@@ -24,7 +24,6 @@#define TRACE_PREFIX_LEN 12#define TRACE_DIR_STR_LEN (sizeof("YYYY-mm-dd-AM-HH-MM-SS") +
TRACE_PREFIX_LEN) -#defineTRACE_CTF_FIELD_SIZE448 #defineTRACE_POINT_NAME_SIZE64 #defineTRACE_CTF_MAGIC0xC1FC1FC1
#defineTRACE_MAX_ARGS32lib/librte_eal/common/eal_trace.h +1
-2@@ -33,7 +32,7 @@ struct trace_point {STAILQ_ENTRY (trace_point) next;rte_trace_point_t *handle;char name[TRACE_POINT_NAME_SIZE ]; -char ctf_field[TRACE_CTF_FIELD_SIZE ]; +char *ctf_field; };enum trace_area_e { --2.23.0
Re: [dpdk-dev] [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event description
Quoted 14 lines · Sunil Kumar Kori
Quoted 14 lines
lib/librte_eal/common/eal_common_trace.c +0
-9@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ; - goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE );
Quoted 8 lines
lib/librte_eal/common/eal_common_trace.c __rte_trace_point_register(rte_trace_point_t *handle, const char *name, · line 511
Quoted 5 lines
lib/librte_eal/common/eal_common_trace.c +5
-0+/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;Although patch looks okay but I have one that how "
tp->ctf_field" is populated because during registration timeRTE_PER_LCORE(ctf_field)will be NULL. So "tp->ctf_field" will always be NULL.
Sorry, I don't understand your comment. RTE_PER_LCORE(ctf_field) is filled at __rte_trace_point_emit_field.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event description
Quoted 32 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 2:22 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event descriptionQuoted 14 lines · Sunil Kumar Kori
Quoted 14 lines
lib/librte_eal/common/eal_common_trace.c +0
-9@@ -505,15 +504,11 @@ __rte_trace_point_register(rte_trace_point_t *handle, const char *name, goto free; } -/* Copy the field data for future use */ -if (rte_strscpy (tp->ctf_field, field,TRACE_CTF_FIELD_SIZE ) <0 ) { -trace_err ("CTF field size is too long" ); - rte_errno =E2BIG ; - goto free; - } - -/* Clear field memory for the next event */ -memset (field,0 ,TRACE_CTF_FIELD_SIZE );Quoted 8 lines
lib/librte_eal/common/eal_common_trace.c __rte_trace_point_register(rte_trace_point_t *handle, const char *name, · line 511
Quoted 5 lines
lib/librte_eal/common/eal_common_trace.c +5
-0+/* Copy the accumulated fields description and clear it for the next +* trace point. +*/ + tp->ctf_field =RTE_PER_LCORE (ctf_field); +RTE_PER_LCORE (ctf_field) =NULL ;Although patch looks okay but I have one that how "
tp->ctf_field" is populated because during registration timeRTE_PER_LCORE(ctf_field)will
Quoted 4 lines
be NULL. So "
tp->ctf_field" will always be NULL.Sorry, I don't understand your comment.
RTE_PER_LCORE(ctf_field)is filled at__rte_trace_point_emit_field.
Yes but I am not able to understand that how tp->ctf_field will be populated with latest memory because RTE_PER_LCORE(ctf_field) is being free and re-allocated at runtime.
Quoted 4 lines
--
David Marchand
Re: [dpdk-dev] [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event description
Quoted 2 lines · Sunil Kumar Kori
Yes but I am not able to understand that how
tp->ctf_fieldwill be populated with latest memory becauseRTE_PER_LCORE(ctf_field)is being free and re-allocated at runtime.
Initially, RTE_PER_LCORE(ctf_field) is NULL.
Then registration for a trace point happens:
__rte_trace_point_register() calls register_fn() which for each field in the trace point calls __rte_trace_point_emit_field().
Each call to __rte_trace_point_emit_field accumulates the previous ctf_field with the new field.
rc = asprintf(&field, "%s %s %s;\n",
RTE_PER_LCORE(ctf_field) != NULL ?
RTE_PER_LCORE(ctf_field) : "",
datatype, in);
free(RTE_PER_LCORE(ctf_field));
RTE_PER_LCORE(ctf_field) = field;Here, RTE_PER_LCORE(ctf_field) is != NULL.
Back to __rte_trace_point_register:
/* Copy the accumulated fields description and clear it for the next
* trace point.
*/
tp->ctf_field = RTE_PER_LCORE(ctf_field);Once stored, RTE_PER_LCORE(ctf_field) is cleared again for the next trace point registration.
RTE_PER_LCORE(ctf_field) = NULL;
Signature · 1 line
David Marchand
Sunil Kumar Kori
Acked-byRe: [dpdk-dev] [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event description
Quoted 40 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 3:33 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [EXT] [PATCH v2 2/4] trace: remove size limit on CTF event descriptionQuoted 2 lines · Sunil Kumar Kori
Yes but I am not able to understand that how
tp->ctf_fieldwill be populated with latest memory becauseRTE_PER_LCORE(ctf_field)is beingfree and re-allocated at runtime.
Initially,
RTE_PER_LCORE(ctf_field)is NULL.Then registration for a trace point happens:
__rte_trace_point_register()callsregister_fn()which for each field in the trace point calls__rte_trace_point_emit_field().Each call to
__rte_trace_point_emit_fieldaccumulates the previousctf_fieldwith the new field.rc = asprintf(&field, "%s %s %s;\n", RTE_PER_LCORE(ctf_field) != NULL ? RTE_PER_LCORE(ctf_field) : "", datatype, in); free(RTE_PER_LCORE(ctf_field)); RTE_PER_LCORE(ctf_field) = field;Here,
RTE_PER_LCORE(ctf_field)is != NULL.Back to
__rte_trace_point_register:/* Copy the accumulated fields description and clear it for the next * trace point. */ tp->ctf_field = RTE_PER_LCORE(ctf_field);
Quoted 7 lines
Once stored,
RTE_PER_LCORE(ctf_field)is cleared again for the next trace point registration.RTE_PER_LCORE(ctf_field) = NULL;
Quoted 2 lines
--
David Marchand
PATCH v2 3/4 trace: fix metadata dump
In reply to David MarchandThe ctf metadata is written to the metadata file without any check for length, so this string must be null terminated.
--- lib/librte_eal/common/eal_common_trace_ctf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
lib/librte_eal/common/eal_common_trace_ctf.c +2 -1
@@ -37,11 +37,12 @@ meta_copy(char **meta, int *offset, char *str, int rc)if (rc <0 )return rc; - ptr =realloc (ptr, count + rc); + ptr =realloc (ptr, count + rc +1 );if (ptr ==NULL ) goto free_str;memcpy (RTE_PTR_ADD (ptr, count), str, rc); + ptr[count + rc] ='\0' ; count += rc;free (str);
Signature · 1 line
2.23.0
Re: [dpdk-dev] [EXT] [PATCH v2 3/4] trace: fix metadata dump
Quoted 41 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 2:33 AM
To: [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; Sunil Kumar Kori <[email protected]>
Subject: [EXT] [PATCH v2 3/4] trace: fix metadata dumpExternal Email
----------------------------------------------------------------------
The ctf metadata is written to the metadata file without any check for length, so this string must be null terminated.Fixes f1a099f5b1f1 ("trace: create CTF TDSL metadata in memory")Signed-off-by David Marchand <[email protected]>---
lib/librte_eal/common/eal_common_trace_ctf.c| 3 ++-1 file changed, 2 insertions(+), 1 deletion(-)lib/librte_eal/common/eal_common_trace_ctf.c +2
-2@@ -37,11 +37,12 @@ meta_copy(char **meta, int *offset, char *str, int rc)if (rc <0 )return rc; - ptr =realloc (ptr, count + rc); + ptr =realloc (ptr, count + rc +1 );if (ptr ==NULL ) goto free_str;memcpy (RTE_PTR_ADD (ptr, count), str, rc); + ptr[count + rc] ='\0' ; count += rc;free (str); --
Quoted 1 line
2.23.0
PATCH v2 4/4 trace: make CTF metadata prettier
In reply to David MarchandThis is simply a cosmetic change.
Before:
event {
id = 17;
name = "lib.eal.alarm.set";
fields := struct {
uint64_t us;uintptr_t cb_fn;uintptr_t cb_arg;int32_t rc;
};
};After:
event {
id = 17;
name = "lib.eal.alarm.set";
fields := struct {
uint64_t us;
uintptr_t cb_fn;
uintptr_t cb_arg;
int32_t rc;
};
};--- lib/librte_eal/common/eal_common_trace.c | 2 +- lib/librte_eal/common/eal_common_trace_ctf.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_trace.c +1 -1
@@ -435,7 +435,7 @@ __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype) fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc =asprintf (&field,"%s%s %s;" , + rc =asprintf (&field,"%s %s %s;\n" ,RTE_PER_LCORE (ctf_field) !=NULL ?RTE_PER_LCORE (ctf_field) :"" , datatype, in);
lib/librte_eal/common/eal_common_trace_ctf.c +1 -1
@@ -232,7 +232,7 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" id = %d;\n" " name = \"%s\";\n" " fields := struct {\n" -" %s\n" +"%s" " };\n" "};\n\n" ,trace_id_get (tp->handle), tp->name, tp->ctf_field != NULL ? tp->ctf_field : "");
Signature · 1 line
2.23.0
Re: [dpdk-dev] [EXT] [PATCH v2 4/4] trace: make CTF metadata prettier
Quoted 70 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, October 29, 2020 2:33 AM
To: [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; Sunil Kumar Kori <[email protected]>
Subject: [EXT] [PATCH v2 4/4] trace: make CTF metadata prettierExternal Email
----------------------------------------------------------------------
This is simply a cosmetic change.Before:
event { id = 17; name = "lib.eal.alarm.set"; fields := struct { uint64_t us;uintptr_t cb_fn;uintptr_t cb_arg;int32_t rc; }; };After:
event { id = 17; name = "lib.eal.alarm.set"; fields := struct { uint64_t us; uintptr_t cb_fn; uintptr_t cb_arg; int32_t rc; }; };Signed-off-by David Marchand <[email protected]>---
lib/librte_eal/common/eal_common_trace.c| 2 +-lib/librte_eal/common/eal_common_trace_ctf.c| 2 +-2 files changed, 2 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_trace.c +1
-1@@ -435,7 +435,7 @@ __rte_trace_point_emit_field(size_t sz, const char *in, const char *datatype) fixup =trace_metadata_fixup_field (in);if (fixup !=NULL ) in = fixup; - rc = asprintf(&field, "%s%s %s;", + rc = asprintf(&field, "%s %s %s;\n", RTE_PER_LCORE(ctf_field) != NULL ? RTE_PER_LCORE(ctf_field) : "",datatype, in);lib/librte_eal/common/eal_common_trace_ctf.c +1
-1@@ -232,7 +232,7 @@ meta_event_emit(char **meta, int *offset, struct trace_point *tp)" id = %d;\n" " name = \"%s\";\n" " fields := struct {\n" -" %s\n" +"%s" " };\n" "};\n\n", trace_id_get(tp->handle), tp->name,tp->ctf_field != NULL ? tp->ctf_field : "");
Quoted 2 lines
--
2.23.0
David Marchand
AppliedQuoted 12 lines · David Marchand
Following recent increase of an internal array that was limiting CTF event descriptions, I had a second look at the code.
All of this is slow path, so I see no reason in keeping this limitation and we can go with dynamic allocations.David Marchand (4):
trace: fixup CTF event description at registration
trace: remove size limit on CTF event description
trace: fix metadata dump
trace: make CTF metadata prettier
Quoted 4 lines
lib/librte_eal/common/eal_common_trace.c | 44 ++--- lib/librte_eal/common/eal_common_trace_ctf.c | 167 +++++-------------- lib/librte_eal/common/eal_trace.h | 4 +- 3 files changed, 69 insertions(+), 146 deletions(-)
Applied.
Signature · 1 line
David Marchand