rtla: Update for tools for 6.14

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/Makefile         |  4 ++-
 tools/tracing/rtla/tests/engine.sh  | 48 +++++++++++++++++++++++++++++
 tools/tracing/rtla/tests/hwnoise.t  | 21 +++++++++++++
 tools/tracing/rtla/tests/osnoise.t  | 19 ++++++++++++
 tools/tracing/rtla/tests/timerlat.t | 27 ++++++++++++++++
 5 files changed, 118 insertions(+), 1 deletion(-)
 create mode 100644 tools/tracing/rtla/tests/engine.sh
 create mode 100644 tools/tracing/rtla/tests/hwnoise.t
 create mode 100644 tools/tracing/rtla/tests/osnoise.t
 create mode 100644 tools/tracing/rtla/tests/timerlat.t

tools/tracing/rtla/Makefile +3 -1

index a6a7dee16622..8b5101457c70 100644
@@ -85,4 +85,6 @@ clean: doc_clean fixdep-clean
 	$(Q)find . -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
 	$(Q)rm -f rtla rtla-static fixdep FEATURE-DUMP rtla-*
 	$(Q)rm -rf feature
-.PHONY: FORCE clean
+check: $(RTLA)
+	RTLA=$(RTLA) prove -o -f tests/
+.PHONY: FORCE clean check

tools/tracing/rtla/tests/engine.sh new file +48 -0

48 lines not on this page · raw message

tools/tracing/rtla/tests/hwnoise.t new file +21 -0

new file mode 100644
index 000000000000..bbed17580537
@@ -0,0 +1,21 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+source tests/engine.sh
+test_begin
+
+set_timeout 2m
+
+check "verify help page" \
+	"hwnoise --help"
+check "detect noise higher than one microsecond" \
+	"hwnoise -c 0 -T 1 -d 5s -q"
+check "set the automatic trace mode" \
+	"hwnoise -a 5 -d 30s"
+check "set scheduling param to the osnoise tracer threads" \
+	"hwnoise -P F:1 -c 0 -r 900000 -d 1M -q"
+check "stop the trace if a single sample is higher than 1 us" \
+	"hwnoise -s 1 -T 1 -t -d 30s"
+check "enable a trace event trigger" \
+	"hwnoise -t -e osnoise:irq_noise trigger=\"hist:key=desc,duration:sort=desc,duration:vals=hitcount\" -d 1m"
+
+test_end

tools/tracing/rtla/tests/osnoise.t new file +19 -0

new file mode 100644
index 000000000000..86596e547893
@@ -0,0 +1,19 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+source tests/engine.sh
+test_begin
+
+set_timeout 2m
+
+check "verify help page" \
+	"osnoise --help"
+check "verify the --priority/-P param" \
+	"osnoise top -P F:1 -c 0 -r 900000 -d 1M -q"
+check "verify the --stop/-s param" \
+	"osnoise top -s 30 -T 1 -t"
+check "verify the  --trace param" \
+	"osnoise hist -s 30 -T 1 -t"
+check "verify the --entries/-E param" \
+	"osnoise hist -P F:1 -c 0 -r 900000 -d 1M -b 10 -E 25"
+
+test_end

tools/tracing/rtla/tests/timerlat.t new file +27 -0

new file mode 100644
index 000000000000..e86f40e5749e
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+source tests/engine.sh
+test_begin
+
+set_timeout 2m
+
+check "verify help page" \
+	"timerlat --help"
+check "verify -s/--stack" \
+	"timerlat top -s 3 -T 10 -t"
+check "verify -P/--priority" \
+	"timerlat top -P F:1 -c 0 -d 1M -q"
+check "test in nanoseconds" \
+	"timerlat top -i 2 -c 0 -n -d 30s"
+check "set the automatic trace mode" \
+	"timerlat top -a 5 --dump-tasks"
+check "print the auto-analysis if hits the stop tracing condition" \
+	"timerlat top --aa-only 5"
+check "disable auto-analysis" \
+	"timerlat top -s 3 -T 10 -t --no-aa"
+check "verify -c/--cpus" \
+	"timerlat hist -c 0 -d 30s"
+check "hist test in nanoseconds" \
+	"timerlat hist -i 2 -c 0 -n -d 30s"
+
+test_end

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #2 lore raw

PATCH 2/14 rtla: Add trace_instance_stop

From: Tomas Glozar <[email protected]>

Support not only turning trace on for the timerlat tracer, but also turning it off.

This will be used in subsequent patches to stop the timerlat tracer without also wiping the trace buffer.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/trace.c | 8 ++++++++
 tools/tracing/rtla/src/trace.h | 1 +
 2 files changed, 9 insertions(+)

tools/tracing/rtla/src/trace.c +8 -0

index 170a706248ab..440323a997c6 100644
@@ -196,6 +196,14 @@ int trace_instance_start(struct trace_instance *trace)
 	return tracefs_trace_on(trace->inst);
 }
 
+/*
+ * trace_instance_stop - stop tracing a given rtla instance
+ */
+int trace_instance_stop(struct trace_instance *trace)
+{
+	return tracefs_trace_off(trace->inst);
+}
+
 /*
  * trace_events_free - free a list of trace events
  */

tools/tracing/rtla/src/trace.h +1 -0

index c7c92dc9a18a..76e1b77291ba 100644
@@ -21,6 +21,7 @@ struct trace_instance {
 
 int trace_instance_init(struct trace_instance *trace, char *tool_name);
 int trace_instance_start(struct trace_instance *trace);
+int trace_instance_stop(struct trace_instance *trace);
 void trace_instance_destroy(struct trace_instance *trace);
 
 struct trace_seq *get_trace_seq(void);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #3 lore raw

PATCH 3/14 rtla/timerlat_hist: Stop timerlat tracer on signal

From: Tomas Glozar <[email protected]>

Currently, when either SIGINT from the user or SIGALRM from the duration timer is caught by rtla-timerlat, stop_tracing is set to break out of the main loop. This is not sufficient for cases where the timerlat tracer is producing more data than rtla can consume, since in that case, rtla is looping indefinitely inside tracefs_iterate_raw_events, never reaches the check of stop_tracing and hangs.

In addition to setting stop_tracing, also stop the timerlat tracer on received signal (SIGINT or SIGALRM). This will stop new samples so that the existing samples may be processed and tracefs_iterate_raw_events eventually exits.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Fixes 1eeb6328e8b3 ("rtla/timerlat: Add timerlat hist mode")

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_hist.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

tools/tracing/rtla/src/timerlat_hist.c +10 -1

index 8b66387e5f35..f1edf1c8a7b0 100644
@@ -1131,9 +1131,12 @@ static struct osnoise_tool
 }
 
 static int stop_tracing;
+static struct trace_instance *hist_inst = NULL;
 static void stop_hist(int sig)
 {
 	stop_tracing = 1;
+	if (hist_inst)
+		trace_instance_stop(hist_inst);
 }
 
 /*

@@ -1180,6 +1183,12 @@ int timerlat_hist_main(int argc, char *argv[])
 	}
 
 	trace = &tool->trace;
+	/*
+	 * Save trace instance into global variable so that SIGINT can stop
+	 * the timerlat tracer.
+	 * Otherwise, rtla could loop indefinitely when overloaded.
+	 */
+	hist_inst = trace;
 
 	retval = enable_timerlat(trace);
 	if (retval) {

@@ -1348,7 +1357,7 @@ int timerlat_hist_main(int argc, char *argv[])
 
 	return_value = 0;
 
-	if (trace_is_off(&tool->trace, &record->trace)) {
+	if (trace_is_off(&tool->trace, &record->trace) && !stop_tracing) {
 		printf("rtla timerlat hit stop tracing\n");
 
 		if (!params->no_aa)

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #4 lore raw

PATCH 4/14 rtla/timerlat_top: Stop timerlat tracer on signal

From: Tomas Glozar <[email protected]>

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Fixes a828cd18bc4a ("rtla: Add timerlat tool and timelart top mode")

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_top.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

tools/tracing/rtla/src/timerlat_top.c +11 -1

index 059b468981e4..d21a21053917 100644
@@ -900,9 +900,12 @@ static struct osnoise_tool
 }
 
 static int stop_tracing;
+static struct trace_instance *top_inst = NULL;
 static void stop_top(int sig)
 {
 	stop_tracing = 1;
+	if (top_inst)
+		trace_instance_stop(top_inst);
 }
 
 /*

@@ -950,6 +953,13 @@ int timerlat_top_main(int argc, char *argv[])
 	}
 
 	trace = &top->trace;
+	/*
+	* Save trace instance into global variable so that SIGINT can stop
+	* the timerlat tracer.
+	* Otherwise, rtla could loop indefinitely when overloaded.
+	*/
+	top_inst = trace;
+
 
 	retval = enable_timerlat(trace);
 	if (retval) {

@@ -1131,7 +1141,7 @@ int timerlat_top_main(int argc, char *argv[])
 
 	return_value = 0;
 
-	if (trace_is_off(&top->trace, &record->trace)) {
+	if (trace_is_off(&top->trace, &record->trace) && !stop_tracing) {
 		printf("rtla timerlat hit stop tracing\n");
 
 		if (!params->no_aa)

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #5 lore raw

PATCH 5/14 rtla/timerlat_hist: Abort event processing on second signal

From: Tomas Glozar <[email protected]>

If either SIGINT is received twice, or after a SIGALRM (that is, after timerlat was supposed to stop), abort processing events currently left in the tracefs buffer and exit immediately.

This allows the user to exit rtla without waiting for processing all events, should that take longer than wanted, at the cost of not processing all samples.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_hist.c | 8 ++++++++
 1 file changed, 8 insertions(+)

tools/tracing/rtla/src/timerlat_hist.c +8 -0

index f1edf1c8a7b0..53ded187b33a 100644
@@ -1134,6 +1134,14 @@ static int stop_tracing;
 static struct trace_instance *hist_inst = NULL;
 static void stop_hist(int sig)
 {
+	if (stop_tracing) {
+		/*
+		 * Stop requested twice in a row; abort event processing and
+		 * exit immediately
+		 */
+		tracefs_iterate_stop(hist_inst->inst);
+		return;
+	}
 	stop_tracing = 1;
 	if (hist_inst)
 		trace_instance_stop(hist_inst);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #6 lore raw

PATCH 6/14 rtla/timerlat_top: Abort event processing on second signal

From: Tomas Glozar <[email protected]>

If either SIGINT is received twice, or after a SIGALRM (that is, after timerlat was supposed to stop), abort processing events currently left in the tracefs buffer and exit immediately.

This allows the user to exit rtla without waiting for processing all events, should that take longer than wanted, at the cost of not processing all samples.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_top.c | 8 ++++++++
 1 file changed, 8 insertions(+)

tools/tracing/rtla/src/timerlat_top.c +8 -0

index d21a21053917..d358cd39f360 100644
@@ -903,6 +903,14 @@ static int stop_tracing;
 static struct trace_instance *top_inst = NULL;
 static void stop_top(int sig)
 {
+	if (stop_tracing) {
+		/*
+		 * Stop requested twice in a row; abort event processing and
+		 * exit immediately
+		 */
+		tracefs_iterate_stop(top_inst->inst);
+		return;
+	}
 	stop_tracing = 1;
 	if (top_inst)
 		trace_instance_stop(top_inst);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #7 lore raw

PATCH 7/14 rtla/osnoise: Distinguish missing workload option

From: Tomas Glozar <[email protected]>

osnoise_set_workload returns -1 for both missing OSNOISE_WORKLOAD option and failure in setting the option.

Return -1 for missing and -2 for failure to distinguish them.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/osnoise.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

tools/tracing/rtla/src/osnoise.c +1 -1

index 245e9344932b..699a83f538a8 100644
@@ -867,7 +867,7 @@ int osnoise_set_workload(struct osnoise_context *context, bool onoff)
 
 	retval = osnoise_options_set_option("OSNOISE_WORKLOAD", onoff);
 	if (retval < 0)
-		return -1;
+		return -2;
 
 	context->opt_workload = onoff;

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #8 lore raw

PATCH 8/14 rtla/timerlat_hist: Set OSNOISE_WORKLOAD for kernel threads

From: Tomas Glozar <[email protected]>

When using rtla timerlat with userspace threads (-u or -U), rtla disables the OSNOISE_WORKLOAD option in /sys/kernel/tracing/osnoise/options. This option is not re-enabled in a subsequent run with kernel-space threads, leading to rtla collecting no results if the previous run exited abnormally:

$ rtla timerlat hist -u
^\Quit (core dumped)
$ rtla timerlat hist -k -d 1s
Index
over:
count:
min:
avg:
max:
ALL:        IRQ       Thr       Usr
count:        0         0         0
min:          -         -         -
avg:          -         -         -
max:          -         -         -

The issue persists until OSNOISE_WORKLOAD is set manually by running:

$ echo OSNOISE_WORKLOAD > /sys/kernel/tracing/osnoise/options

Set OSNOISE_WORKLOAD when running rtla with kernel-space threads if available to fix the issue.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Fixes ed774f7481fa ("rtla/timerlat_hist: Add timerlat user-space support")

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_hist.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

tools/tracing/rtla/src/timerlat_hist.c +9 -6

index 53ded187b33a..d4bd02c01c53 100644
@@ -1085,12 +1085,15 @@ timerlat_hist_apply_config(struct osnoise_tool *tool, struct timerlat_hist_param
 		}
 	}
 
-	if (params->user_hist) {
-		retval = osnoise_set_workload(tool->context, 0);
-		if (retval) {
-			err_msg("Failed to set OSNOISE_WORKLOAD option\n");
-			goto out_err;
-		}
+	/*
+	* Set workload according to type of thread if the kernel supports it.
+	* On kernels without support, user threads will have already failed
+	* on missing timerlat_fd, and kernel threads do not need it.
+	*/
+	retval = osnoise_set_workload(tool->context, params->kernel_workload);
+	if (retval < -1) {
+		err_msg("Failed to set OSNOISE_WORKLOAD option\n");
+		goto out_err;
 	}
 
 	return 0;

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #9 lore raw

PATCH 9/14 rtla/timerlat_top: Set OSNOISE_WORKLOAD for kernel threads

From: Tomas Glozar <[email protected]>

$ rtla timerlat top -u
^\Quit (core dumped)
$ rtla timerlat top -k -d 1s
                                     Timer Latency
  0 00:00:01   |          IRQ Timer Latency (us)        |         Thread Timer Latency (us)
CPU COUNT      |      cur       min       avg       max |      cur       min       avg       max

The issue persists until OSNOISE_WORKLOAD is set manually by running:

$ echo OSNOISE_WORKLOAD > /sys/kernel/tracing/osnoise/options

Set OSNOISE_WORKLOAD when running rtla with kernel-space threads if available to fix the issue.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Fixes cdca4f4e5e8e ("rtla/timerlat_top: Add timerlat user-space support")

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/timerlat_top.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

tools/tracing/rtla/src/timerlat_top.c +9 -6

index d358cd39f360..f387597d3ac2 100644
@@ -851,12 +851,15 @@ timerlat_top_apply_config(struct osnoise_tool *top, struct timerlat_top_params *
 		}
 	}
 
-	if (params->user_top) {
-		retval = osnoise_set_workload(top->context, 0);
-		if (retval) {
-			err_msg("Failed to set OSNOISE_WORKLOAD option\n");
-			goto out_err;
-		}
+	/*
+	* Set workload according to type of thread if the kernel supports it.
+	* On kernels without support, user threads will have already failed
+	* on missing timerlat_fd, and kernel threads do not need it.
+	*/
+	retval = osnoise_set_workload(top->context, params->kernel_workload);
+	if (retval < -1) {
+		err_msg("Failed to set OSNOISE_WORKLOAD option\n");
+		goto out_err;
 	}
 
 	if (isatty(STDOUT_FILENO) && !params->quiet)

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #10 lore raw

PATCH 10/14 tools/rtla: Add osnoise_trace_is_off()

From: Costa Shulyupin <[email protected]>

All of the users of trace_is_off() passes in &record->trace as the second parameter, where record is a pointer to a struct osnoise_tool. This record could be NULL and there is a hidden dependency that the trace field is the first field to allow &record->trace to work with a NULL record pointer.

In order to make this code a bit more robust, as record shouldn't be dereferenced if it is NULL, even if the code does work, create a new function called osnoise_trace_is_off() that takes the pointer to a struct osnoise_tool as its second parameter. This way it can properly test if it is NULL before it dereferences it.

The old function trace_is_off() is removed and the function osnoise_trace_is_off() is added into osnoise.c which is what the struct osnoise_tool is associated with.

Cc John Kacur <[email protected]>

Cc "Luis Claudio R. Goncalves" <[email protected]>

Cc Eder Zulian <[email protected]>

Cc Dan Carpenter <[email protected]>

Cc Tomas Glozar <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Costa Shulyupin <[email protected]>

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/osnoise.c       | 16 ++++++++++++++++
 tools/tracing/rtla/src/osnoise.h       |  1 +
 tools/tracing/rtla/src/osnoise_hist.c  |  4 ++--
 tools/tracing/rtla/src/osnoise_top.c   |  4 ++--
 tools/tracing/rtla/src/timerlat_hist.c |  4 ++--
 tools/tracing/rtla/src/timerlat_top.c  |  6 +++---
 tools/tracing/rtla/src/trace.c         | 19 -------------------
 tools/tracing/rtla/src/trace.h         |  1 -
 8 files changed, 26 insertions(+), 29 deletions(-)

tools/tracing/rtla/src/osnoise.c +16 -0

index 699a83f538a8..f521f052cbd3 100644
@@ -1079,6 +1079,22 @@ struct osnoise_tool *osnoise_init_trace_tool(char *tracer)
 	return NULL;
 }
 
+bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record)
+{
+	/*
+	 * The tool instance is always present, it is the one used to collect
+	 * data.
+	 */
+	if (!tracefs_trace_is_on(tool->trace.inst))
+		return true;
+
+	/*
+	 * The trace record instance is only enabled when -t is set. IOW, when the system
+	 * is tracing.
+	 */
+	return record && !tracefs_trace_is_on(record->trace.inst);
+}
+
 static void osnoise_usage(int err)
 {
 	int i;

tools/tracing/rtla/src/osnoise.h +1 -0

index 555f4f4903cc..1dc188baddef 100644
@@ -104,6 +104,7 @@ struct osnoise_tool {
 void osnoise_destroy_tool(struct osnoise_tool *top);
 struct osnoise_tool *osnoise_init_tool(char *tool_name);
 struct osnoise_tool *osnoise_init_trace_tool(char *tracer);
+bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record);
 
 int osnoise_hist_main(int argc, char *argv[]);
 int osnoise_top_main(int argc, char **argv);

tools/tracing/rtla/src/osnoise_hist.c +2 -2

index 214e2c93fde0..f250f999a4ee 100644
@@ -970,7 +970,7 @@ int osnoise_hist_main(int argc, char *argv[])
 			goto out_hist;
 		}
 
-		if (trace_is_off(&tool->trace, &record->trace))
+		if (osnoise_trace_is_off(tool, record))
 			break;
 	}

@@ -980,7 +980,7 @@ int osnoise_hist_main(int argc, char *argv[])
 
 	return_value = 0;
 
-	if (trace_is_off(&tool->trace, &record->trace)) {
+	if (osnoise_trace_is_off(tool, record)) {
 		printf("rtla osnoise hit stop tracing\n");
 		if (params->trace_output) {
 			printf("  Saving trace to %s\n", params->trace_output);

tools/tracing/rtla/src/osnoise_top.c +2 -2

index 45647495ce3b..6d50653ae224 100644
@@ -801,7 +801,7 @@ int osnoise_top_main(int argc, char **argv)
 		if (!params->quiet)
 			osnoise_print_stats(params, tool);
 
-		if (trace_is_off(&tool->trace, &record->trace))
+		if (osnoise_trace_is_off(tool, record))
 			break;
 
 	}

@@ -810,7 +810,7 @@ int osnoise_top_main(int argc, char **argv)
 
 	return_value = 0;
 
-	if (trace_is_off(&tool->trace, &record->trace)) {
+	if (osnoise_trace_is_off(tool, record)) {
 		printf("osnoise hit stop tracing\n");
 		if (params->trace_output) {
 			printf("  Saving trace to %s\n", params->trace_output);

tools/tracing/rtla/src/timerlat_hist.c +2 -2

index d4bd02c01c53..91aedb44da01 100644
@@ -1347,7 +1347,7 @@ int timerlat_hist_main(int argc, char *argv[])
 			goto out_hist;
 		}
 
-		if (trace_is_off(&tool->trace, &record->trace))
+		if (osnoise_trace_is_off(tool, record))
 			break;
 
 		/* is there still any user-threads ? */

@@ -1368,7 +1368,7 @@ int timerlat_hist_main(int argc, char *argv[])
 
 	return_value = 0;
 
-	if (trace_is_off(&tool->trace, &record->trace) && !stop_tracing) {
+	if (osnoise_trace_is_off(tool, record) && !stop_tracing) {
 		printf("rtla timerlat hit stop tracing\n");
 
 		if (!params->no_aa)

tools/tracing/rtla/src/timerlat_top.c +3 -3

index f387597d3ac2..51115f92e15e 100644
@@ -1114,7 +1114,7 @@ int timerlat_top_main(int argc, char *argv[])
 	while (!stop_tracing) {
 		sleep(params->sleep_time);
 
-		if (params->aa_only && !trace_is_off(&top->trace, &record->trace))
+		if (params->aa_only && !osnoise_trace_is_off(top, record))
 			continue;
 
 		retval = tracefs_iterate_raw_events(trace->tep,

@@ -1131,7 +1131,7 @@ int timerlat_top_main(int argc, char *argv[])
 		if (!params->quiet)
 			timerlat_print_stats(params, top);
 
-		if (trace_is_off(&top->trace, &record->trace))
+		if (osnoise_trace_is_off(top, record))
 			break;
 
 		/* is there still any user-threads ? */

@@ -1152,7 +1152,7 @@ int timerlat_top_main(int argc, char *argv[])
 
 	return_value = 0;
 
-	if (trace_is_off(&top->trace, &record->trace) && !stop_tracing) {
+	if (osnoise_trace_is_off(top, record) && !stop_tracing) {
 		printf("rtla timerlat hit stop tracing\n");
 
 		if (!params->no_aa)

tools/tracing/rtla/src/trace.c +0 ~~-19~~

index 440323a997c6..80b14b8a3c2e 100644
@@ -530,25 +530,6 @@ void trace_events_destroy(struct trace_instance *instance,
 	trace_events_free(events);
 }
 
-int trace_is_off(struct trace_instance *tool, struct trace_instance *trace)
-{
-	/*
-	 * The tool instance is always present, it is the one used to collect
-	 * data.
-	 */
-	if (!tracefs_trace_is_on(tool->inst))
-		return 1;
-
-	/*
-	 * The trace instance is only enabled when -t is set. IOW, when the system
-	 * is tracing.
-	 */
-	if (trace && !tracefs_trace_is_on(trace->inst))
-		return 1;
-
-	return 0;
-}
-
 /*
  * trace_set_buffer_size - set the per-cpu tracing buffer size.
  */

tools/tracing/rtla/src/trace.h +0 -1

index 76e1b77291ba..c3e03f7df770 100644
@@ -48,5 +48,4 @@ int trace_events_enable(struct trace_instance *instance,
 
 int trace_event_add_filter(struct trace_events *event, char *filter);
 int trace_event_add_trigger(struct trace_events *event, char *trigger);
-int trace_is_off(struct trace_instance *tool, struct trace_instance *trace);
 int trace_set_buffer_size(struct trace_instance *trace, int size);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #11 lore raw

PATCH 11/14 rtla: Count missed trace events

From: Tomas Glozar <[email protected]>

Add function collect_missed_events to trace.c to act as a callback for tracefs_follow_missed_events, summing the number of total missed events into a new field missing_events of struct trace_instance.

In case record->missed_events is negative, trace->missed_events is set to UINT64_MAX to signify an unknown number of events was missed.

The callback is activated on initialization of the trace instance.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/trace.c | 34 ++++++++++++++++++++++++++++++++++
 tools/tracing/rtla/src/trace.h |  1 +
 2 files changed, 35 insertions(+)

tools/tracing/rtla/src/trace.c +34 -0

index 80b14b8a3c2e..94e490782f14 100644
@@ -126,6 +126,31 @@ collect_registered_events(struct tep_event *event, struct tep_record *record,
 	return 0;
 }
 
+/*
+ * collect_missed_events - record number of missed events
+ *
+ * If rtla cannot keep up with events generated by tracer, events are going
+ * to fall out of the ring buffer.
+ * Collect how many events were missed so it can be reported to the user.
+ */
+static int
+collect_missed_events(struct tep_event *event, struct tep_record *record,
+		      int cpu, void *context)
+{
+	struct trace_instance *trace = context;
+
+	if (trace->missed_events == UINT64_MAX)
+		return 0;
+
+	if (record->missed_events > 0)
+		trace->missed_events += record->missed_events;
+	else
+		/* Events missed but no data on how many */
+		trace->missed_events = UINT64_MAX;
+
+	return 0;
+}
+
 /*
  * trace_instance_destroy - destroy and free a rtla trace instance
  */

@@ -181,6 +206,15 @@ int trace_instance_init(struct trace_instance *trace, char *tool_name)
 	 */
 	tracefs_trace_off(trace->inst);
 
+	/*
+	 * Collect the number of events missed due to tracefs buffer
+	 * overflow.
+	 */
+	trace->missed_events = 0;
+	tracefs_follow_missed_events(trace->inst,
+				     collect_missed_events,
+				     trace);
+
 	return 0;
 
 out_err:

tools/tracing/rtla/src/trace.h +1 -0

index c3e03f7df770..a6e88709604b 100644
@@ -17,6 +17,7 @@ struct trace_instance {
 	struct tracefs_instance		*inst;
 	struct tep_handle		*tep;
 	struct trace_seq		*seq;
+	unsigned long long		missed_events;
 };
 
 int trace_instance_init(struct trace_instance *trace, char *tool_name);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #12 lore raw

PATCH 12/14 rtla: Count all processed events

From: Tomas Glozar <[email protected]>

Add a field processed_events to struct trace_instance and increment it in collect_registered_events, regardless of whether a handler is registered for the event.

The purpose is to calculate the percentage of events that were missed due to tracefs buffer overflow.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/trace.c | 4 ++++
 tools/tracing/rtla/src/trace.h | 1 +
 2 files changed, 5 insertions(+)

tools/tracing/rtla/src/trace.c +4 -0

index 94e490782f14..728f5029d533 100644
@@ -118,6 +118,8 @@ collect_registered_events(struct tep_event *event, struct tep_record *record,
 	struct trace_instance *trace = context;
 	struct trace_seq *s = trace->seq;
 
+	trace->processed_events++;
+
 	if (!event->handler)
 		return 0;

@@ -215,6 +217,8 @@ int trace_instance_init(struct trace_instance *trace, char *tool_name)
 				     collect_missed_events,
 				     trace);
 
+	trace->processed_events = 0;
+
 	return 0;
 
 out_err:

tools/tracing/rtla/src/trace.h +1 -0

index a6e88709604b..3cd40dd3f06c 100644
@@ -18,6 +18,7 @@ struct trace_instance {
 	struct tep_handle		*tep;
 	struct trace_seq		*seq;
 	unsigned long long		missed_events;
+	unsigned long long		processed_events;
 };
 
 int trace_instance_init(struct trace_instance *trace, char *tool_name);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #13 lore raw

PATCH 13/14 rtla: Add function to report missed events

From: Tomas Glozar <[email protected]>

Add osnoise_report_missed_events to be used to report the number of missed events either during or after an osnoise or timerlat run. Also, display the percentage of missed events compared to the total number of received events.

If an unknown number of missed events was reported during the run, the entire number of missed events is reported as unknown.

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Cc Gabriele Monaco <[email protected]>

Link https://lore.kernel.org/[email protected] open here

Signed-off-by Steven Rostedt (Google) <[email protected]>

---
 tools/tracing/rtla/src/osnoise.c | 20 ++++++++++++++++++++
 tools/tracing/rtla/src/osnoise.h |  1 +
 2 files changed, 21 insertions(+)

tools/tracing/rtla/src/osnoise.c +20 -0

index f521f052cbd3..85f398b89597 100644
@@ -1095,6 +1095,26 @@ bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record
 	return record && !tracefs_trace_is_on(record->trace.inst);
 }
 
+/*
+ * osnoise_report_missed_events - report number of events dropped by trace
+ * buffer
+ */
+void
+osnoise_report_missed_events(struct osnoise_tool *tool)
+{
+	unsigned long long total_events;
+
+	if (tool->trace.missed_events == UINT64_MAX)
+		printf("unknown number of events missed, results might not be accurate\n");
+	else if (tool->trace.missed_events > 0) {
+		total_events = tool->trace.processed_events + tool->trace.missed_events;
+
+		printf("%lld (%.2f%%) events missed, results might not be accurate\n",
+		       tool->trace.missed_events,
+		       (double) tool->trace.missed_events / total_events * 100.0);
+	}
+}
+
 static void osnoise_usage(int err)
 {
 	int i;

tools/tracing/rtla/src/osnoise.h +1 -0

index 1dc188baddef..91835a7d8c2b 100644
@@ -104,6 +104,7 @@ struct osnoise_tool {
 void osnoise_destroy_tool(struct osnoise_tool *top);
 struct osnoise_tool *osnoise_init_tool(char *tool_name);
 struct osnoise_tool *osnoise_init_trace_tool(char *tracer);
+void osnoise_report_missed_events(struct osnoise_tool *tool);
 bool osnoise_trace_is_off(struct osnoise_tool *tool, struct osnoise_tool *record);
 
 int osnoise_hist_main(int argc, char *argv[]);

Signature · 1 line

2.45.2

Steven Rostedt

author maintainer

2025-01-24 18:48 UTC #14 lore raw

PATCH 14/14 rtla: Report missed event count

From: Tomas Glozar <[email protected]>

Print how many events were missed by trace buffer overflow in the main instance at the end of the run (for hist) or during the run (for top).

Cc John Kacur <[email protected]>

Cc Luis Goncalves <[email protected]>

Link https://lore.kernel.org/[email protected] open here