RTLA: An interface for osnoise/timerlat tracers

The rtla is a meta-tool that includes a set of commands that aims to analyze the real-time properties of Linux. But instead of testing Linux as a black box, rtla leverages kernel tracing capabilities to provide precise information about the properties and root causes of unexpected results.

rtla --help works and provide information about the available options.

This is a set of utils and tracer helper functions. They are used by rtla mostly to parse config, display data and some trace operations that are not part of libtracefs (because they are only useful it for this case).

Cc Steven Rostedt <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc Tom Zanussi <[email protected]>

Cc Masami Hiramatsu <[email protected]>

Cc Juri Lelli <[email protected]>

Cc Clark Williams <[email protected]>

Cc John Kacur <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Sebastian Andrzej Siewior <[email protected]>

Cc Daniel Bristot de Oliveira <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Signed-off-by Daniel Bristot de Oliveira <[email protected]>

---
 tools/tracing/rtla/src/trace.c | 192 +++++++++++++++
 tools/tracing/rtla/src/trace.h |  27 ++
 tools/tracing/rtla/src/utils.c | 433 +++++++++++++++++++++++++++++++++
 tools/tracing/rtla/src/utils.h |  56 +++++
 4 files changed, 708 insertions(+)
 create mode 100644 tools/tracing/rtla/src/trace.c
 create mode 100644 tools/tracing/rtla/src/trace.h
 create mode 100644 tools/tracing/rtla/src/utils.c
 create mode 100644 tools/tracing/rtla/src/utils.h

tools/tracing/rtla/src/trace.c new file +192 -0

new file mode 100644
index 000000000000..ab11c29cdcc2
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+#include <sys/sendfile.h>
+#include <tracefs.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "trace.h"
+#include "utils.h"
+
+/*
+ * enable_tracer_by_name - enable a tracer on the given instance
+ */
+int enable_tracer_by_name(struct tracefs_instance *inst, const char *tracer)
+{
+	enum tracefs_tracers t;
+	int retval;
+
+	t = TRACEFS_TRACER_CUSTOM;
+
+	debug_msg("enabling %s tracer\n", tracer);
+
+	retval = tracefs_tracer_set(inst, t, tracer);
+	if (retval < 0) {
+		if (errno == ENODEV)
+			err_msg("tracer %s not found!\n", tracer);
+
+		err_msg("failed to enable the tracer %s\n", tracer);
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * disable_tracer - set nop tracer to the insta
+ */
+void disable_tracer(struct tracefs_instance *inst)
+{
+	enum tracefs_tracers t = TRACEFS_TRACER_NOP;
+	int retval;
+
+	retval = tracefs_tracer_set(inst, t);
+	if (retval < 0)
+		err_msg("oops, error disabling tracer\n");
+}
+
+/*
+ * create_instance - create a trace instance with *instance_name
+ */
+struct tracefs_instance *create_instance(char *instance_name)
+{
+	return tracefs_instance_create(instance_name);
+}
+
+/*
+ * destroy_instance - remove a trace instance and free the data
+ */
+void destroy_instance(struct tracefs_instance *inst)
+{
+	tracefs_instance_destroy(inst);
+	tracefs_instance_free(inst);
+}
+
+/*
+ * save_trace_to_file - save the trace output of the instance to the file
+ */
+int save_trace_to_file(struct tracefs_instance *inst, const char *filename)
+{
+	const char *file = "trace";
+	mode_t mode = 0644;
+	char *buffer[4096];
+	int out_fd, in_fd;
+	int retval = -1;
+
+	in_fd = tracefs_instance_file_open(inst, file, O_RDONLY);
+	if (in_fd < 0) {
+		err_msg("Failed to open trace file\n");
+		return -1;
+	}
+
+	out_fd = creat(filename, mode);
+	if (out_fd < 0) {
+		err_msg("Failed to create output file %s\n", filename);
+		goto out_close_in;
+	}
+
+	do {
+		retval = read(in_fd, buffer, sizeof(buffer));
+		if (retval <= 0)
+			goto out_close;
+
+		retval = write(out_fd, buffer, retval);
+		if (retval < 0)
+			goto out_close;
+	} while (retval > 0);
+
+	retval = 0;
+out_close:
+	close(out_fd);
+out_close_in:
+	close(in_fd);
+	return retval;
+}
+
+/*
+ * collect_registered_events - call the existing callback function for the event
+ *
+ * If an event has a registered callback function, call it.
+ * Otherwise, ignore the event.
+ */
+int
+collect_registered_events(struct tep_event *event, struct tep_record *record,
+			  int cpu, void *context)
+{
+	struct trace_instance *trace = context;
+	struct trace_seq *s = trace->seq;
+
+	if (!event->handler)
+		return 0;
+
+	event->handler(s, record, event, context);
+
+	return 0;
+}
+
+/*
+ * trace_instance_destroy - destroy and free a rtla trace instance
+ */
+void trace_instance_destroy(struct trace_instance *trace)
+{
+	if (trace->inst) {
+		disable_tracer(trace->inst);
+		destroy_instance(trace->inst);
+	}
+
+	if (trace->seq)
+		free(trace->seq);
+
+	if (trace->tep)
+		tep_free(trace->tep);
+}
+
+/*
+ * trace_instance_init - create an rtla trace instance
+ *
+ * It is more than the tracefs instance, as it contains other
+ * things required for the tracing, such as the local events and
+ * a seq file.
+ *
+ * Note that the trace instance is returned disabled. This allows
+ * the tool to apply some other configs, like setting priority
+ * to the kernel threads, before starting generating trace entries.
+ */
+int trace_instance_init(struct trace_instance *trace, char *tool_name)
+{
+	trace->seq = calloc(1, sizeof(*trace->seq));
+	if (!trace->seq)
+		goto out_err;
+
+	trace_seq_init(trace->seq);
+
+	trace->inst = create_instance(tool_name);
+	if (!trace->inst)
+		goto out_err;
+
+	trace->tep = tracefs_local_events(NULL);
+	if (!trace->tep)
+		goto out_err;
+
+	/*
+	 * Let the main enable the record after setting some other
+	 * things such as the priority of the tracer's threads.
+	 */
+	tracefs_trace_off(trace->inst);
+
+	return 0;
+
+out_err:
+	trace_instance_destroy(trace);
+	return 1;
+}
+
+/*
+ * trace_instance_start - start tracing a given rtla instance
+ */
+int trace_instance_start(struct trace_instance *trace)
+{
+	return tracefs_trace_on(trace->inst);
+}

tools/tracing/rtla/src/trace.h new file +27 -0

new file mode 100644
index 000000000000..7c3930909b70
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <tracefs.h>
+#include <stddef.h>
+
+struct trace_instance {
+	struct tracefs_instance		*inst;
+	struct tep_handle		*tep;
+	struct trace_seq		*seq;
+};
+
+int trace_instance_init(struct trace_instance *trace, char *tool_name);
+int trace_instance_start(struct trace_instance *trace);
+void trace_instance_destroy(struct trace_instance *trace);
+
+struct trace_seq *get_trace_seq(void);
+int enable_tracer_by_name(struct tracefs_instance *inst, const char *tracer);
+void disable_tracer(struct tracefs_instance *inst);
+
+int enable_osnoise(struct trace_instance *trace);
+int enable_timerlat(struct trace_instance *trace);
+
+struct tracefs_instance *create_instance(char *instance_name);
+void destroy_instance(struct tracefs_instance *inst);
+
+int save_trace_to_file(struct tracefs_instance *inst, const char *filename);
+int collect_registered_events(struct tep_event *tep, struct tep_record *record,
+			      int cpu, void *context);

tools/tracing/rtla/src/utils.c new file +195 -0

new file mode 100644
index 000000000000..4a35e62b1514
@@ -0,0 +1,433 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira <[email protected]>
+ */
+
+#include <proc/readproc.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <errno.h>
+#include <sched.h>
+#include <stdio.h>
+
+#include "utils.h"
+
+#define MAX_MSG_LENGTH	1024
+int config_debug;
+
+/*
+ * err_msg - print an error message to the stderr
+ */
+void err_msg(const char *fmt, ...)
+{
+	char message[MAX_MSG_LENGTH];
+	va_list ap;
+
+	va_start(ap, fmt);
+	vsnprintf(message, sizeof(message), fmt, ap);
+	va_end(ap);
+
+	fprintf(stderr, "%s", message);
+}
+
+/*
+ * debug_msg - print a debug message to stderr if debug is set
+ */
+void debug_msg(const char *fmt, ...)
+{
+	char message[MAX_MSG_LENGTH];
+	va_list ap;
+
+	if (!config_debug)
+		return;
+
+	va_start(ap, fmt);
+	vsnprintf(message, sizeof(message), fmt, ap);
+	va_end(ap);
+
+	fprintf(stderr, "%s", message);
+}
+
+/*
+ * get_long_from_str - get a long int from a string
+ */
+long get_long_from_str(char *start)
+{
+	long value;
+	char *end;
+
+	errno = 0;
+	value = strtol(start, &end, 10);
+	if (errno || start == end)
+		return -1;
+
+	return value;
+}
+
+/*
+ * get_duration - fill output with a human readable duration since start_time
+ */
+void get_duration(time_t start_time, char *output, int output_size)
+{
+	time_t now = time(NULL);
+	struct tm *tm_info;
+	time_t duration;
+
+	duration = difftime(now, start_time);
+	tm_info = localtime(&duration);
+
+	snprintf(output, output_size, "%3d %02d:%02d:%02d",
+			tm_info->tm_yday,
+			tm_info->tm_hour - 1,
+			tm_info->tm_min,
+			tm_info->tm_sec);
+}
+
+/*
+ * parse_cpu_list - parse a cpu_list filling a char vector with cpus set
+ *
+ * Receives a cpu list, like 1-3,5 (cpus 1, 2, 3, 5), and then set the char
+ * in the monitored_cpus.
+ *
+ * XXX: convert to a bitmask.
+ */
+int parse_cpu_list(char *cpu_list, char **monitored_cpus)
+{
+	char *mon_cpus;
+	const char *p;
+	int end_cpu;
+	int nr_cpus;
+	int cpu;
+	int i;
+
+	nr_cpus = sysconf(_SC_NPROCESSORS_CONF);
+
+	mon_cpus = malloc(nr_cpus * sizeof(char));
+	memset(mon_cpus, 0, (nr_cpus * sizeof(char)));
+
+	for (p = cpu_list; *p; ) {
+		cpu = atoi(p);
+		if (cpu < 0 || (!cpu && *p != '0') || cpu >= nr_cpus)
+			goto err;
+
+		while (isdigit(*p))
+			p++;
+		if (*p == '-') {
+			p++;
+			end_cpu = atoi(p);
+			if (end_cpu < cpu || (!end_cpu && *p != '0') || end_cpu >= nr_cpus)
+				goto err;
+			while (isdigit(*p))
+				p++;
+		} else
+			end_cpu = cpu;
+
+		if (cpu == end_cpu) {
+			debug_msg("cpu_list: adding cpu %d\n", cpu);
+			mon_cpus[cpu] = 1;
+		} else {
+			for (i = cpu; i <= end_cpu; i++) {
+				debug_msg("cpu_list: adding cpu %d\n", i);
+				mon_cpus[i] = 1;
+			}
+		}
+
+		if (*p == ',')
+			p++;
+	}
+
+	*monitored_cpus = mon_cpus;
+
+	return 0;
+
+err:
+	debug_msg("Error parsing the cpu list %s", cpu_list);
+	return 1;
+}
+
+/*
+ * parse_duration - parse duration with s/m/h/d suffix converting it to seconds
+ */
+long parse_seconds_duration(char *val)
+{
+	char *end;
+	long t;
+
+	t = strtol(val, &end, 10);
+
+	if (end) {
+		switch (*end) {
+		case 's':
+		case 'S':
+			break;
+		case 'm':
+		case 'M':
+			t *= 60;
+			break;
+		case 'h':
+		case 'H':
+			t *= 60 * 60;
+			break;
+
+		case 'd':
+		case 'D':
+			t *= 24 * 60 * 60;
+			break;
+		}
+	}
+
+	return t;
+}
+
+/*
+ * parse_ns_duration - parse duration with ns/us/ms/s converting it to nanoseconds
+ */
+long parse_ns_duration(char *val)
+{
+	char *end;
+	long t;
+
+	t = strtol(val, &end, 10);
+
+	if (end) {

tools/tracing/rtla/src/utils.c new file +227 -0

+		}
+		return -1;
+	}
+
+	return t;
+}
+
+/*
+ * This is a set of helper functions to use SCHED_DEADLINE.
+ */
+#ifdef __x86_64__
+# define __NR_sched_setattr	314
+# define __NR_sched_getattr	315
+#elif __i386__
+# define __NR_sched_setattr	351
+# define __NR_sched_getattr	352
+#elif __arm__
+# define __NR_sched_setattr	380
+# define __NR_sched_getattr	381
+#elif __aarch64__
+# define __NR_sched_setattr	274
+# define __NR_sched_getattr	275
+#elif __powerpc__
+# define __NR_sched_setattr	355
+# define __NR_sched_getattr	356
+#elif __s390x__
+# define __NR_sched_setattr	345
+# define __NR_sched_getattr	346
+#endif
+
+#define SCHED_DEADLINE		6
+
+static inline int sched_setattr(pid_t pid, const struct sched_attr *attr,
+				unsigned int flags) {
+	return syscall(__NR_sched_setattr, pid, attr, flags);
+}
+
+static inline int sched_getattr(pid_t pid, struct sched_attr *attr,
+				unsigned int size, unsigned int flags)
+{
+	return syscall(__NR_sched_getattr, pid, attr, size, flags);
+}
+
+int __set_sched_attr(int pid, struct sched_attr *attr)
+{
+	int flags = 0;
+	int retval;
+
+	retval = sched_setattr(pid, attr, flags);
+	if (retval < 0) {
+		err_msg("boost_with_deadline failed to boost pid %d: %s\n",
+			pid, strerror(errno));
+		return 1;
+	}
+
+	return 0;
+}
+/*
+ * set_comm_sched_attr - set sched params to threads starting with char *comm
+ *
+ * This function uses procps to list the currently running threads and then
+ * set the sched_attr *attr to the threads that start with char *comm. It is
+ * mainly used to set the priority to the kernel threads created by the
+ * tracers.
+ */
+int set_comm_sched_attr(const char *comm, struct sched_attr *attr)
+{
+	int flags = PROC_FILLCOM | PROC_FILLSTAT;
+	PROCTAB *ptp;
+	proc_t task;
+	int retval;
+
+	ptp = openproc(flags);
+	if (!ptp) {
+		err_msg("error openproc()\n");
+		return -ENOENT;
+	}
+
+	memset(&task, 0, sizeof(task));
+
+	while (readproc(ptp, &task)) {
+		retval = strncmp(comm, task.cmd, strlen(comm));
+		if (retval)
+			continue;
+		retval = __set_sched_attr(task.tid, attr);
+		if (retval)
+			goto out_err;
+	}
+
+	closeproc(ptp);
+	return 0;
+
+out_err:
+	closeproc(ptp);
+	return 1;
+}
+
+#define INVALID_VAL	(~0L)
+static long get_long_ns_after_colon(char *start)
+{
+	long val = INVALID_VAL;
+
+	/* find the ":" */
+	start = strstr(start, ":");
+	if (!start)
+		return -1;
+
+	/* skip ":" */
+	start++;
+	val = parse_ns_duration(start);
+
+	return val;
+}
+
+static long get_long_after_colon(char *start)
+{
+	long val = INVALID_VAL;
+
+	/* find the ":" */
+	start = strstr(start, ":");
+	if (!start)
+		return -1;
+
+	/* skip ":" */
+	start++;
+	val = get_long_from_str(start);
+
+	return val;
+}
+
+/*
+ * parse priority in the format:
+ * SCHED_OTHER:
+ *		o:<prio>
+ *		O:<prio>
+ * SCHED_RR:
+ *		r:<prio>
+ *		R:<prio>
+ * SCHED_FIFO:
+ *		f:<prio>
+ *		F:<prio>
+ * SCHED_DEADLINE:
+ *		d:runtime:period
+ *		D:runtime:period
+ */
+int parse_prio(char *arg, struct sched_attr *sched_param)
+{
+	long prio;
+	long runtime;
+	long period;
+
+	memset(sched_param, 0, sizeof(*sched_param));
+	sched_param->size = sizeof(*sched_param);
+
+	switch (arg[0]) {
+	case 'd':
+	case 'D':
+		/* d:runtime:period */
+		if (strlen(arg) < 4)
+			return -1;
+
+		runtime = get_long_ns_after_colon(arg);
+		if (runtime == INVALID_VAL)
+			return -1;
+
+		period = get_long_ns_after_colon(&arg[2]);
+		if (period == INVALID_VAL)
+			return -1;
+
+		if (runtime > period)
+			return -1;
+
+		sched_param->sched_policy   = SCHED_DEADLINE;
+		sched_param->sched_runtime  = runtime;
+		sched_param->sched_deadline = period;
+		sched_param->sched_period   = period;
+		break;
+	case 'f':
+	case 'F':
+		/* f:prio */
+		prio = get_long_after_colon(arg);
+		if (prio == INVALID_VAL)
+			return -1;
+
+		if (prio < sched_get_priority_min(SCHED_FIFO))
+			return -1;
+		if (prio > sched_get_priority_max(SCHED_FIFO))
+			return -1;
+
+		sched_param->sched_policy   = SCHED_FIFO;
+		sched_param->sched_priority = prio;
+		break;
+	case 'r':
+	case 'R':
+		/* r:prio */
+		prio = get_long_after_colon(arg);
+		if (prio == INVALID_VAL)
+			return -1;
+
+		if (prio < sched_get_priority_min(SCHED_RR))
+			return -1;
+		if (prio > sched_get_priority_max(SCHED_RR))
+			return -1;
+
+		sched_param->sched_policy   = SCHED_RR;
+		sched_param->sched_priority = prio;
+		break;
+	case 'o':
+	case 'O':
+		/* o:prio */
+		prio = get_long_after_colon(arg);
+		if (prio == INVALID_VAL)
+			return -1;
+
+		if (prio < sched_get_priority_min(SCHED_OTHER))
+			return -1;
+		if (prio > sched_get_priority_max(SCHED_OTHER))
+			return -1;
+
+		sched_param->sched_policy   = SCHED_OTHER;
+		sched_param->sched_priority = prio;
+		break;
+	default:
+		return -1;
+	}
+	return 0;
+}

tools/tracing/rtla/src/utils.h new file +56 -0

new file mode 100644
index 000000000000..2bce867387b4
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <stdint.h>
+#include <time.h>
+
+/*
+ * '18446744073709551615\0'
+ */
+#define BUFF_U64_STR_SIZE	24
+
+#define container_of(ptr, type, member)({			\
+	const typeof(((type *)0)->member) *__mptr = (ptr);	\
+	(type *)((char *)__mptr - offsetof(type, member)) ; })
+
+extern int config_debug;
+void debug_msg(const char *fmt, ...);
+void err_msg(const char *fmt, ...);
+
+long parse_seconds_duration(char *val);
+void get_duration(time_t start_time, char *output, int output_size);
+
+int parse_cpu_list(char *cpu_list, char **monitored_cpus);
+long get_long_from_str(char *start);
+
+static inline void
+update_min(unsigned long long *a, unsigned long long *b)
+{
+	if (*a > *b)
+		*a = *b;
+}
+
+static inline void
+update_max(unsigned long long *a, unsigned long long *b)
+{
+	if (*a < *b)
+		*a = *b;
+}
+
+static inline void
+update_sum(unsigned long long *a, unsigned long long *b)
+{
+	*a += *b;
+}
+
+struct sched_attr {
+	uint32_t size;
+	uint32_t sched_policy;
+	uint64_t sched_flags;
+	int32_t sched_nice;
+	uint32_t sched_priority;
+	uint64_t sched_runtime;
+	uint64_t sched_deadline;
+	uint64_t sched_period;
+};
+
+int parse_prio(char *arg, struct sched_attr *sched_param);
+int set_comm_sched_attr(const char *comm, struct sched_attr *attr);

Signature · 1 line

2.31.1

Quoted 8 lines

tools/tracing/rtla/src/utils.c new file +8 -0

+long parse_ns_duration(char *val)
+{
+	char *end;
+	long t;
+
+	t = strtol(val, &end, 10);
+
+	if (end) {

tools/tracing/rtla/src/utils.c line 205

Quoted 10 lines

tools/tracing/rtla/src/utils.c new file +10 -0

+		if (!strncmp(end, "ns", 2)) {
+			return t;
+		} else if (!strncmp(end, "us", 2)) {
+			t *= 1000;
+			return t;
+		} else if (!strncmp(end, "ms", 2)) {
+			t *= 1000 * 1000;
+			return t;
+		} else if (!strncmp(end, "s", 1)) {
+			t *= 1000 * 1000 * 100;  

                               ^^^

Ah, 1000 not 100.

Quoted 11 lines

tools/tracing/rtla/src/utils.c new file +11 -0

+/*
+ * parse_ns_duration - parse duration with ns/us/ms/s converting it to nanoseconds
+ */
+long parse_ns_duration(char *val)
+{
+	char *end;
+	long t;
+
+	t = strtol(val, &end, 10);
+
+	if (end) {

tools/tracing/rtla/src/utils.c line 205

Quoted 10 lines

tools/tracing/rtla/src/utils.c new file +10 -0

+		if (!strncmp(end, "ns", 2)) {
+			return t;
+		} else if (!strncmp(end, "us", 2)) {
+			t *= 1000;
+			return t;
+		} else if (!strncmp(end, "ms", 2)) {
+			t *= 1000 * 1000;
+			return t;
+		} else if (!strncmp(end, "s", 1)) {
+			t *= 1000 * 1000 * 100;

                               ^^^

Ah, 1000 not 100.

Steven, I am tempted to start placing fixes patches on top of this series. What do you think?

Quoted 2 lines · On Wed, 27 Oct 2021 18:24:01 +0200 Daniel Bristot de Oliveira…

Steven, I am tempted to start placing fixes patches on top of this series. What do you think?

Please don't. But you can also wait a few days before you post another version. You don't need to post a new version for every comment. Especially when the reviewers are not done reviewing the patches.

The default is to wait a week, although I would say at least wait 3 days after the last comment was made.

-- Steve

tools/tracing/rtla/src/osnoise.c new file +6 -0

+/*
+ * osnoise_set_cpus - configure osnoise to run on *cpus
+ *
+ * "osnoise/cpus" file is used to set the cpus in which osnoise/timerlat
+ * will run. This function opens this file, saves the current value,
+ * and set the cpus passed as argument.

tools/tracing/rtla/src/osnoise.c new file +17 -0

+	tracefs_put_tracing_file(osnoise_cpus);
+
+	return 0;
+
+out_close:
+	close(context->cpus_fd);
+	context->cpus_fd = -1;
+out_err:
+	tracefs_put_tracing_file(osnoise_cpus);
+	return 1;
+}
+
+/*
+ * osnoise_restore_cpus - restore the original "osnoise/cpus"
+ *
+ * osnoise_set_cpus() saves the original data for the "osnoise/cpus"
+ * file. This function restore the original config it was previously

tools/tracing/rtla/src/osnoise.c new file +15 -0

+	if (!retval)
+		err_msg("could not restore original osnoise cpus\n");
+
+	free(context->orig_cpus);
+}
+
+/*
+ * osnoise_get_runtime - return the original "osnoise/runtime_us" value
+ *
+ * It also saves the value to be restored.
+ */
+unsigned long long osnoise_get_runtime(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *runtime_path;

tools/tracing/rtla/src/osnoise.c new file +31 -0

+	retval = read(context->runtime_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->runtime_us = get_long_from_str(buffer);
+	if (!context->runtime_us)
+		goto out_close;
+
+	context->orig_runtime_us = context->runtime_us;
+
+	tracefs_put_tracing_file(runtime_path);
+
+	return context->runtime_us;
+
+out_close:
+	close(context->runtime_fd);
+	context->runtime_fd = -1;
+out_err:
+	tracefs_put_tracing_file(runtime_path);
+	return 0;
+}
+
+/*
+ * osnoise_get_period - return the original "osnoise/period_us" value
+ *
+ * It also saves the value to be restored.
+ */
+unsigned long long osnoise_get_period(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *period_path;

tools/tracing/rtla/src/osnoise.c new file +140 -0

+	retval = read(context->period_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->period_us = get_long_from_str(buffer);
+	if (!context->period_us)
+		goto out_close;
+
+	context->orig_period_us = context->period_us;
+
+	tracefs_put_tracing_file(period_path);
+
+	return context->period_us;
+
+out_close:
+	close(context->period_fd);
+	context->period_fd = -1;
+out_err:
+	tracefs_put_tracing_file(period_path);
+	return 0;
+}
+
+static int __osnoise_write_runtime(struct osnoise_context *context,
+				   unsigned long long runtime)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (context->runtime_fd == -1) {
+		err_msg("tried to write osnoise runtime before getting it\n");
+		return 0;
+	}
+
+	snprintf(buffer, sizeof(buffer), "%llu\n", runtime);
+
+	retval = write(context->runtime_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		return -1;
+
+	context->runtime_us = runtime;
+	return 0;
+}
+
+static int __osnoise_write_period(struct osnoise_context *context,
+				  unsigned long long period)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (context->period_fd == -1)
+		return 0;
+
+	snprintf(buffer, sizeof(buffer), "%llu\n", period);
+
+	retval = write(context->period_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		return -1;
+
+	context->period_us = period;
+	return 0;
+}
+
+/*
+ * osnoise_set_runtime_period - set osnoise runtime and period
+ *
+ * Osnoise's runtime and period are related as runtime <= period.
+ * Thus, this function saves the original values, and then tries
+ * to set the runtime and period if they are != 0.
+ */
+int osnoise_set_runtime_period(struct osnoise_context *context,
+			       unsigned long long runtime,
+			       unsigned long long period)
+{
+	unsigned long long curr_runtime_us;
+	unsigned long long curr_period_us;
+	int retval;
+
+	if (!period && !runtime)
+		return 0;
+
+	curr_runtime_us = osnoise_get_runtime(context);
+	curr_period_us = osnoise_get_period(context);
+
+	if (!curr_period_us || !curr_runtime_us)
+		return -1;
+
+	if (!period) {
+		if (runtime > curr_period_us)
+			return -1;
+		return __osnoise_write_runtime(context, runtime);
+	} else if (!runtime) {
+		if (period < curr_runtime_us)
+			return -1;
+		return __osnoise_write_period(context, period);
+	}
+
+	if (runtime > curr_period_us) {
+		retval = __osnoise_write_period(context, period);
+		if (retval)
+			return -1;
+		retval = __osnoise_write_runtime(context, runtime);
+		if (retval)
+			return -1;
+	} else {
+		retval = __osnoise_write_runtime(context, runtime);
+		if (retval)
+			return -1;
+		retval = __osnoise_write_period(context, period);
+		if (retval)
+			return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * osnoise_restore_runtime_period - restore the original runtime and period
+ */
+void osnoise_restore_runtime_period(struct osnoise_context *context)
+{
+	unsigned long long runtime = context->orig_runtime_us;
+	unsigned long long period = context->orig_period_us;
+	int retval;
+
+	if (context->runtime_fd < 0 && context->period_fd < 0)
+		return;
+
+	retval = osnoise_set_runtime_period(context, runtime, period);
+	if (retval)
+		err_msg("Could not restore original osnoise runtime/period\n");
+}
+
+/*
+ * osnoise_get_stop_us - read and save the original "stop_tracing_us"
+ */
+static long long
+osnoise_get_stop_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *stop_path;

tools/tracing/rtla/src/osnoise.c new file +55 -0

+	retval = read(context->stop_us_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->stop_us = get_long_from_str(buffer);
+	context->orig_stop_us = context->stop_us;
+
+	tracefs_put_tracing_file(stop_path);
+
+	return context->stop_us;
+
+out_close:
+	close(context->stop_us_fd);
+	context->stop_us_fd = -1;
+out_err:
+	tracefs_put_tracing_file(stop_path);
+	return -1;
+}
+
+/*
+ * osnoise_set_stop_us - set "stop_tracing_us"
+ */
+int osnoise_set_stop_us(struct osnoise_context *context, long long stop_us)
+{
+	long long curr_stop_us = osnoise_get_stop_us(context);
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (curr_stop_us < 0)
+		return -1;
+
+	snprintf(buffer, BUFF_U64_STR_SIZE, "%lld\n", stop_us);
+
+	retval = write(context->stop_us_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		return -1;
+
+	context->stop_us = stop_us;
+
+	return 0;
+}
+
+/*
+ * osnoise_restore_stop_us - restore the original stop_tracing_us
+ */
+void osnoise_restore_stop_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (context->stop_us_fd < 0)
+		return;
+
+	if (context->orig_stop_us == context->stop_us)
+		return;

+ */

tools/tracing/rtla/src/osnoise.c new file +5 -0

+static long long
+osnoise_get_stop_total_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *stop_path;

tools/tracing/rtla/src/osnoise.c new file +11 -0

+	retval = read(context->stop_total_us_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->stop_total_us = get_long_from_str(buffer);
+	context->orig_stop_total_us = context->stop_total_us;
+
+	tracefs_put_tracing_file(stop_path);
+
+	return context->stop_total_us;
+

+ */

tools/tracing/rtla/src/osnoise.c new file +12 -0

+int osnoise_set_stop_total_us(struct osnoise_context *context, long long stop_total_us)
+{
+	long long curr_stop_total_us = osnoise_get_stop_total_us(context);
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (curr_stop_total_us < 0)
+		return -1;
+
+	snprintf(buffer, BUFF_U64_STR_SIZE, "%lld\n", stop_total_us);
+
+	retval = write(context->stop_total_us_fd, buffer, strlen(buffer) + 1);

+ */

tools/tracing/rtla/src/osnoise.c new file +26 -0

+void osnoise_restore_stop_total_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (context->stop_total_us_fd < 0)
+		return;
+
+	if (context->orig_stop_total_us == context->stop_total_us)
+		return;
+
+	snprintf(buffer, BUFF_U64_STR_SIZE, "%lld\n", context->orig_stop_total_us);
+
+	retval = write(context->stop_total_us_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		err_msg("Could not restore original osnoise stop_total_us\n");
+}
+
+/*
+ * osnoise_get_timerlat_period_us - read and save the original "timerlat_period_us"
+ */
+static long long
+osnoise_get_timerlat_period_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *stop_path;

tools/tracing/rtla/src/osnoise.c new file +71 -0

+	retval = read(context->timerlat_period_us_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->timerlat_period_us = get_long_from_str(buffer);
+	context->orig_timerlat_period_us = context->timerlat_period_us;
+
+	tracefs_put_tracing_file(stop_path);
+
+	return context->timerlat_period_us;
+
+out_close:
+	close(context->timerlat_period_us_fd);
+	context->timerlat_period_us_fd = -1;
+out_err:
+	tracefs_put_tracing_file(stop_path);
+	return -1;
+}
+
+/*
+ * osnoise_set_timerlat_period_us - set "timerlat_period_us"
+ */
+int osnoise_set_timerlat_period_us(struct osnoise_context *context, long long timerlat_period_us)
+{
+	long long curr_timerlat_period_us = osnoise_get_timerlat_period_us(context);
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (curr_timerlat_period_us < 0)
+		return -1;
+
+	snprintf(buffer, BUFF_U64_STR_SIZE, "%lld\n", timerlat_period_us);
+
+	retval = write(context->timerlat_period_us_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		return -1;
+
+	context->timerlat_period_us = timerlat_period_us;
+
+	return 0;
+}
+
+/*
+ * osnoise_restore_timerlat_period_us - restore "timerlat_period_us"
+ */
+void osnoise_restore_timerlat_period_us(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	int retval;
+
+	if (context->timerlat_period_us_fd < 0)
+		return;
+
+	if (context->orig_timerlat_period_us == context->timerlat_period_us)
+		return;
+
+	snprintf(buffer, BUFF_U64_STR_SIZE, "%lld\n", context->orig_timerlat_period_us);
+
+	retval = write(context->timerlat_period_us_fd, buffer, strlen(buffer) + 1);
+	if (retval < 0)
+		err_msg("Could not restore original osnoise timerlat_period_us\n");
+}
+
+/*
+ * osnoise_get_print_stack - read and save the original "print_stack"
+ */
+static long long
+osnoise_get_print_stack(struct osnoise_context *context)
+{
+	char buffer[BUFF_U64_STR_SIZE];
+	char *stop_path;

tools/tracing/rtla/src/osnoise.c new file +26 -0

+	retval = read(context->print_stack_fd, &buffer, sizeof(buffer));
+	if (!retval)
+		goto out_close;
+
+	context->print_stack = get_long_from_str(buffer);
+	context->orig_print_stack = context->print_stack;
+
+	tracefs_put_tracing_file(stop_path);
+
+	return context->print_stack;
+
+out_close:
+	close(context->print_stack_fd);
+	context->print_stack_fd = -1;
+out_err:
+	tracefs_put_tracing_file(stop_path);
+	return -1;
+}
+
+/*
+ * osnoise_set_print_stack - set "print_stack"
+ */
+int osnoise_set_print_stack(struct osnoise_context *context, long long print_stack)
+{
+	long long curr_print_stack = osnoise_get_print_stack(context);
+	char buffer[BUFF_U64_STR_SIZE];

Quoted 9 lines · Daniel Bristot de Oliveira

tools/tracing/rtla/src/osnoise.c new file +9 -0

+/*
+ * osnoise_set_cpus - configure osnoise to run on *cpus
+ *
+ * "osnoise/cpus" file is used to set the cpus in which osnoise/timerlat
+ * will run. This function opens this file, saves the current value,
+ * and set the cpus passed as argument.
+ */
+int osnoise_set_cpus(struct osnoise_context *context, char *cpus)
+{

tools/tracing/rtla/src/osnoise.c line 34

Quoted 7 lines

tools/tracing/rtla/src/osnoise.c new file +7 -0

+	char *osnoise_cpus = tracefs_get_tracing_file("osnoise/cpus");
+	char curr_cpus[1024];
+	int retval;
+
+	context->cpus_fd = open(osnoise_cpus, O_RDWR);
+	if (!context->cpus_fd)
+		goto out_err;

The above check should be "context->cpus_fd < 0".

tools/tracing/rtla/src/osnoise.c line 42

Quoted 6 lines

tools/tracing/rtla/src/osnoise.c new file +6 -0

+	retval = read(context->cpus_fd, &curr_cpus, sizeof(curr_cpus));
+	if (!retval)
+		goto out_close;
+	context->orig_cpus = strdup(curr_cpus);
+	if (!context->orig_cpus)
+		goto out_err;

Need to close ->cpus_fd;

if (!context->orig_cpus)

    goto out_close;

tools/tracing/rtla/src/osnoise.c line 46

Quoted 3 lines

tools/tracing/rtla/src/osnoise.c new file +3 -0

+	retval = write(context->cpus_fd, cpus, strlen(cpus) + 1);
+	if (!retval)
+		goto out_err;

Same as above. Use "goto out_close" instead.

Quoted 21 lines

tools/tracing/rtla/src/osnoise.c new file +21 -0

+	tracefs_put_tracing_file(osnoise_cpus);
+
+	return 0;
+
+out_close:
+	close(context->cpus_fd);
+	context->cpus_fd = -1;
+out_err:
+	tracefs_put_tracing_file(osnoise_cpus);
+	return 1;
+}
+
+/*
+ * osnoise_restore_cpus - restore the original "osnoise/cpus"
+ *
+ * osnoise_set_cpus() saves the original data for the "osnoise/cpus"
+ * file. This function restore the original config it was previously
+ * modified.
+ */
+void osnoise_restore_cpus(struct osnoise_context *context)
+{

tools/tracing/rtla/src/osnoise.c line 74

Quoted 6 lines

tools/tracing/rtla/src/osnoise.c new file +6 -0

+	int retval;
+
+	if (!context->orig_cpus)
+		return;
+
+	retval = write(context->cpus_fd, context->orig_cpus, strlen(context->orig_cpus));

__osnoise_write_runtime() check "context->cpus_fd == -1". Is it possible here we need to check "context->cpus_fd == -1".

The rtla osnoise hist tool collects all osnoise:sample_threshold occurrence in a histogram, displaying the results in a user-friendly way. The tool also allows many configurations of the osnoise tracer and the collection of the tracer output.

Here is one example of the rtla osnoise hist tool output:
---------- %< ----------
[root@f34 ~]# rtla osnoise hist --bucket-size 10 --entries 100 -c 0-8 -d 1M -r 9000 -P F:1 # RTLA osnoise histogram
# Time unit is microseconds (us)
# Duration: 0 00:01:00
Index CPU-000 CPU-001 CPU-002 CPU-003 CPU-004 CPU-005 CPU-006 CPU-007 CPU-008

 0           430       434       352       455       440       463       467       436       484
 10           88        88        92       141       120       100       126       166       100
 20           19         7        12        22         8         8        13        13        16
 30            6         0         2         0         1         2         2         1         0
 50            0         0         0         0         0         0         1         0         0
 over:         0         0         0         0         0         0         0         0         0
 count:      543       529       458       618       569       573       609       616       600
 min:          0         0         0         0         0         0         0         0         0
 avg:          0         0         0         0         0         0         0         0         0

max: 30 20 30 20 30 30 50 30 20
---------- >% ----------

Running
- rtla osnoise hist --help

provides information about the available options.

Cc Steven Rostedt <[email protected]>

Cc Ingo Molnar <[email protected]>

Cc Tom Zanussi <[email protected]>

Cc Masami Hiramatsu <[email protected]>

Cc Juri Lelli <[email protected]>

Cc Clark Williams <[email protected]>

Cc John Kacur <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Thomas Gleixner <[email protected]>

Cc Sebastian Andrzej Siewior <[email protected]>

Cc Daniel Bristot de Oliveira <[email protected]>

Cc [email protected]

Cc [email protected]

Cc [email protected]

Signed-off-by Daniel Bristot de Oliveira <[email protected]>

---
 tools/tracing/rtla/src/osnoise.c      |   6 +-
 tools/tracing/rtla/src/osnoise.h      |   1 +
 tools/tracing/rtla/src/osnoise_hist.c | 793 ++++++++++++++++++++++++++
 3 files changed, 799 insertions(+), 1 deletion(-)
 create mode 100644 tools/tracing/rtla/src/osnoise_hist.c

tools/tracing/rtla/src/osnoise.c +5 -1

index b9866dfdda66..6c6358ebf66f 100644
@@ -790,7 +790,8 @@ static void osnoise_usage(void)
 		"  usage: [rtla] osnoise [MODE] ...",
 		"",
 		"  modes:",
-		"     top  - prints the summary from osnoise tracer",
+		"     top   - prints the summary from osnoise tracer",
+		"     hist  - prints a histogram of osnoise samples",
 		"",
 		"if no MODE is given, the top mode is called, passing the arguments",
 		NULL,

@@ -825,6 +826,9 @@ int osnoise_main(int argc, char *argv[])
 	} else if (strcmp(argv[1], "top") == 0) {
 		osnoise_top_main(argc-1, &argv[1]);
 		exit(0);
+	} else if (strcmp(argv[1], "hist") == 0) {
+		osnoise_hist_main(argc-1, &argv[1]);
+		exit(0);
 	}
 
 usage:

tools/tracing/rtla/src/osnoise.h +1 -0

index 4882ee275ea0..dbaad7a162a2 100644
@@ -82,5 +82,6 @@ void osnoise_destroy_tool(struct osnoise_tool *top);
 struct osnoise_tool *osnoise_init_tool(char *tool_name);
 struct osnoise_tool *osnoise_init_trace_tool(char *tracer);
 
+int osnoise_hist_main(int argc, char *argv[]);
 int osnoise_top_main(int argc, char **argv);
 int osnoise_main(int argc, char **argv);

tools/tracing/rtla/src/osnoise_hist.c new file +102 -0

new file mode 100644
index 000000000000..c93f30d69795
@@ -0,0 +1,793 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira <[email protected]>
+ */
+
+#include <getopt.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <time.h>
+
+#include "utils.h"
+#include "osnoise.h"
+
+struct osnoise_hist_params {
+	char			*cpus;
+	char			*monitored_cpus;
+	char			*trace_output;
+	unsigned long long	runtime;
+	unsigned long long	period;
+	long long		stop_us;
+	long long		stop_total_us;
+	int			sleep_time;
+	int			duration;
+	int			set_sched;
+	int			output_divisor;
+	struct sched_attr	sched_param;
+
+	char			no_header;
+	char			no_summary;
+	char			no_index;
+	char			with_zeros;
+	int			bucket_size;
+	int			entries;
+};
+
+struct osnoise_hist_cpu {
+	int			*samples;
+	int			*thread;
+
+	int			count;
+	int			thread_count;
+
+	unsigned long long	min_sample;
+	unsigned long long	sum_sample;
+	unsigned long long	max_sample;
+
+	unsigned long long	min_thread;
+	unsigned long long	sum_thread;
+	unsigned long long	max_thread;
+};
+
+struct osnoise_hist_data {
+	struct tracefs_hist	*trace_hist;
+	struct osnoise_hist_cpu	*hist;
+	int			entries;
+	int			bucket_size;
+	int			nr_cpus;
+};
+
+/*
+ * osnoise_free_histogram - free runtime data
+ */
+static void
+osnoise_free_histogram(struct osnoise_hist_data *data)
+{
+	int cpu;
+
+	/* one histogram for IRQ and one for thread, per CPU */
+	for (cpu = 0; cpu < data->nr_cpus; cpu++) {
+		if (data->hist[cpu].samples)
+			free(data->hist[cpu].samples);
+
+		if (data->hist[cpu].thread)
+			free(data->hist[cpu].thread);
+	}
+
+	/* one set of histograms per CPU */
+	if (data->hist)
+		free(data->hist);
+
+	free(data);
+}
+
+/*
+ * osnoise_alloc_histogram - alloc runtime data
+ */
+static struct osnoise_hist_data
+*osnoise_alloc_histogram(int nr_cpus, int entries, int bucket_size)
+{
+	struct osnoise_hist_data *data;
+	int cpu;
+
+	data = calloc(1, sizeof(*data));
+	if (!data)
+		return NULL;
+
+	data->entries = entries;
+	data->bucket_size = bucket_size;