Skip to contentSkip to conversation

trace-cmd: Improve multi-instance support

PATCH New Howard Cochran 1 participant 6 messages Apr 22, 2015 Documentation

lore
Share & tools
lore
#0 lore raw

v3 cover trace-cmd: Improve multi-instance support

NOTE: The previous two versions of this patch set were distributed off-list.

v3 changes:
* Improved clarification of options in documentation of trace-cmd reset and added examples.

v2 changes:
* Rebased this patch series on Steve's simplification of first_instance logic.

Improve the completeness and consistency of support for multiple buffer instances in the extract, snapshot, stat, start, stop, reset, and restart sub-commands.

For the commands that lacked it, added support for -B and -t options, consistent with the existing behavior of stop, reset, and restart. That is, including a -B instance-name option will cause these commands to exclude the top buffer instance unless -t is also provided.

Exception: trace-cmd start retains its existing behavior regarding the top instance, consistent with trace-cmd record.

For all of these sub-commands except start, add a new -a option, which causes it to operate on all existing instances. As with -B, -a will exclude the top instance unless -t is also given.

Updated documentation accordingly.

Howard Cochran (5):
trace-cmd extract: Add -B and -t for multiple instances trace-cmd extract: Add -a option for all instances trace-cmd stop, reset, restart: Add -a for all instances trace-cmd snapshot: Fully support multiple instances trace-cmd stat: Support -a for all buffers. Update docs.

 Documentation/trace-cmd-extract.1.txt  |  16 ++++
 Documentation/trace-cmd-reset.1.txt    |  57 ++++++++++--
 Documentation/trace-cmd-snapshot.1.txt |  16 +++-
 Documentation/trace-cmd-stat.1.txt     |  19 ++++
 Documentation/trace-cmd-stop.1.txt     |  13 ++-
 trace-cmd.c                            |   2 +-
 trace-local.h                          |   6 +-
 trace-record.c                         | 160 ++++++++++++++++++++++++++++-----
 trace-snapshot.c                       |  90 ++++++++++---------
 trace-stat.c                           |  15 +++-
 trace-usage.c                          |  42 ++++++---
 11 files changed, 341 insertions(+), 95 deletions(-)
Signature · 1 line
1.9.1
#1 lore raw

PATCH v3 1/5 trace-cmd extract: Add -B and -t for multiple instances

Add -B option to trace-cmd extract so that it can extract listed instances. As with start, stop, and reset, the top level instance is not included when any -B are given. However, -t makes it also include the top instance. As with other commands, if neither option is given, only operate on the top instance.

---
 trace-record.c | 20 +++++++++++++-------
 trace-usage.c  |  6 ++++--
 2 files changed, 17 insertions(+), 9 deletions(-)
trace-record.c +13 -7
index 6e0ab3b..8340416 100644
@@ -523,17 +523,20 @@ static int create_recorder(struct buffer_instance *instance, int cpu,
 
 static void flush_threads(void)
 {
+	struct buffer_instance *instance;
 	long ret;
 	int i;
 
 	if (!cpu_count)
 		return;
 
-	for (i = 0; i < cpu_count; i++) {
-		/* Extract doesn't support sub buffers yet */
-		ret = create_recorder(&top_instance, i, TRACE_TYPE_EXTRACT, NULL);
-		if (ret < 0)
-			die("error reading ring buffer");
+	for_all_instances(instance) {
+		for (i = 0; i < cpu_count; i++) {
+			/* Extract doesn't support sub buffers yet */
+			ret = create_recorder(instance, i, TRACE_TYPE_EXTRACT, NULL);
+			if (ret < 0)
+				die("error reading ring buffer");
+		}
 	}
 }
 
@@ -3873,7 +3876,7 @@ void trace_record (int argc, char **argv)
 		};
 
 		if (extract)
-			opts = "+haf:Fp:co:O:sr:g:l:n:P:N:tb:ksiT";
+			opts = "+haf:Fp:co:O:sr:g:l:n:P:N:tb:B:ksiT";
 		else
 			opts = "+hae:f:Fp:cC:dDGo:O:s:r:vg:l:n:P:N:tb:R:B:ksSiTm:M:H:";
 		c = getopt_long (argc-1, argv+1, opts, long_options, &option_index);
@@ -4061,7 +4064,10 @@ void trace_record (int argc, char **argv)
 			instance->cpumask = optarg;
 			break;
 		case 't':
-			use_tcp = 1;
+			if (extract)
+				topt = 1; /* Extract top instance also */
+			else
+				use_tcp = 1;
 			break;
 		case 'b':
 			instance->buffer_size = atoi(optarg);
trace-usage.c +4 -2
index 520b14b..a708a85 100644
@@ -62,9 +62,11 @@ static struct usage_help usage_help[] = {
 	{
 		"extract",
 		"extract a trace from the kernel",
-		" %s extract [-p plugin][-O option][-o file][-s]\n"
-		"          Uses same options as record, but only reads an existing trace.\n"
+		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-t]\n"
+		"          Uses similar options as record, but only reads an existing trace.\n"
 		"          -s : extract the snapshot instead of the main buffer\n"
+		"          -B : extract a given buffer (more than one may be specified)\n"
+		"          -t : include the top level buffer (useful with -B)\n"
 	},
 	{
 		"stop",
Signature · 1 line
1.9.1
#2 lore raw

PATCH v3 2/5 trace-cmd extract: Add -a option for all instances

In reply to Howard Cochran

This option will extract all instances that currently exist in the system, including the top instance. This differs from the meaning of -a for record and stream (enable all events), which would have no purpose for extract. Such difference in meaning already exists for -s, so this seemed reasonable.

Had to fix a bug in create_instance(), in which it ignored its parameter.

---
 Documentation/trace-cmd-extract.1.txt | 16 +++++++
 trace-local.h                         |  2 +-
 trace-record.c                        | 81 +++++++++++++++++++++++++++++++++--
 trace-usage.c                         |  5 ++-
 4 files changed, 98 insertions(+), 6 deletions(-)
Documentation/trace-cmd-extract.1.txt +16 -0
index ea98c20..73eda4e 100644
@@ -52,6 +52,22 @@ OPTIONS
     does cause the extract routine to disable all tracing. That is,
     the end of the extract will perform something similar to trace-cmd-reset(1).
 
+*-B* 'buffer-name'::
+    If the kernel supports multiple buffers, this will extract the trace for
+    only the given buffer. It does not affect any other buffer. This may be
+    used multiple times to specify different buffers. When this option is
+    used, the top level instance will not be extracted unless *-t* is given.
+
+*-a*::
+    Extract all existing buffer instances. When this option is used, the
+    top level instance will not be extracted unless *-t* is given.
+
+*-t*::
+    Extracts the top level instance buffer. Without the *-B* or *-a* option
+    this is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be extracted.
+
+
 SEE ALSO
 --------
 trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),
trace-local.h +1 -1
index ab2bd34..fb2ce71 100644
@@ -176,7 +176,7 @@ extern struct buffer_instance *first_instance;
 #define for_all_instances(i) for (i = first_instance; i; \
 				  i = i == &top_instance ? buffer_instances : (i)->next)
 
-struct buffer_instance *create_instance(char *name);
+struct buffer_instance *create_instance(const char *name);
 void add_instance(struct buffer_instance *instance);
 char *get_instance_file(struct buffer_instance *instance, const char *file);
 
trace-record.c +78 -3

103 lines not on this page · raw message

trace-usage.c +3 -2
index a708a85..39de386 100644
@@ -62,11 +62,12 @@ static struct usage_help usage_help[] = {
 	{
 		"extract",
 		"extract a trace from the kernel",
-		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-t]\n"
+		" %s extract [-p plugin][-O option][-o file][-B buf][-s][-a][-t]\n"
 		"          Uses similar options as record, but only reads an existing trace.\n"
 		"          -s : extract the snapshot instead of the main buffer\n"
 		"          -B : extract a given buffer (more than one may be specified)\n"
-		"          -t : include the top level buffer (useful with -B)\n"
+		"          -a : extract all buffers (except top one)\n"
+		"          -t : extract the top level buffer (useful with -B and -a)\n"
 	},
 	{
 		"stop",
Signature · 1 line
1.9.1
#3 lore raw

PATCH v3 3/5 trace-cmd stop, reset, restart: Add -a for all instances

In reply to Howard Cochran

As for extract, -a means all instances except the top one. Use -t to get the top one too.

---
 Documentation/trace-cmd-reset.1.txt | 57 +++++++++++++++++++++++++++++++++----
 Documentation/trace-cmd-stop.1.txt  | 13 ++++++---
 trace-record.c                      | 57 ++++++++++++++++++++++++++++++-------
 trace-usage.c                       | 21 ++++++++------
 4 files changed, 119 insertions(+), 29 deletions(-)
Documentation/trace-cmd-reset.1.txt +51 -6
index 06a60af..bff4530 100644
@@ -22,6 +22,9 @@ the data in the ring buffer, and the options that were used are all lost.
 
 OPTIONS
 -------
+Please note that the order that options are specified on the command line is
+significant. See EXAMPLES.
+
 *-b* 'buffer_size'::
     When the kernel boots, the Ftrace ring buffer is of a minimal size (3
     pages per CPU). The first time the tracer is used, the ring buffer size
@@ -32,6 +35,17 @@ OPTIONS
 
     trace-cmd reset -b 1
 
+    The buffer instance affected is the one (or ones) specified by the most
+    recently preceding *-B*, *-t*, or *-a* option:
+
+    When used after *-B*, resizes the buffer instance that precedes it on
+    the command line.
+
+    When used after *-a*, resizes all buffer instances except the top one.
+
+    When used after *-t* or before any *-B* or *-a*, resizes the top
+    instance.
+
 *-B* 'buffer-name'::
     If the kernel supports multiple buffers, this will reset the trace for
     only the given buffer. It does not affect any other buffer. This may be
@@ -39,15 +53,46 @@ OPTIONS
     will not be reset if this option is given (unless the *-t* option is
     also supplied).
 
+*-a*::
+    Reset the trace for all existing buffer instances. When this option
+    is used, the top level instance will not be reset unless *-t* is given.
+
 *-d*::
-    Delete the previously specified buffer instance that was specified by
-    the *-B* option. It is invalid to use with *-t* as the top level instance
-    can not be deleted.
+    This option deletes the instance buffer(s) specified by the most recently
+    preceding *-B* or *-a* option. Because the top-level instance buffer
+    cannot be deleted, it is invalid to use this immediatly following *-t* or
+    prior to any *-B* or *-a* option on the command line.
 
 *-t*::
-    Resets the top level instance buffer. Without the *-B* option this is the
-    same as the default. But if *-B* is used, this is required if the top
-    level instance buffer should also be reset.
+    Resets the top level instance buffer. Without the *-B* or *-a* option
+    this is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be reset.
+
+EXAMPLES
+--------
+
+Reset tracing for instance-one and set its per-cpu buffer size to 4096kb.
+Also deletes instance-two. The top level instance and any other instances
+remain unaffected:
+
+    trace-cmd reset -B instance-one -b 4096 -B instance-two -d
+
+Delete all instance buffers. Top level instance remains unaffected:
+
+    trace-cmd reset -a -d
+
+Delete all instance buffers and also reset the top instance:
+
+    trace-cmd reset -t -a -d
+
+Invalid. This command implies an attempt to delete the top instance:
+
+    trace-cmd reset -a -t -d
+
+Reset the top instance and set its per-cpu buffer size to 1024kb. If any
+instance buffers exist, they will be unaffected:
+
+    trace-cmd reset -b 1024
 
 
 SEE ALSO
Documentation/trace-cmd-stop.1.txt +9 -4
index c3f8b33..9d4093e 100644
@@ -30,12 +30,17 @@ OPTIONS
 *-B* 'buffer-name'::
     If the kernel supports multiple buffers, this will stop the trace for
     only the given buffer. It does not affect any other buffer. This may be
-    used multiple times to specify different buffers.
+    used multiple times to specify different buffers. When this option is
+    used, the top level instance will not be stopped unless *-t* is given.
+
+*-a*::
+    Stop the trace for all existing buffer instances. When this option
+    is used, the top level instance will not be stopped unless *-t* is given.
 
 *-t*::
-    Stops the top level instance buffer. Without the *-B* option this is the
-    same as the default. But if *-B* is used, this is required if the top
-    level instance buffer should also be stopped.
+    Stops the top level instance buffer. Without the *-B* or *-a* option this
+    is the same as the default. But if *-B* or *-a* is used, this is
+    required if the top level instance buffer should also be stopped.
 
 SEE ALSO
 --------
trace-record.c +47 -10

109 lines not on this page · raw message

trace-usage.c +12 -9
index 39de386..c56661b 100644
@@ -72,20 +72,22 @@ static struct usage_help usage_help[] = {
 	{
 		"stop",
 		"stop the kernel from recording trace data",
-		" %s stop [-B buf [-B buf]..] [-t]\n"
+		" %s stop [-B buf [-B buf]..] [-a] [-t]\n"
 		"          Stops the tracer from recording more data.\n"
 		"          Used in conjunction with start\n"
-		"          -B stop a given buffer (more than one may be specified)\n "
-		"          -t stop the top level buffer (needed if -B is specified)\n"
+		"          -B stop a given buffer (more than one may be specified)\n"
+		"          -a stop all buffers (except top one)\n"
+		"          -t stop the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"restart",
 		"restart the kernel trace data recording",
-		" %s restart [-B buf [-B buf]..] [-t]\n"
+		" %s restart [-B buf [-B buf]..] [-a] [-t]\n"
 		"          Restarts recording after a trace-cmd stop.\n"
 		"          Used in conjunction with stop\n"
-		"          -B restart a given buffer (more than one may be specified)\n "
-		"          -t restart the top level buffer (needed if -B is specified)\n"
+		"          -B restart a given buffer (more than one may be specified)\n"
+		"          -a restart all buffers (except top one)\n"
+		"          -t restart the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"show",
@@ -114,13 +116,14 @@ static struct usage_help usage_help[] = {
 	{
 		"reset",
 		"disable all kernel tracing and clear the trace buffers",
-		" %s reset [-b size][-B buf][-d][-t]\n"
+		" %s reset [-b size][-B buf][-a][-d][-t]\n"
 		"          Disables the tracer (may reset trace file)\n"
 		"          Used in conjunction with start\n"
 		"          -b change the kernel buffer size (in kilobytes per CPU)\n"
-		"          -B reset the given buffer instance (top instance ignored)\n"
 		"          -d delete the previous specified instance\n"
-		"          -t still reset the top instance if -B option is given\n"
+		"          -B reset the given buffer instance (may specify multiple -B)\n"
+		"          -a reset all instances (except top one)\n"
+		"          -t reset the top level instance (useful with -B or -a)\n"
 	},
 	{
 		"report",
Signature · 1 line
1.9.1
#4 lore raw

PATCH v3 4/5 trace-cmd snapshot: Fully support multiple instances

In reply to Howard Cochran

Add support for multiple -B instances, the same as stop, reset, restart, and extract sub-commands. -a operates on all instances except the top one -t also includes the top instance

---
 Documentation/trace-cmd-snapshot.1.txt | 16 +++++-
 trace-cmd.c                            |  2 +-
 trace-local.h                          |  4 ++
 trace-record.c                         |  4 +-
 trace-snapshot.c                       | 90 ++++++++++++++++++----------------
 trace-usage.c                          |  8 ++-
 6 files changed, 75 insertions(+), 49 deletions(-)
Documentation/trace-cmd-snapshot.1.txt +14 -2
index 98b8924..767ab67 100644
@@ -41,8 +41,20 @@ OPTIONS
 
 *-B* 'buf'::
     If a buffer instance was created, then the *-B* option will operate on
-    the snapshot within the buffer.
-    
+    the snapshot within the buffer. This may be used multiple times to
+    specify different buffers. When this option is used, the top level
+    instance will be ignored unless *-t* is given.
+
+*-a*::
+    Operate on the snapshot for all existing buffer instances.
+    When this option is used, the top level instance will be ignored
+    unless *-t* is given.
+
+*-t*::
+    Operate on the snapshot within the top level instance buffer. Without
+    the *-B* or *-a* option this is the same as the default.  But if *-B*
+    or *-a* is used, this is required if the top level instance buffer
+    should also be affected.
 
 SEE ALSO
 --------
trace-cmd.c +1 -1
index 4c5b564..647a2b4 100644
@@ -72,7 +72,7 @@ void *malloc_or_die(unsigned int size)
 	return data;
 }
 
-static void dump_file_content(const char *path)
+void dump_file_content(const char *path)
 {
 	char buf[BUFSIZ];
 	ssize_t n;
trace-local.h +4 -0
index fb2ce71..982fe6f 100644
@@ -47,7 +47,9 @@ struct pid_record_data {
 	struct pevent_record	*record;
 };
 
+void dump_file_content(const char *path);
 void show_file(const char *name);
+int write_file(const char *file, const char *str, const char *type);
 
 struct tracecmd_input *read_trace_header(const char *file);
 int read_trace_files(void);
@@ -179,6 +181,8 @@ extern struct buffer_instance *first_instance;
 struct buffer_instance *create_instance(const char *name);
 void add_instance(struct buffer_instance *instance);
 char *get_instance_file(struct buffer_instance *instance, const char *file);
+void add_all_instances(void);
+void update_first_instance(int topt);
 
 void show_instance_file(struct buffer_instance *instance, const char *name);
 int count_cpus(void);
trace-record.c +2 -2
index 7c6b7e9..ae0ebbf 100644
@@ -1466,7 +1466,7 @@ static void reset_events(void)
 		reset_events_instance(instance);
 }
 
-static int write_file(const char *file, const char *str, const char *type)
+int write_file(const char *file, const char *str, const char *type)
 {
 	char buf[BUFSIZ];
 	int fd;
@@ -3778,7 +3778,7 @@ static void add_hook(struct buffer_instance *instance, const char *arg)
 	}
 }
 
-static void update_first_instance(int topt)
+void update_first_instance(int topt)
 {
 	if (topt || !buffer_instances)
 		first_instance = &top_instance;
trace-snapshot.c +48 -42

128 lines not on this page · raw message

trace-usage.c +6 -2
index c56661b..56c68b4 100644
@@ -244,13 +244,17 @@ static struct usage_help usage_help[] = {
 	{
 		"snapshot",
 		"take snapshot of running trace",
-		" %s snapshot [-s][-r][-f][-B buf][-c cpu]\n"
+		" %s snapshot [-s][-r][-f][-c cpu][-B buf][-a][-t]\n"
 		"          -s take a snapshot of the trace buffer\n"
 		"          -r reset current snapshot\n"
 		"          -f free the snapshot buffer\n"
 		"            without the above three options, display snapshot\n"
 		"          -c operate on the snapshot buffer for the given CPU\n"
-		"          -B operate on the snapshot buffer for a tracing buffer instance.\n"
+		"          -B operate on the snapshot for a given instance\n"
+		"             (maybe be specified multiple times)\n"
+		"          -a operate on the snapshot for all instances\n"
+		"          -t also operate on the snapshot for top instance\n"
+		"             (useful with -B or -a)\n"
 	},
 	{
 		"stack",
Signature · 1 line
1.9.1
#5 lore raw

PATCH v3 5/5 trace-cmd stat: Support -a for all buffers. Update docs.

In reply to Howard Cochran

As for stop, restart, reset, extract, -a means show status of all buffer instances except the top one. Use -t to get the top one too.

Also added documentation for the existing -B and -t options.

---
 Documentation/trace-cmd-stat.1.txt | 19 +++++++++++++++++++
 trace-stat.c                       | 15 +++++++++++----
 trace-usage.c                      |  6 +++++-
 3 files changed, 35 insertions(+), 5 deletions(-)
Documentation/trace-cmd-stat.1.txt +19 -0
index 7e4263e..62c2e18 100644
@@ -41,6 +41,25 @@ system. The status that it shows is:
 
 *Uprobes:* Shows any uprobes that are defined for tracing.
 
+OPTIONS
+-------
+*-B* 'buffer-name'::
+    If the kernel supports multiple buffers, this will display the status of
+    of the given buffer instance. This may be used multiple times to
+    specify different buffers. When this option is used, the status of the
+    top level instance will not be shown unless *-t* is given.
+
+*-a*::
+    Display the status of all existing buffer instances. When this option
+    is used, the status of the top level instance will not be displayed
+    unless *-t* is given.
+
+*-t*::
+    Display the status of the top level instance buffer. Without the *-B*
+    or *-a* option, this is the same as the default.  But if *-B* or *-a*
+    is used, this is required if the top level instance buffer should also
+    be displayed.
+
 SEE ALSO
 --------
 trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),
trace-stat.c +11 -4
index 6c38719..9bd773e 100644
@@ -865,6 +865,12 @@ static void report_traceon(struct buffer_instance *instance)
 
 static void stat_instance(struct buffer_instance *instance)
 {
+	if (instance->name) {
+		/* Lack of trailing newline is intentional */
+		printf("\nInstance: %s\n====================",
+			instance->name);
+	}
+
 	report_plugin(instance);
 	report_events(instance);
 	report_event_filters(instance);
@@ -887,7 +893,7 @@ void trace_stat (int argc, char **argv)
 	int c;
 
 	for (;;) {
-		c = getopt(argc-1, argv+1, "tB:");
+		c = getopt(argc-1, argv+1, "hatB:");
 		if (c == -1)
 			break;
 		switch (c) {
@@ -897,9 +903,9 @@ void trace_stat (int argc, char **argv)
 		case 'B':
 			instance = create_instance(optarg);
 			add_instance(instance);
-			/* top instance requires direct access */
-			if (!topt && is_top_instance(first_instance))
-				first_instance = instance;
+			break;
+		case 'a':
+			add_all_instances();
 			break;
 		case 't':
 			/* Force to use top instance */
@@ -911,6 +917,7 @@ void trace_stat (int argc, char **argv)
 			usage(argv);
 		}
 	}
+	update_first_instance(topt);
 
 	for_all_instances(instance) {
 		stat_instance(instance);
trace-usage.c +5 -1
index 56c68b4..6e85fdf 100644
@@ -182,7 +182,11 @@ static struct usage_help usage_help[] = {
 	{
 		"stat",
 		"show the status of the running tracing (ftrace) system",
-		" %s stat"
+		" %s stat [-B buf][-a][-t]\n"
+		"          Shows the status of the tracing system"
+		"          -B status for a given buffer (more than one may be specified)\n"
+		"          -a status for all buffers (except top one)\n"
+		"          -t status the top level buffer (useful with -B or -a)\n"
 	},
 	{
 		"split",
Signature · 1 line
1.9.1