Skip to contentSkip to conversation

Add VM kernel tracing over vsock sockets

PATCH Superseded Slavomir Kaslev 2 participants 12 messages Jan 14, 2019 – Jan 15, 2019 Kbuild

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v3 cover Add VM kernel tracing over vsock sockets

This patchset adds support for tracing guest kernels to trace-cmd.

Changes in v3:
- addressed Steven's feedback
- detect and disable guest tracing if <linux/vm_sockets.h> is not available
- the --date flag is now treated as global for all guest instances
- fixed a bug that caused --date to be ignored for host tracing data when tracing guests

Changes in v2:
- rebased on top of protocol V3
- fixed system clock timestamps with the --date flag

Slavomir Kaslev (5):
trace-cmd: Minor refactoring
trace-cmd: Add tracecmd_create_recorder_virt function
trace-cmd: Add TRACE_REQ and TRACE_RESP messages
trace-cmd: Add buffer instance flags for tracing in guest and agent
context
trace-cmd: Add VM kernel tracing over vsock sockets transport

Steven Rostedt (VMware) (1):
trace-cmd: Detect if vsock sockets are available

 Makefile                       |   7 +
 include/trace-cmd/trace-cmd.h  |  13 +
 lib/trace-cmd/trace-recorder.c |  53 ++-
 tracecmd/Makefile              |   6 +-
 tracecmd/include/trace-local.h |  19 +
 tracecmd/trace-agent.c         | 229 ++++++++++
 tracecmd/trace-cmd.c           |   3 +
 tracecmd/trace-msg.c           | 199 ++++++++-
 tracecmd/trace-record.c        | 757 ++++++++++++++++++++++++++++-----
 tracecmd/trace-usage.c         |  13 +-
 10 files changed, 1185 insertions(+), 114 deletions(-)
 create mode 100644 tracecmd/trace-agent.c
Signature · 1 line
2.19.1
#1 lore raw

PATCH v3 1/6 trace-cmd: Minor refactoring

Pass struct common_record_context in trace-record.c instead of explicitly passing necessary options through additional arguments.

---
 tracecmd/trace-record.c | 91 +++++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 45 deletions(-)
tracecmd/trace-record.c +46 -45
index 01330ee..a8c3464 100644
@@ -191,6 +191,36 @@ enum {
 	RESET_HIGH_PRIO		= 100000,
 };
 
+enum trace_cmd {
+	CMD_extract,
+	CMD_start,
+	CMD_stream,
+	CMD_profile,
+	CMD_record,
+	CMD_record_agent,
+};
+
+struct common_record_context {
+	enum trace_cmd curr_cmd;
+	struct buffer_instance *instance;
+	const char *output;
+	char *date2ts;
+	char *max_graph_depth;
+	int data_flags;
+
+	int record_all;
+	int total_disable;
+	int disable;
+	int events;
+	int global;
+	int filtered;
+	int date;
+	int manual;
+	int topt;
+	int do_child;
+	int run_command;
+};
+
 static void add_reset_file(const char *file, const char *val, int prio)
 {
 	struct reset_file *reset;
@@ -2880,10 +2910,10 @@ again:
 	return msg_handle;
 }
 
-static void add_options(struct tracecmd_output *handle, char *date2ts, int flags);
+static void add_options(struct tracecmd_output *handle, struct common_record_context *ctx);
 
 static struct tracecmd_msg_handle *
-setup_connection(struct buffer_instance *instance, char *date2ts, int flags)
+setup_connection(struct buffer_instance *instance, struct common_record_context *ctx)
 {
 	struct tracecmd_msg_handle *msg_handle;
 	struct tracecmd_output *network_handle;
@@ -2893,7 +2923,7 @@ setup_connection(struct buffer_instance *instance, char *date2ts, int flags)
 	/* Now create the handle through this socket */
 	if (msg_handle->version == V3_PROTOCOL) {
 		network_handle = tracecmd_create_init_fd_msg(msg_handle, listed_events);
-		add_options(network_handle, date2ts, flags);
+		add_options(network_handle, ctx);
 		tracecmd_write_cpus(network_handle, instance->cpu_count);
 		tracecmd_write_options(network_handle);
 		tracecmd_msg_finish_sending_data(msg_handle);
@@ -2915,7 +2945,7 @@ static void finish_network(struct tracecmd_msg_handle *msg_handle)
 	free(host);
 }
 
-void start_threads(enum trace_type type, int global, char *date2ts, int flags)
+void start_threads(enum trace_type type, struct common_record_context *ctx)
 {
 	struct buffer_instance *instance;
 	int *brass = NULL;
@@ -2937,7 +2967,7 @@ void start_threads(enum trace_type type, int global, char *date2ts, int flags)
 		int x, pid;
 
 		if (host) {
-			instance->msg_handle = setup_connection(instance, date2ts, flags);
+			instance->msg_handle = setup_connection(instance, ctx);
 			if (!instance->msg_handle)
 				die("Failed to make connection");
 		}
@@ -2952,7 +2982,7 @@ void start_threads(enum trace_type type, int global, char *date2ts, int flags)
 								   brass[0],
 								   instance->cpu_count,
 								   hooks, handle_init,
-								   global);
+								   ctx->global);
 				if (!pids[i].stream)
 					die("Creating stream for %d", i);
 			} else
@@ -3091,19 +3121,19 @@ enum {
 	DATA_FL_OFFSET		= 2,
 };
 
-static void add_options(struct tracecmd_output *handle, char *date2ts, int flags)
+static void add_options(struct tracecmd_output *handle, struct common_record_context *ctx)
 {
 	int type = 0;
 
-	if (date2ts) {
-		if (flags & DATA_FL_DATE)
+	if (ctx->date2ts) {
+		if (ctx->data_flags & DATA_FL_DATE)
 			type = TRACECMD_OPTION_DATE;
-		else if (flags & DATA_FL_OFFSET)
+		else if (ctx->data_flags & DATA_FL_OFFSET)
 			type = TRACECMD_OPTION_OFFSET;
 	}
 
 	if (type)
-		tracecmd_add_option(handle, type, strlen(date2ts)+1, date2ts);
+		tracecmd_add_option(handle, type, strlen(ctx->date2ts)+1, ctx->date2ts);
 
 	tracecmd_add_option(handle, TRACECMD_OPTION_TRACECLOCK, 0, NULL);
 	add_option_hooks(handle);
@@ -3111,7 +3141,7 @@ static void add_options(struct tracecmd_output *handle, char *date2ts, int flags
 
 }
 
-static void record_data(char *date2ts, int flags)
+static void record_data(struct common_record_context *ctx)
 {
 	struct tracecmd_option **buffer_options;
 	struct tracecmd_output *handle;
@@ -3166,7 +3196,7 @@ static void record_data(char *date2ts, int flags)
 		if (!handle)
 			die("Error creating output file");
 
-		add_options(handle, date2ts, flags);
+		add_options(handle, ctx);
 
 		/* Only record the top instance under TRACECMD_OPTION_CPUSTAT*/
 		if (!no_top_instance() && !top_instance.msg_handle) {
@@ -4469,35 +4499,6 @@ void trace_reset(int argc, char **argv)
 	exit(0);
 }
 
-enum trace_cmd {
-	CMD_extract,
-	CMD_start,
-	CMD_stream,
-	CMD_profile,
-	CMD_record
-};
-
-struct common_record_context {
-	enum trace_cmd curr_cmd;
-	struct buffer_instance *instance;
-	const char *output;
-	char *date2ts;
-	char *max_graph_depth;
-	int data_flags;
-
-	int record_all;
-	int total_disable;
-	int disable;
-	int events;
-	int global;
-	int filtered;
-	int date;
-	int manual;
-	int topt;
-	int do_child;
-	int run_command;
-};
-
 static void init_common_record_context(struct common_record_context *ctx,
 				       enum trace_cmd curr_cmd)
 {
@@ -4986,7 +4987,7 @@ static void record_trace(int argc, char **argv,
 	if (type & (TRACE_TYPE_RECORD | TRACE_TYPE_STREAM)) {
 		signal(SIGINT, finish);
 		if (!latency)
-			start_threads(type, ctx->global, ctx->date2ts, ctx->data_flags);
+			start_threads(type, ctx);
 	} else {
 		update_task_filter();
 		tracecmd_enable_tracing();
@@ -5017,7 +5018,7 @@ static void record_trace(int argc, char **argv,
 		tracecmd_disable_all_tracing(0);
 
 	if (IS_RECORD(ctx)) {
-		record_data(ctx->date2ts, ctx->data_flags);
+		record_data(ctx);
 		delete_thread_data();
 	} else
 		print_stats();
@@ -5097,7 +5098,7 @@ void trace_extract(int argc, char **argv)
 		ctx.date2ts = get_date_to_ts();
 	}
 
-	record_data(ctx.date2ts, ctx.data_flags);
+	record_data(&ctx);
 	delete_thread_data();
 	destroy_stats();
 	finalize_record_trace(&ctx);
Signature · 1 line
2.19.1
#2 lore raw

PATCH v3 2/6 trace-cmd: Detect if vsock sockets are available

In reply to Slavomir Kaslev

From: "Steven Rostedt (VMware)" <[email protected]>

Detect and define VSOCK if vsock sockets are available on the system. This macro is used to disable VM remote tracing features on older kernels.

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)
Makefile +7 -0
index 27d3683..abc4f0c 100644
@@ -205,6 +205,13 @@ CFLAGS ?= -g -Wall
 CPPFLAGS ?=
 LDFLAGS ?=
 
+VSOCK_DEFINED := $(shell if (echo "\#include <linux/vm_sockets.h>" | $(CC) -E - >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi)
+
+export VSOCK_DEFINED
+ifeq ($(VSOCK_DEFINED), 1)
+CFLAGS += -DVSOCK
+endif
+
 export CFLAGS
 export INCLUDES
 
Signature · 1 line
2.19.1
#3 lore raw

PATCH v3 3/6 trace-cmd: Add tracecmd_create_recorder_virt function

In reply to Slavomir Kaslev

Add tracecmd_create_recorder_virt function that will be used for tracing VM guests.

---
 include/trace-cmd/trace-cmd.h  |  1 +
 lib/trace-cmd/trace-recorder.c | 53 ++++++++++++++++++++++++----------
 2 files changed, 39 insertions(+), 15 deletions(-)
include/trace-cmd/trace-cmd.h +1 -0
index 99a455c..f0747c5 100644
@@ -277,6 +277,7 @@ enum {
 void tracecmd_free_recorder(struct tracecmd_recorder *recorder);
 struct tracecmd_recorder *tracecmd_create_recorder(const char *file, int cpu, unsigned flags);
 struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags);
+struct tracecmd_recorder *tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd);
 struct tracecmd_recorder *tracecmd_create_recorder_maxkb(const char *file, int cpu, unsigned flags, int maxkb);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *buffer);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer);
lib/trace-cmd/trace-recorder.c +38 -15

81 lines not on this page · raw message

Signature · 1 line
2.19.1
#4 lore raw

Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function

Quoted 15 lines · On Mon, 14 Jan 2019 17:27:57 +0200 Slavomir Kaslev <[email protected]> wrote:
lib/trace-cmd/trace-recorder.c +3 -2
@@ -194,8 +196,9 @@ tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *
 	return tracecmd_create_buffer_recorder_fd2(fd, -1, cpu, flags, buffer, 0);
 }
 
-struct tracecmd_recorder *
-tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer)
+static struct tracecmd_recorder *
+__tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+				  const char *buffer)
 {
 	struct tracecmd_recorder *recorder;
 	int fd;
@@ -258,6 +261,26 @@ tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags,
 	goto out;
 }

lib/trace-cmd/trace-recorder.c tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags, · line 272

Quoted 10 lines
lib/trace-cmd/trace-recorder.c +9 -0
 
+struct tracecmd_recorder *
+tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+				const char *buffer)
+{
+	return __tracecmd_create_buffer_recorder(file, cpu, flags, buffer);
+}
+
+struct tracecmd_recorder *
+tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd)

As this looks to be something that may be a library call someday, I would keep flags as a parameter.

lib/trace-cmd/trace-recorder.c tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags, · line 277

Quoted 5 lines
lib/trace-cmd/trace-recorder.c +5 -0
+{
+	struct tracecmd_recorder *recorder;
+
+	recorder = __tracecmd_create_buffer_recorder(
+		file, cpu, TRACECMD_RECORD_NOSPLICE, NULL);

Then you could pass this as:

	recorder = __tracecmd_create_buffer_recorder(file, cpu,
			flags | TRACECMD_RECORD_NOSPLICE, NULL);

-- Steve

Quoted 9 lines
lib/trace-cmd/trace-recorder.c +6 -0
+	if (recorder)
+		recorder->trace_fd = trace_fd;
+
+	return recorder;
+}
+
 struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags)
 {
 	const char *tracing;
#5 lore raw

Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function

Quoted 42 lines · Steven Rostedt
Quoted 15 lines · On Mon, 14 Jan 2019 17:27:57 +0200 Slavomir Kaslev <[email protected]> wrote:
lib/trace-cmd/trace-recorder.c +3 -2
@@ -194,8 +196,9 @@ tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *
      return tracecmd_create_buffer_recorder_fd2(fd, -1, cpu, flags, buffer, 0);
 }
 
-struct tracecmd_recorder *
-tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer)
+static struct tracecmd_recorder *
+__tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+                               const char *buffer)
 {
      struct tracecmd_recorder *recorder;
      int fd;
@@ -258,6 +261,26 @@ tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags,
      goto out;
 }

lib/trace-cmd/trace-recorder.c tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags, · line 272

Quoted 10 lines
lib/trace-cmd/trace-recorder.c +9 -0
 
+struct tracecmd_recorder *
+tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+                             const char *buffer)
+{
+     return __tracecmd_create_buffer_recorder(file, cpu, flags, buffer);
+}
+
+struct tracecmd_recorder *
+tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd)

As this looks to be something that may be a library call someday, I would keep flags as a parameter.

lib/trace-cmd/trace-recorder.c tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags, · line 277

Quoted 5 lines
lib/trace-cmd/trace-recorder.c +5 -0
+{
+     struct tracecmd_recorder *recorder;
+
+     recorder = __tracecmd_create_buffer_recorder(
+             file, cpu, TRACECMD_RECORD_NOSPLICE, NULL);

Then you could pass this as:

Quoted 3 lines

recorder = __tracecmd_create_buffer_recorder(file, cpu,
flags | TRACECMD_RECORD_NOSPLICE, NULL);

Makes sense but (continues below)

Quoted 4 lines

-- Steve

Quoted 2 lines
lib/trace-cmd/trace-recorder.c +2 -0
+     if (recorder)
+             recorder->trace_fd = trace_fd;

it's this part that cannot be done outside of trace-recorder.c since recorder->trace_fd is not exposed. tracecmd_create_recorder_virt just allows a user to provide his own file descriptor from where tracing data should be read (in this case from the agent's vsock sockets)

On an unrelated note, the TRACECMD_RECORD_NOSPLICE flag is only needed on Linux version <4.20. Do we want to detect if splicing on vsock sockets works on startup and use this flag only if necessary?

Quoted 8 lines
Quoted 7 lines
lib/trace-cmd/trace-recorder.c +4 -0
+
+     return recorder;
+}
+
 struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags)
 {
      const char *tracing;
#6 lore raw

Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function

Quoted 3 lines · On Tue, 15 Jan 2019 14:21:11 +0000 Slavomir Kaslev <[email protected]> wrote:

On an unrelated note, the TRACECMD_RECORD_NOSPLICE flag is only needed on Linux version <4.20. Do we want to detect if splicing on vsock sockets works on startup and use this flag only if necessary?

Yes, sounds reasonable.

-- Steve

#7 lore raw

PATCH v3 4/6 trace-cmd: Add TRACE_REQ and TRACE_RESP messages

In reply to Slavomir Kaslev

Add TRACE_REQ and TRACE_RESP messages which are used for initiating guest VM tracing.

---
 include/trace-cmd/trace-cmd.h |  12 ++
 tracecmd/trace-msg.c          | 199 +++++++++++++++++++++++++++++++++-
 2 files changed, 210 insertions(+), 1 deletion(-)
include/trace-cmd/trace-cmd.h +12 -0
index f0747c5..484340a 100644
@@ -337,6 +337,18 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);
 bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);
 void tracecmd_msg_set_done(struct tracecmd_msg_handle *msg_handle);
 
+int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
+				int argc, char **argv);
+int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
+				int *argc, char ***argv);
+
+int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
+				 int nr_cpus, int page_size, int *ports);
+int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
+				 int *nr_cpus, int *page_size, int **ports);
+
+int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle);
+
 /* --- Plugin handling --- */
 extern struct tep_plugin_option trace_ftrace_options[];
 
tracecmd/trace-msg.c +198 -1

226 lines not on this page · raw message

Signature · 1 line
2.19.1
#8 lore raw

PATCH v3 5/6 trace-cmd: Add buffer instance flags for tracing in guest and agent context

In reply to Slavomir Kaslev

Add BUFFER_FL_GUEST and BUFFER_FL_AGENT flags to differentiate when trace-record.c is being called to trace guest or the VM tracing agent.

Also disable functions talking to the local tracefs when called in recording guest instances context.

---
 tracecmd/include/trace-local.h |  2 ++
 tracecmd/trace-record.c        | 55 ++++++++++++++++++++++++++++++++--
 2 files changed, 55 insertions(+), 2 deletions(-)
tracecmd/include/trace-local.h +2 -0
index a1a06e9..f19c8bb 100644
@@ -149,6 +149,8 @@ char *strstrip(char *str);
 enum buffer_instance_flags {
 	BUFFER_FL_KEEP		= 1 << 0,
 	BUFFER_FL_PROFILE	= 1 << 1,
+	BUFFER_FL_GUEST		= 1 << 2,
+	BUFFER_FL_AGENT		= 1 << 3,
 };
 
 struct func_list {
tracecmd/trace-record.c +53 -2

169 lines not on this page · raw message

Signature · 1 line
2.19.1
#9 lore raw

PATCH v3 6/6 trace-cmd: Add VM kernel tracing over vsock sockets transport

In reply to Slavomir Kaslev

To test, start a VM and assign it a valid (> 2) unused CID:

    you@host # qemu-system-x86_64 \ -name guest2 --uuid
    7edfdf4a-cb9e-11e8-b38a-173b58342476 \ -m 4096 -boot d -enable-kvm -smp 3
    -net nic -net user -hda $HOME/vm/guest2.img \ -device
    vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=3

and start trace-cmd agent on the guest as root:

you@guest2 # trace-cmd agent

Finally, start trace-cmd record on the host (running as root is only necessary if the host will be traced too):

you@host $ trace-cmd record -A guest2 -e irq -e sched

---
 tracecmd/Makefile              |   6 +-
 tracecmd/include/trace-local.h |  17 +
 tracecmd/trace-agent.c         | 229 ++++++++++++
 tracecmd/trace-cmd.c           |   3 +
 tracecmd/trace-record.c        | 613 ++++++++++++++++++++++++++++++---
 tracecmd/trace-usage.c         |  13 +-
 6 files changed, 829 insertions(+), 52 deletions(-)
 create mode 100644 tracecmd/trace-agent.c
tracecmd/Makefile +5 -1
index 3a11024..865b1c6 100644
@@ -33,13 +33,17 @@ TRACE_CMD_OBJS += trace-output.o
 TRACE_CMD_OBJS += trace-usage.o
 TRACE_CMD_OBJS += trace-msg.o
 
+ifeq ($(VSOCK_DEFINED), 1)
+TRACE_CMD_OBJS += trace-agent.o
+endif
+
 ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o)
 
 all_objs := $(sort $(ALL_OBJS))
 all_deps := $(all_objs:$(bdir)/%.o=$(bdir)/.%.d)
 
 CONFIG_INCLUDES =
-CONFIG_LIBS	=
+CONFIG_LIBS	= -lrt
 CONFIG_FLAGS	=
 
 all: $(TARGETS)
tracecmd/include/trace-local.h +17 -0
index f19c8bb..5072d8e 100644
@@ -12,6 +12,8 @@
 #include "trace-cmd.h"
 #include "event-utils.h"
 
+#define TRACE_AGENT_DEFAULT_PORT	823
+
 extern int debug;
 extern int quiet;
 
@@ -64,6 +66,8 @@ void trace_split(int argc, char **argv);
 
 void trace_listen(int argc, char **argv);
 
+void trace_agent(int argc, char **argv);
+
 void trace_restore(int argc, char **argv);
 
 void trace_clear(int argc, char **argv);
@@ -88,6 +92,10 @@ void trace_list(int argc, char **argv);
 
 void trace_usage(int argc, char **argv);
 
+int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
+		       int cpus, int *fds,
+		       int argc, char **argv);
+
 struct hook_list;
 
 void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
@@ -176,6 +184,7 @@ struct buffer_instance {
 	struct func_list	*notrace_funcs;
 
 	const char		*clock;
+	int			*client_ports;
 
 	struct trace_seq	*s_save;
 	struct trace_seq	*s_print;
@@ -190,6 +199,14 @@ struct buffer_instance {
 	int			tracing_on_fd;
 	int			buffer_size;
 	int			cpu_count;
+
+	int			argc;
+	char			**argv;
+	int			argv_cap;
+
+	int			cid;
+	int			port;
+	int			*fds;
 };
 
 extern struct buffer_instance top_instance;
tracecmd/trace-agent.c new file +229 -0

229 lines not on this page · raw message

tracecmd/trace-cmd.c +3 -0
index 797b303..3ae5e2e 100644
@@ -83,6 +83,9 @@ struct command commands[] = {
 	{"hist", trace_hist},
 	{"mem", trace_mem},
 	{"listen", trace_listen},
+#ifdef VSOCK
+	{"agent", trace_agent},
+#endif
 	{"split", trace_split},
 	{"restore", trace_restore},
 	{"stack", trace_stack},
tracecmd/trace-record.c +563 -50

865 lines not on this page · raw message

tracecmd/trace-usage.c +12 -1
index 9ea1906..93e85ba 100644
@@ -231,11 +231,22 @@ static struct usage_help usage_help[] = {
 		"listen on a network socket for trace clients",
 		" %s listen -p port[-D][-o file][-d dir][-l logfile]\n"
 		"          Creates a socket to listen for clients.\n"
-		"          -D create it in daemon mode.\n"
+		"          -p port number to listen on.\n"
+		"          -D run in daemon mode.\n"
 		"          -o file name to use for clients.\n"
 		"          -d directory to store client files.\n"
 		"          -l logfile to write messages to.\n"
 	},
+#ifdef VSOCK
+	{
+		"agent",
+		"listen on a vsock socket for trace clients",
+		" %s agent -p port[-D]\n"
+		"          Creates a vsock socket to listen for clients.\n"
+		"          -p port number to listen on.\n"
+		"          -D run in daemon mode.\n"
+	},
+#endif
 	{
 		"list",
 		"list the available events, plugins or options",
Signature · 1 line
2.19.1
#10 lore raw

Re: [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport

On Mon, 14 Jan 2019 17:28:00 +0200
Slavomir Kaslev <[email protected]> wrote:
\

Quoted 172 lines
tracecmd/trace-agent.c new file +169 -0
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2018 VMware Inc, Slavomir Kaslev <[email protected]>
+ *
+ * based on prior implementation by Yoshihiro Yunomae
+ * Copyright (C) 2013 Hitachi, Ltd.
+ * Yoshihiro YUNOMAE <[email protected]>
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <linux/vm_sockets.h>
+
+#include "trace-local.h"
+#include "trace-msg.h"
+
+static int make_vsock(unsigned port)
+{
+	struct sockaddr_vm addr = {
+		.svm_family = AF_VSOCK,
+		.svm_cid = VMADDR_CID_ANY,
+		.svm_port = port,
+	};
+	int sd;
+
+	sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+	if (sd < 0)
+		return -1;
+
+	setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
+
+	if (bind(sd, (struct sockaddr *)&addr, sizeof(addr)))
+		return -1;
+
+	if (listen(sd, SOMAXCONN))
+		return -1;
+
+	return sd;
+}
+
+static int get_vsock_port(int sd)
+{
+	struct sockaddr_vm addr;
+	socklen_t addr_len = sizeof(addr);
+
+	if (getsockname(sd, (struct sockaddr *)&addr, &addr_len))
+		return -1;
+
+	if (addr.svm_family != AF_VSOCK)
+		return -1;
+
+	return addr.svm_port;
+}
+
+static void make_vsocks(int nr, int **fds, int **ports)
+{
+	int i, fd, port;
+
+	*fds = calloc(nr, sizeof(*fds));
+	*ports = calloc(nr, sizeof(*ports));
+	if (!*fds || !*ports)
+		die("Failed to allocate memory");
+
+	for (i = 0; i < nr; i++) {
+		fd = make_vsock(VMADDR_PORT_ANY);
+		if (fd < 0)
+			die("Failed to open vsock socket");
+
+		port = get_vsock_port(fd);
+		if (port < 0)
+			die("Failed to get vsock socket address");
+
+		(*fds)[i] = fd;
+		(*ports)[i] = port;
+	}
+}
+
+static void free_vsocks(int nr, int *fds, int *ports)
+{
+	int i;
+
+	for (i = 0; i < nr; i++)
+		close(fds[i]);
+	free(fds);
+	free(ports);
+}
+
+static void agent_handle(int sd, int nr_cpus, int page_size)
+{
+	struct tracecmd_msg_handle *msg_handle;
+	int *fds, *ports;
+	char **argv = NULL;
+	int argc = 0;
+
+	msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_CLIENT);
+	if (!msg_handle)
+		die("Failed to allocate message handle");
+
+	if (tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv))
+		die("Failed to receive trace request");
+
+	make_vsocks(nr_cpus, &fds, &ports);
+
+	if (tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports))
+		die("Failed to send trace response");
+
+	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
+
+	free_vsocks(nr_cpus, fds, ports);
+	free(argv[0]);
+	free(argv);
+	tracecmd_msg_handle_close(msg_handle);
+	exit(0);
+}
+
+static volatile pid_t handler_pid;
+
+static void handle_sigchld(int sig)
+{
+	int wstatus;
+	pid_t pid;
+
+	for (;;) {
+		pid = waitpid(-1, &wstatus, WNOHANG);
+		if (pid <= 0)
+			break;
+
+		if (pid == handler_pid)
+			handler_pid = 0;
+	}
+}
+
+static void agent_serve(unsigned port)
+{
+	int sd, cd, nr_cpus;
+	pid_t pid;
+
+	signal(SIGCHLD, handle_sigchld);
+
+	nr_cpus = count_cpus();
+	page_size = getpagesize();
+
+	sd = make_vsock(port);
+	if (sd < 0)
+		die("Failed to open vsock socket");
+
+	for (;;) {
+		cd = accept(sd, NULL, NULL);
+		if (cd < 0) {
+			if (errno == EINTR)
+				continue;
+			die("accept");
+		}
+
+		if (handler_pid)
+			goto busy;
+
+		pid = fork();
+		if (pid == 0) {
+			close(sd);

tracecmd/trace-agent.c line 179

Quoted 10 lines
tracecmd/trace-agent.c new file +10 -0
+			signal(SIGCHLD, SIG_DFL);
+			agent_handle(cd, nr_cpus, page_size);
+		}
+		if (pid > 0)
+			handler_pid = pid;
+
+	busy:
+		close(cd);
+	}
+

Hmm, I don't see any break from the above for (;;) and that's an infinite loop. That makes this dead code below.

Quoted 92 lines
tracecmd/trace-agent.c new file +50 -0
+	close(sd);
+	signal(SIGCHLD, SIG_DFL);
+}
 
 
 
+
+void trace_agent(int argc, char **argv)
+{
+	bool do_daemon = false;
+	unsigned port = TRACE_AGENT_DEFAULT_PORT;
+
+	if (argc < 2)
+		usage(argv);
+
+	if (strcmp(argv[1], "agent") != 0)
+		usage(argv);
+
+	for (;;) {
+		int c, option_index = 0;
+		static struct option long_options[] = {
+			{"port", required_argument, NULL, 'p'},
+			{"help", no_argument, NULL, '?'},
+			{NULL, 0, NULL, 0}
+		};
+
+		c = getopt_long(argc-1, argv+1, "+hp:D",
+				long_options, &option_index);
+		if (c == -1)
+			break;
+		switch (c) {
+		case 'h':
+			usage(argv);
+			break;
+		case 'p':
+			port = atoi(optarg);
+			break;
+		case 'D':
+			do_daemon = true;
+			break;
+		default:
+			usage(argv);
+		}
+	}
+
+	if ((argc - optind) >= 2)
+		usage(argv);
+
+	if (do_daemon && daemon(1, 0))
+		die("daemon");
+
+	agent_serve(port);
+}
tracecmd/trace-cmd.c +3 -0
index 797b303..3ae5e2e 100644
@@ -83,6 +83,9 @@ struct command commands[] = {
 	{"hist", trace_hist},
 	{"mem", trace_mem},
 	{"listen", trace_listen},
+#ifdef VSOCK
+	{"agent", trace_agent},
+#endif
 	{"split", trace_split},
 	{"restore", trace_restore},
 	{"stack", trace_stack},
tracecmd/trace-record.c +3 -2
index 012371c..6c8754e 100644
@@ -33,6 +33,9 @@
 #include <errno.h>
 #include <limits.h>
 #include <libgen.h>
+#ifdef VSOCK
+#include <linux/vm_sockets.h>
+#endif
 
 #include "trace-local.h"
 #include "trace-msg.h"
@@ -74,8 +77,6 @@ static int buffers;
 static int clear_function_filters;
 
 static char *host;
-static int *client_ports;
-static int sfd;
 
 /* Max size to let a per cpu file get */
 static int max_kb;
@@ -171,6 +172,15 @@ static struct tracecmd_recorder *recorder;
 

tracecmd/trace-record.c static struct tracecmd_recorder *recorder; · line 182

Quoted 10 lines
tracecmd/trace-record.c +8 -0
 static int ignore_event_not_found = 0;
 
+static inline size_t grow_cap(size_t old_cap)
+{
+	size_t cap = 3 * old_cap / 2;
+
+	if (cap < 16)
+		cap = 16;
+	return cap;
+}

I'm a bit confused by what the grow_cap() is for.

Quoted 24 lines
tracecmd/trace-record.c +17 -0
+
 static inline int is_top_instance(struct buffer_instance *instance)
 {
 	return instance == &top_instance;
@@ -518,6 +528,36 @@ static char *get_temp_file(struct buffer_instance *instance, int cpu)
 	return file;
 }
 
+static char *get_guest_file(const char *file, const char *guest)
+{
+	size_t guest_len = strlen(guest);
+	size_t file_len = strlen(file);
+	size_t base_len, idx = 0;
+	const char *p;
+	char *out;
+
+	out = malloc(file_len + guest_len + 2 /* dash and \0 */);
+	if (!out)
+		return NULL;
+
+	p = strrchr(file, '.');
+	if (p && p != file)
+		base_len = p - file;
+	else

tracecmd/trace-record.c static char *get_temp_file(struct buffer_instance *instance, int cpu) · line 556

Quoted 10 lines
tracecmd/trace-record.c +10 -0
+		base_len = file_len;
+
+	memcpy(out, file, base_len);
+	idx += base_len;
+	out[idx++] = '-';
+	memcpy(out + idx, guest, guest_len);
+	idx += guest_len;
+	memcpy(out + idx, p, file_len - base_len);
+	idx += file_len - base_len;
+	out[idx] = '\0';

Can't we pretty much replace this whole function with:

	p = strrchr(file, '.');
	if (p && p != file) {
		base_len = p - file;
	else
		base_len = strlen(file);

	asprintf(&out, "%.*s-%s%s", base_len, file, guest, file + base_len);

?

Quoted 187 lines
tracecmd/trace-record.c +150 -4
+
+	return out;
+}
+
 static void put_temp_file(char *file)
 {
 	free(file);
@@ -623,6 +663,16 @@ static void delete_thread_data(void)
 	}
 }
 
+static void tell_guests_to_stop(void)
+{
+	struct buffer_instance *instance;
+
+	for_all_instances(instance) {
+		if (instance->flags & BUFFER_FL_GUEST)
+			tracecmd_msg_handle_close(instance->msg_handle);
+	}
+}
+
 static void stop_threads(enum trace_type type)
 {
 	struct timeval tv = { 0, 0 };
@@ -632,6 +682,8 @@ static void stop_threads(enum trace_type type)
 	if (!recorder_threads)
 		return;
 
+	tell_guests_to_stop();
+
 	/* Tell all threads to finish up */
 	for (i = 0; i < recorder_threads; i++) {
 		if (pids[i].pid > 0) {
@@ -2571,14 +2623,14 @@ static void flush(int sig)
 		tracecmd_stop_recording(recorder);
 }
 
-static void connect_port(int cpu)
+static int connect_port(const char *host, unsigned port)
 {
 	struct addrinfo hints;
 	struct addrinfo *results, *rp;
-	int s;
+	int s, sfd;
 	char buf[BUFSIZ];
 
-	snprintf(buf, BUFSIZ, "%d", client_ports[cpu]);
+	snprintf(buf, BUFSIZ, "%d", port);
 
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = AF_UNSPEC;
@@ -2605,7 +2657,190 @@ static void connect_port(int cpu)
 
 	freeaddrinfo(results);
 
-	client_ports[cpu] = sfd;
+	return sfd;
+}
+
+#ifdef VSOCK
+static int open_vsock(unsigned cid, unsigned port)
+{
+	struct sockaddr_vm addr = {
+		.svm_family = AF_VSOCK,
+		.svm_cid = cid,
+		.svm_port = port,
+	};
+	int sd;
+
+	sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+	if (sd < 0)
+		return -1;
+
+	if (connect(sd, (struct sockaddr *)&addr, sizeof(addr)))
+		return -1;
+
+	return sd;
+}
+#else
+static inline int open_vsock(unsigned cid, unsigned port)
+{
+	die("vsock is not supported");
+	return -1;
+}
+#endif
+
+static int do_accept(int sd)
+{
+	int cd;
+
+	for (;;) {
+		cd = accept(sd, NULL, NULL);
+		if (cd < 0) {
+			if (errno == EINTR)
+				continue;
+			die("accept");
+		}
+
+		return cd;
+	}
+
+	return -1;
+}
+
+static bool is_digits(const char *s)
+{
+	const char *p;
+
+	for (p = s; *p; p++)
+		if (!isdigit(*p))
+			return false;
+
+	return true;
+}
+
+struct guest {
+	char *name;
+	int cid;
+	int pid;
+};
+
+static size_t guests_cap, guests_len;
+static struct guest *guests;
+
+static char *get_qemu_guest_name(char *arg)
+{
+	char *tok, *end = arg;
+
+	while ((tok = strsep(&end, ","))) {
+		if (strncmp(tok, "guest=", 6) == 0)
+			return tok + 6;
+	}
+
+	return arg;
+}
+
+static void read_qemu_guests(void)
+{
+	static bool initialized = false;
+	struct dirent *entry;
+	char path[PATH_MAX];
+	DIR *dir;
+
+	if (initialized)
+		return;
+
+	initialized = true;
+	dir = opendir("/proc");
+	if (!dir)
+		die("opendir");
+
+	for (entry = readdir(dir); entry; entry = readdir(dir)) {
+		bool is_qemu = false, last_was_name = false;
+		struct guest guest = {};
+		char *p, *arg = NULL;
+		size_t arg_size = 0;
+		FILE *f;
+
+		if (!(entry->d_type == DT_DIR && is_digits(entry->d_name)))
+			continue;
+
+		guest.pid = atoi(entry->d_name);
+		snprintf(path, sizeof(path), "/proc/%s/cmdline", entry->d_name);
+		f = fopen(path, "r");
+		if (!f)
+			continue;
+
+		while (getdelim(&arg, &arg_size, 0, f) != -1) {
+			if (!is_qemu && strstr(arg, "qemu-system-")) {
+				is_qemu = true;
+				continue;
+			}
+
+			if (!is_qemu)
+				continue;
+
+			if (strcmp(arg, "-name") == 0) {
+				last_was_name = true;
+				continue;
+			}
+
+			if (last_was_name) {
+				guest.name = strdup(get_qemu_guest_name(arg));
+				last_was_name = false;
+				continue;
+			}
+

tracecmd/trace-record.c static void connect_port(int cpu) · line 2800

Quoted 10 lines
tracecmd/trace-record.c +10 -0
+			p = strstr(arg, "guest-cid=");
+			if (p) {
+				guest.cid = atoi(p + 10);
+				continue;
+			}
+		}
+
+		if (is_qemu) {
+			if (guests_cap == guests_len) {
+				guests_cap = grow_cap(guests_cap);

This isn't a fast path is it? I'm not sure why we are using guests_cap() here. Note, realloc allocs more than required and is pretty much a nop if the allocated amount is still within its allocation that it made the first time.

-- Steve

Quoted 14 lines
tracecmd/trace-record.c +13 -0
+				guests = realloc(guests,
+						 guests_cap * sizeof(*guests));
+			}
+			guests[guests_len++] = guest;
+		}
+
+		free(arg);
+		fclose(f);
+	}
+
+	closedir(dir);
+}
+
 
#11 lore raw

Re: [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport

Quoted 284 lines · Steven Rostedt

On Mon, 14 Jan 2019 17:28:00 +0200
Slavomir Kaslev <[email protected]> wrote:
\

Quoted 172 lines
tracecmd/trace-agent.c new file +169 -0
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2018 VMware Inc, Slavomir Kaslev <[email protected]>
+ *
+ * based on prior implementation by Yoshihiro Yunomae
+ * Copyright (C) 2013 Hitachi, Ltd.
+ * Yoshihiro YUNOMAE <[email protected]>
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <linux/vm_sockets.h>
+
+#include "trace-local.h"
+#include "trace-msg.h"
+
+static int make_vsock(unsigned port)
+{
+     struct sockaddr_vm addr = {
+             .svm_family = AF_VSOCK,
+             .svm_cid = VMADDR_CID_ANY,
+             .svm_port = port,
+     };
+     int sd;
+
+     sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+     if (sd < 0)
+             return -1;
+
+     setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
+
+     if (bind(sd, (struct sockaddr *)&addr, sizeof(addr)))
+             return -1;
+
+     if (listen(sd, SOMAXCONN))
+             return -1;
+
+     return sd;
+}
+
+static int get_vsock_port(int sd)
+{
+     struct sockaddr_vm addr;
+     socklen_t addr_len = sizeof(addr);
+
+     if (getsockname(sd, (struct sockaddr *)&addr, &addr_len))
+             return -1;
+
+     if (addr.svm_family != AF_VSOCK)
+             return -1;
+
+     return addr.svm_port;
+}
+
+static void make_vsocks(int nr, int **fds, int **ports)
+{
+     int i, fd, port;
+
+     *fds = calloc(nr, sizeof(*fds));
+     *ports = calloc(nr, sizeof(*ports));
+     if (!*fds || !*ports)
+             die("Failed to allocate memory");
+
+     for (i = 0; i < nr; i++) {
+             fd = make_vsock(VMADDR_PORT_ANY);
+             if (fd < 0)
+                     die("Failed to open vsock socket");
+
+             port = get_vsock_port(fd);
+             if (port < 0)
+                     die("Failed to get vsock socket address");
+
+             (*fds)[i] = fd;
+             (*ports)[i] = port;
+     }
+}
+
+static void free_vsocks(int nr, int *fds, int *ports)
+{
+     int i;
+
+     for (i = 0; i < nr; i++)
+             close(fds[i]);
+     free(fds);
+     free(ports);
+}
+
+static void agent_handle(int sd, int nr_cpus, int page_size)
+{
+     struct tracecmd_msg_handle *msg_handle;
+     int *fds, *ports;
+     char **argv = NULL;
+     int argc = 0;
+
+     msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_CLIENT);
+     if (!msg_handle)
+             die("Failed to allocate message handle");
+
+     if (tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv))
+             die("Failed to receive trace request");
+
+     make_vsocks(nr_cpus, &fds, &ports);
+
+     if (tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports))
+             die("Failed to send trace response");
+
+     trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
+
+     free_vsocks(nr_cpus, fds, ports);
+     free(argv[0]);
+     free(argv);
+     tracecmd_msg_handle_close(msg_handle);
+     exit(0);
+}
+
+static volatile pid_t handler_pid;
+
+static void handle_sigchld(int sig)
+{
+     int wstatus;
+     pid_t pid;
+
+     for (;;) {
+             pid = waitpid(-1, &wstatus, WNOHANG);
+             if (pid <= 0)
+                     break;
+
+             if (pid == handler_pid)
+                     handler_pid = 0;
+     }
+}
+
+static void agent_serve(unsigned port)
+{
+     int sd, cd, nr_cpus;
+     pid_t pid;
+
+     signal(SIGCHLD, handle_sigchld);
+
+     nr_cpus = count_cpus();
+     page_size = getpagesize();
+
+     sd = make_vsock(port);
+     if (sd < 0)
+             die("Failed to open vsock socket");
+
+     for (;;) {
+             cd = accept(sd, NULL, NULL);
+             if (cd < 0) {
+                     if (errno == EINTR)
+                             continue;
+                     die("accept");
+             }
+
+             if (handler_pid)
+                     goto busy;
+
+             pid = fork();
+             if (pid == 0) {
+                     close(sd);

tracecmd/trace-agent.c line 179

Quoted 10 lines
tracecmd/trace-agent.c new file +10 -0
+                     signal(SIGCHLD, SIG_DFL);
+                     agent_handle(cd, nr_cpus, page_size);
+             }
+             if (pid > 0)
+                     handler_pid = pid;
+
+     busy:
+             close(cd);
+     }
+

Hmm, I don't see any break from the above for (;;) and that's an infinite loop. That makes this dead code below.

Quoted 94 lines
tracecmd/trace-agent.c new file +50 -0
+     close(sd);
+     signal(SIGCHLD, SIG_DFL);
+}
 
 
 
+
+void trace_agent(int argc, char **argv)
+{
+     bool do_daemon = false;
+     unsigned port = TRACE_AGENT_DEFAULT_PORT;
+
+     if (argc < 2)
+             usage(argv);
+
+     if (strcmp(argv[1], "agent") != 0)
+             usage(argv);
+
+     for (;;) {
+             int c, option_index = 0;
+             static struct option long_options[] = {
+                     {"port", required_argument, NULL, 'p'},
+                     {"help", no_argument, NULL, '?'},
+                     {NULL, 0, NULL, 0}
+             };
+
+             c = getopt_long(argc-1, argv+1, "+hp:D",
+                             long_options, &option_index);
+             if (c == -1)
+                     break;
+             switch (c) {
+             case 'h':
+                     usage(argv);
+                     break;
+             case 'p':
+                     port = atoi(optarg);
+                     break;
+             case 'D':
+                     do_daemon = true;
+                     break;
+             default:
+                     usage(argv);
+             }
+     }
+
+     if ((argc - optind) >= 2)
+             usage(argv);
+
+     if (do_daemon && daemon(1, 0))
+             die("daemon");
+
+     agent_serve(port);
+}
tracecmd/trace-cmd.c +3 -0
index 797b303..3ae5e2e 100644
@@ -83,6 +83,9 @@ struct command commands[] = {
      {"hist", trace_hist},
      {"mem", trace_mem},
      {"listen", trace_listen},
+#ifdef VSOCK
+     {"agent", trace_agent},
+#endif
      {"split", trace_split},
      {"restore", trace_restore},
      {"stack", trace_stack},
tracecmd/trace-record.c +3 -2
index 012371c..6c8754e 100644
@@ -33,6 +33,9 @@
 #include <errno.h>
 #include <limits.h>
 #include <libgen.h>
+#ifdef VSOCK
+#include <linux/vm_sockets.h>
+#endif
 
 #include "trace-local.h"
 #include "trace-msg.h"
@@ -74,8 +77,6 @@ static int buffers;
 static int clear_function_filters;
 
 static char *host;
-static int *client_ports;
-static int sfd;
 
 /* Max size to let a per cpu file get */
 static int max_kb;
@@ -171,6 +172,15 @@ static struct tracecmd_recorder *recorder;
 
 static int ignore_event_not_found = 0;
 
Quoted 10 lines

tracecmd/trace-record.c static struct tracecmd_recorder *recorder; · line 182

Quoted 8 lines
tracecmd/trace-record.c +8 -0
+static inline size_t grow_cap(size_t old_cap)
+{
+     size_t cap = 3 * old_cap / 2;
+
+     if (cap < 16)
+             cap = 16;
+     return cap;
+}

I'm a bit confused by what the grow_cap() is for.

I did some benchmarking awhile ago and calling realloc on each new entry when appending is indeed linear (which implies that realloc amortizes to constant).
grow_cap() grows the buffer exponentially and hence calls realloc only log(n) times, which turns out to be again linear but with much smaller constant.
Git uses something similar they call ALLOC_GROW:
https://github.com/git/git/blob/0f086e6dcabfbf059e7483ebd7d41080faec3d77/cache.h#L650

I agree that neither of the current uses of grow_cap are not on performance critical paths so I'll drop it in the next version of this series.

Quoted 46 lines
Quoted 25 lines
tracecmd/trace-record.c +17 -0
+
 static inline int is_top_instance(struct buffer_instance *instance)
 {
      return instance == &top_instance;
@@ -518,6 +528,36 @@ static char *get_temp_file(struct buffer_instance *instance, int cpu)
      return file;
 }
 
+static char *get_guest_file(const char *file, const char *guest)
+{
+     size_t guest_len = strlen(guest);
+     size_t file_len = strlen(file);
+     size_t base_len, idx = 0;
+     const char *p;
+     char *out;
+
+     out = malloc(file_len + guest_len + 2 /* dash and \0 */);
+     if (!out)
+             return NULL;
+
+     p = strrchr(file, '.');
+     if (p && p != file)
+             base_len = p - file;
+     else

tracecmd/trace-record.c static char *get_temp_file(struct buffer_instance *instance, int cpu) · line 556

Quoted 10 lines
tracecmd/trace-record.c +10 -0
+             base_len = file_len;
+
+     memcpy(out, file, base_len);
+     idx += base_len;
+     out[idx++] = '-';
+     memcpy(out + idx, guest, guest_len);
+     idx += guest_len;
+     memcpy(out + idx, p, file_len - base_len);
+     idx += file_len - base_len;
+     out[idx] = '\0';

Can't we pretty much replace this whole function with:

        p = strrchr(file, '.');
        if (p && p != file) {
                base_len = p - file;
        else
                base_len = strlen(file);

        asprintf(&out, "%.*s-%s%s", base_len, file, guest, file + base_len);
Quoted 1 line

?

Good catch. This is a leftover from the virt-server branch which I simplified but didn't know about the %.*s format magic.

Quoted 222 lines
Quoted 189 lines
tracecmd/trace-record.c +150 -4
+
+     return out;
+}
+
 static void put_temp_file(char *file)
 {
      free(file);
@@ -623,6 +663,16 @@ static void delete_thread_data(void)
      }
 }
 
+static void tell_guests_to_stop(void)
+{
+     struct buffer_instance *instance;
+
+     for_all_instances(instance) {
+             if (instance->flags & BUFFER_FL_GUEST)
+                     tracecmd_msg_handle_close(instance->msg_handle);
+     }
+}
+
 static void stop_threads(enum trace_type type)
 {
      struct timeval tv = { 0, 0 };
@@ -632,6 +682,8 @@ static void stop_threads(enum trace_type type)
      if (!recorder_threads)
              return;
 
+     tell_guests_to_stop();
+
      /* Tell all threads to finish up */
      for (i = 0; i < recorder_threads; i++) {
              if (pids[i].pid > 0) {
@@ -2571,14 +2623,14 @@ static void flush(int sig)
              tracecmd_stop_recording(recorder);
 }
 
-static void connect_port(int cpu)
+static int connect_port(const char *host, unsigned port)
 {
      struct addrinfo hints;
      struct addrinfo *results, *rp;
-     int s;
+     int s, sfd;
      char buf[BUFSIZ];
 
-     snprintf(buf, BUFSIZ, "%d", client_ports[cpu]);
+     snprintf(buf, BUFSIZ, "%d", port);
 
      memset(&hints, 0, sizeof(hints));
      hints.ai_family = AF_UNSPEC;
@@ -2605,7 +2657,190 @@ static void connect_port(int cpu)
 
      freeaddrinfo(results);
 
-     client_ports[cpu] = sfd;
+     return sfd;
+}
+
+#ifdef VSOCK
+static int open_vsock(unsigned cid, unsigned port)
+{
+     struct sockaddr_vm addr = {
+             .svm_family = AF_VSOCK,
+             .svm_cid = cid,
+             .svm_port = port,
+     };
+     int sd;
+
+     sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+     if (sd < 0)
+             return -1;
+
+     if (connect(sd, (struct sockaddr *)&addr, sizeof(addr)))
+             return -1;
+
+     return sd;
+}
+#else
+static inline int open_vsock(unsigned cid, unsigned port)
+{
+     die("vsock is not supported");
+     return -1;
+}
+#endif
+
+static int do_accept(int sd)
+{
+     int cd;
+
+     for (;;) {
+             cd = accept(sd, NULL, NULL);
+             if (cd < 0) {
+                     if (errno == EINTR)
+                             continue;
+                     die("accept");
+             }
+
+             return cd;
+     }
+
+     return -1;
+}
+
+static bool is_digits(const char *s)
+{
+     const char *p;
+
+     for (p = s; *p; p++)
+             if (!isdigit(*p))
+                     return false;
+
+     return true;
+}
+
+struct guest {
+     char *name;
+     int cid;
+     int pid;
+};
+
+static size_t guests_cap, guests_len;
+static struct guest *guests;
+
+static char *get_qemu_guest_name(char *arg)
+{
+     char *tok, *end = arg;
+
+     while ((tok = strsep(&end, ","))) {
+             if (strncmp(tok, "guest=", 6) == 0)
+                     return tok + 6;
+     }
+
+     return arg;
+}
+
+static void read_qemu_guests(void)
+{
+     static bool initialized = false;
+     struct dirent *entry;
+     char path[PATH_MAX];
+     DIR *dir;
+
+     if (initialized)
+             return;
+
+     initialized = true;
+     dir = opendir("/proc");
+     if (!dir)
+             die("opendir");
+
+     for (entry = readdir(dir); entry; entry = readdir(dir)) {
+             bool is_qemu = false, last_was_name = false;
+             struct guest guest = {};
+             char *p, *arg = NULL;
+             size_t arg_size = 0;
+             FILE *f;
+
+             if (!(entry->d_type == DT_DIR && is_digits(entry->d_name)))
+                     continue;
+
+             guest.pid = atoi(entry->d_name);
+             snprintf(path, sizeof(path), "/proc/%s/cmdline", entry->d_name);
+             f = fopen(path, "r");
+             if (!f)
+                     continue;
+
+             while (getdelim(&arg, &arg_size, 0, f) != -1) {
+                     if (!is_qemu && strstr(arg, "qemu-system-")) {
+                             is_qemu = true;
+                             continue;
+                     }
+
+                     if (!is_qemu)
+                             continue;
+
+                     if (strcmp(arg, "-name") == 0) {
+                             last_was_name = true;
+                             continue;
+                     }
+
+                     if (last_was_name) {
+                             guest.name = strdup(get_qemu_guest_name(arg));
+                             last_was_name = false;
+                             continue;
+                     }
+

tracecmd/trace-record.c static void connect_port(int cpu) · line 2800

Quoted 10 lines
tracecmd/trace-record.c +10 -0
+                     p = strstr(arg, "guest-cid=");
+                     if (p) {
+                             guest.cid = atoi(p + 10);
+                             continue;
+                     }
+             }
+
+             if (is_qemu) {
+                     if (guests_cap == guests_len) {
+                             guests_cap = grow_cap(guests_cap);

This isn't a fast path is it? I'm not sure why we are using guests_cap() here. Note, realloc allocs more than required and is pretty much a nop if the allocated amount is still within its allocation that it made the first time.

-- Steve

Quoted 14 lines
tracecmd/trace-record.c +13 -0
+                             guests = realloc(guests,
+                                              guests_cap * sizeof(*guests));
+                     }
+                     guests[guests_len++] = guest;
+             }
+
+             free(arg);
+             fclose(f);
+     }
+
+     closedir(dir);
+}
+