Skip to contentSkip to conversation

Add VM kernel tracing over vsockets and FIFOs

RFC PATCH Superseded Slavomir Kaslev 1 participant 14 messages Feb 18, 2019 Kbuild

v7 of 15 Superseded Latest is v15 →

lore
Share & tools
lore
#0 lore raw

v7 cover Add VM kernel tracing over vsockets and FIFOs

This patchset adds support for tracing guest kernels to trace-cmd.

The first two patches in the series are minor cleanups which can be merged separately.

Changes in v7:
- fixed potential memory leak in add_arg()
- call msg_free() in tracecmd_msg_recv_trace_req() eventhough we're stealing the message buffer
- added is_guest and is_agent macros and use throughout tracecmd_record.c instead of testing instance->flags bits
- avoid memory allocation in make_dir()

Changes in v6:
- added specialized data transfer path over FIFOs making single splice() per iteration
- made tracecmd_msg_recv_trace_req/tracecmd_msg_recv_trace_resp more defensive to invalid messages

Changes in v5:
- add FIFOs transport for tracing data
- fixed a bug in tracecmd_msg_wait_close

Changes in v4:
- detect and use splice(2) on vsock sockets if possible
- switch port numbers to unsigned int
- trace-cmd record --date flag is now set for all guests if provided by the user
- removed grow_cap and exponential buffer size growth

Changes in v3:
- addressed Steven's feedback
- detect and disable guest tracing if <linux/vm_sockets.h> is not available
- the --date flag is now treated as global for all guest instances
- fixed a bug that caused --date to be ignored for host tracing data when tracing guests

Changes in v2:
- rebased on top of protocol V3
- fixed system clock timestamps with the --date flag

Slavomir Kaslev (12):
trace-cmd: Minor cleanup in tracecmd_start_recording()
trace-cmd: Minor cleanup in print_stat()
trace-cmd: Add tracecmd_create_recorder_virt function
trace-cmd: Add TRACE_REQ and TRACE_RESP messages
trace-cmd: Add buffer instance flags for tracing in guest and agent
context
trace-cmd: Add VM kernel tracing over vsockets transport
trace-cmd: Use splice(2) for vsockets if available
trace-cmd: Add trace-cmd setup-guest command
trace-cmd: Try to autodetect number of guest CPUs in setup-guest if
not specified
trace-cmd: Add setup-guest flag for attaching FIFOs to the guest VM
config
trace-cmd: Add splice() recording from FIFO without additional pipe
buffer
trace-cmd: Add VM tracing over FIFOs transport

Steven Rostedt (VMware) (1):
trace-cmd: Detect if vsockets are available

 Makefile                       |   7 +
 include/trace-cmd/trace-cmd.h  |  16 +-
 lib/trace-cmd/trace-recorder.c | 140 ++++--
 tracecmd/Makefile              |   7 +-
 tracecmd/include/trace-local.h |  29 ++
 tracecmd/trace-agent.c         | 257 +++++++++++
 tracecmd/trace-cmd.c           |   4 +
 tracecmd/trace-msg.c           | 219 ++++++++-
 tracecmd/trace-record.c        | 779 ++++++++++++++++++++++++++++++---
 tracecmd/trace-setup-guest.c   | 245 +++++++++++
 tracecmd/trace-usage.c         |  22 +-
 11 files changed, 1637 insertions(+), 88 deletions(-)
 create mode 100644 tracecmd/trace-agent.c
 create mode 100644 tracecmd/trace-setup-guest.c

--
2.19.1

#1 lore raw

RFC PATCH v7 1/13 trace-cmd: Minor cleanup in tracecmd_start_recording()

Initialize struct timespec req only once instead of on every loop iteration since sleep is being constant throughout tracecmd_start_recording() execution.

No changes in behavior intended.

---
 lib/trace-cmd/trace-recorder.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
lib/trace-cmd/trace-recorder.c +6 -5
index 0b8d98a..0bd0657 100644
@@ -466,7 +466,10 @@ long tracecmd_flush_recording(struct tracecmd_recorder *recorder)
 
 int tracecmd_start_recording(struct tracecmd_recorder *recorder, unsigned long sleep)
 {
-	struct timespec req;
+	struct timespec req = {
+		.tv_sec = sleep / 1000000,
+		.tv_nsec = (sleep % 1000000) * 1000,
+	};
 	long read = 1;
 	long ret;
 
@@ -474,11 +477,9 @@ int tracecmd_start_recording(struct tracecmd_recorder *recorder, unsigned long s
 
 	do {
 		/* Only sleep if we did not read anything last time */
-		if (!read && sleep) {
-			req.tv_sec = sleep / 1000000;
-			req.tv_nsec = (sleep % 1000000) * 1000;
+		if (!read && sleep)
 			nanosleep(&req, NULL);
-		}
+
 		read = 0;
 		do {
 			if (recorder->flags & TRACECMD_RECORD_NOSPLICE)
Signature · 1 line
2.19.1
#2 lore raw

RFC PATCH v7 2/13 trace-cmd: Minor cleanup in print_stat()

In reply to Slavomir Kaslev

Do an early out in print_stat() when quiet is set instead of rechecking it on each loop iteration.

No changes in behavior intended.

---
 tracecmd/trace-record.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
tracecmd/trace-record.c +5 -4
index 8beefab..88b4a55 100644
@@ -3106,13 +3106,14 @@ static void print_stat(struct buffer_instance *instance)
 {
 	int cpu;
 
+	if (quiet)
+		return;
+
 	if (!is_top_instance(instance))
-		if (!quiet)
-			printf("\nBuffer: %s\n\n", instance->name);
+		printf("\nBuffer: %s\n\n", instance->name);
 
 	for (cpu = 0; cpu < instance->cpu_count; cpu++)
-		if (!quiet)
-			trace_seq_do_printf(&instance->s_print[cpu]);
+		trace_seq_do_printf(&instance->s_print[cpu]);
 }
 
 enum {
Signature · 1 line
2.19.1
#3 lore raw

RFC PATCH v7 3/13 trace-cmd: Detect if vsockets are available

In reply to Slavomir Kaslev

From: "Steven Rostedt (VMware)" <[email protected]>

Detect and define VSOCK if vsockets are available on the system. This macro is used to disable VM remote tracing features on older kernels.

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)
Makefile +7 -0
index b780718..99e4f39 100644
@@ -204,6 +204,13 @@ CFLAGS ?= -g -Wall
 CPPFLAGS ?=
 LDFLAGS ?=
 
+VSOCK_DEFINED := $(shell if (echo "\#include <linux/vm_sockets.h>" | $(CC) -E - >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi)
+
+export VSOCK_DEFINED
+ifeq ($(VSOCK_DEFINED), 1)
+CFLAGS += -DVSOCK
+endif
+
 export CFLAGS
 export INCLUDES
 
Signature · 1 line
2.19.1
#4 lore raw

RFC PATCH v7 4/13 trace-cmd: Add tracecmd_create_recorder_virt function

In reply to Slavomir Kaslev

Add tracecmd_create_recorder_virt function that will be used for tracing VM guests.

---
 include/trace-cmd/trace-cmd.h  |  1 +
 lib/trace-cmd/trace-recorder.c | 53 ++++++++++++++++++++++++----------
 2 files changed, 39 insertions(+), 15 deletions(-)
include/trace-cmd/trace-cmd.h +1 -0
index ca4452b..5961355 100644
@@ -277,6 +277,7 @@ enum {
 void tracecmd_free_recorder(struct tracecmd_recorder *recorder);
 struct tracecmd_recorder *tracecmd_create_recorder(const char *file, int cpu, unsigned flags);
 struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags);
+struct tracecmd_recorder *tracecmd_create_recorder_virt(const char *file, int cpu, unsigned flags, int trace_fd);
 struct tracecmd_recorder *tracecmd_create_recorder_maxkb(const char *file, int cpu, unsigned flags, int maxkb);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *buffer);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer);
lib/trace-cmd/trace-recorder.c +38 -15

81 lines not on this page · raw message

Signature · 1 line
2.19.1
#5 lore raw

RFC PATCH v7 5/13 trace-cmd: Add TRACE_REQ and TRACE_RESP messages

In reply to Slavomir Kaslev

Add TRACE_REQ and TRACE_RESP messages which are used for initiating guest VM tracing.

---
 include/trace-cmd/trace-cmd.h |  12 ++
 tracecmd/trace-msg.c          | 211 +++++++++++++++++++++++++++++++++-
 2 files changed, 222 insertions(+), 1 deletion(-)
include/trace-cmd/trace-cmd.h +12 -0
index 5961355..6a21e66 100644
@@ -331,6 +331,18 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);
 bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);
 void tracecmd_msg_set_done(struct tracecmd_msg_handle *msg_handle);
 
+int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
+				int argc, char **argv);
+int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
+				int *argc, char ***argv);
+
+int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
+				 int nr_cpus, int page_size,
+				 unsigned int *ports);
+int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
+				 int *nr_cpus, int *page_size,
+				 unsigned int **ports);
+
 /* --- Plugin handling --- */
 extern struct tep_plugin_option trace_ftrace_options[];
 
tracecmd/trace-msg.c +210 -1

238 lines not on this page · raw message

Signature · 1 line
2.19.1
#6 lore raw

RFC PATCH v7 6/13 trace-cmd: Add buffer instance flags for tracing in guest and agent context

In reply to Slavomir Kaslev

Add BUFFER_FL_GUEST and BUFFER_FL_AGENT flags to differentiate when trace-record.c is being called to trace guest or the VM tracing agent.

Also disable functions talking to the local tracefs when called in recording guest instances context.

---
 tracecmd/include/trace-local.h |  5 ++++
 tracecmd/trace-record.c        | 55 ++++++++++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 2 deletions(-)
tracecmd/include/trace-local.h +5 -0
index a1a06e9..f43c9d8 100644
@@ -149,6 +149,8 @@ char *strstrip(char *str);
 enum buffer_instance_flags {
 	BUFFER_FL_KEEP		= 1 << 0,
 	BUFFER_FL_PROFILE	= 1 << 1,
+	BUFFER_FL_GUEST		= 1 << 2,
+	BUFFER_FL_AGENT		= 1 << 3,
 };
 
 struct func_list {
@@ -198,6 +200,9 @@ extern struct buffer_instance *first_instance;
 #define for_all_instances(i) for (i = first_instance; i; \
 				  i = i == &top_instance ? buffer_instances : (i)->next)
 
+#define is_agent(instance)	((instance)->flags & BUFFER_FL_AGENT)
+#define is_guest(instance)	((instance)->flags & BUFFER_FL_GUEST)
+
 struct buffer_instance *create_instance(const char *name);
 void add_instance(struct buffer_instance *instance, int cpu_count);
 char *get_instance_file(struct buffer_instance *instance, const char *file);
tracecmd/trace-record.c +53 -2

169 lines not on this page · raw message

Signature · 1 line
2.19.1
#8 lore raw

RFC PATCH v7 8/13 trace-cmd: Use splice(2) for vsockets if available

In reply to Slavomir Kaslev

Detect if splice(2) reading is supported for vsockets (Linux 4.20 and later) and use it, or fallback to read/write-ing otherwise.

---
 lib/trace-cmd/trace-recorder.c |  6 ++--
 tracecmd/trace-record.c        | 56 ++++++++++++++++++++++++++++++++--
 2 files changed, 56 insertions(+), 6 deletions(-)
lib/trace-cmd/trace-recorder.c +2 -4
index 1b84bb5..d66f112 100644
@@ -386,10 +386,8 @@ static long splice_data(struct tracecmd_recorder *recorder)
 	ret = splice(recorder->brass[0], NULL, recorder->fd, NULL,
 		     read, recorder->fd_flags);
 	if (ret < 0) {
-		if (errno != EAGAIN && errno != EINTR) {
-			warning("recorder error in splice output");
-			return -1;
-		}
+		if (errno != EAGAIN && errno != EINTR)
+			return ret;
 		return total_read;
 	} else
 		update_fd(recorder, ret);
tracecmd/trace-record.c +54 -2

79 lines not on this page · raw message

Signature · 1 line
2.19.1
#7 lore raw

RFC PATCH v7 7/13 trace-cmd: Add VM kernel tracing over vsockets transport

In reply to Slavomir Kaslev

This patch adds VM tracing over vsockets. The new trace-cmd agent command needs run on each guest we want to trace:

you@guest2 # trace-cmd agent

Then trace-cmd record on the host can collect data from both the host and several guests simultaneously:

you@host $ trace-cmd record -A guest1 -e irq -e sched \
-A guest2 -p function -e all

---
 tracecmd/Makefile              |   6 +-
 tracecmd/include/trace-local.h |  16 +
 tracecmd/trace-agent.c         | 225 +++++++++++++
 tracecmd/trace-cmd.c           |   3 +
 tracecmd/trace-record.c        | 596 ++++++++++++++++++++++++++++++---
 tracecmd/trace-usage.c         |  13 +-
 6 files changed, 814 insertions(+), 45 deletions(-)
 create mode 100644 tracecmd/trace-agent.c
tracecmd/Makefile +5 -1
index 3a11024..865b1c6 100644
@@ -33,13 +33,17 @@ TRACE_CMD_OBJS += trace-output.o
 TRACE_CMD_OBJS += trace-usage.o
 TRACE_CMD_OBJS += trace-msg.o
 
+ifeq ($(VSOCK_DEFINED), 1)
+TRACE_CMD_OBJS += trace-agent.o
+endif
+
 ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o)
 
 all_objs := $(sort $(ALL_OBJS))
 all_deps := $(all_objs:$(bdir)/%.o=$(bdir)/.%.d)
 
 CONFIG_INCLUDES =
-CONFIG_LIBS	=
+CONFIG_LIBS	= -lrt
 CONFIG_FLAGS	=
 
 all: $(TARGETS)
tracecmd/include/trace-local.h +16 -0
index f43c9d8..95cf512 100644
@@ -12,6 +12,8 @@
 #include "trace-cmd.h"
 #include "event-utils.h"
 
+#define TRACE_AGENT_DEFAULT_PORT	823
+
 extern int debug;
 extern int quiet;
 
@@ -64,6 +66,8 @@ void trace_split(int argc, char **argv);
 
 void trace_listen(int argc, char **argv);
 
+void trace_agent(int argc, char **argv);
+
 void trace_restore(int argc, char **argv);
 
 void trace_clear(int argc, char **argv);
@@ -88,6 +92,10 @@ void trace_list(int argc, char **argv);
 
 void trace_usage(int argc, char **argv);
 
+int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
+		       int cpus, int *fds,
+		       int argc, char **argv);
+
 struct hook_list;
 
 void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
@@ -176,6 +184,7 @@ struct buffer_instance {
 	struct func_list	*notrace_funcs;
 
 	const char		*clock;
+	unsigned int		*client_ports;
 
 	struct trace_seq	*s_save;
 	struct trace_seq	*s_print;
@@ -190,6 +199,13 @@ struct buffer_instance {
 	int			tracing_on_fd;
 	int			buffer_size;
 	int			cpu_count;
+
+	int			argc;
+	char			**argv;
+
+	int			cid;
+	int			port;
+	int			*fds;
 };
 
 extern struct buffer_instance top_instance;
tracecmd/trace-agent.c new file +225 -0

225 lines not on this page · raw message

tracecmd/trace-cmd.c +3 -0
index 797b303..3ae5e2e 100644
@@ -83,6 +83,9 @@ struct command commands[] = {
 	{"hist", trace_hist},
 	{"mem", trace_mem},
 	{"listen", trace_listen},
+#ifdef VSOCK
+	{"agent", trace_agent},
+#endif
 	{"split", trace_split},
 	{"restore", trace_restore},
 	{"stack", trace_stack},
tracecmd/trace-record.c +553 -43

848 lines not on this page · raw message

tracecmd/trace-usage.c +12 -1
index 9ea1906..e845f50 100644
@@ -231,11 +231,22 @@ static struct usage_help usage_help[] = {
 		"listen on a network socket for trace clients",
 		" %s listen -p port[-D][-o file][-d dir][-l logfile]\n"
 		"          Creates a socket to listen for clients.\n"
-		"          -D create it in daemon mode.\n"
+		"          -p port number to listen on.\n"
+		"          -D run in daemon mode.\n"
 		"          -o file name to use for clients.\n"
 		"          -d directory to store client files.\n"
 		"          -l logfile to write messages to.\n"
 	},
+#ifdef VSOCK
+	{
+		"agent",
+		"listen on a vsocket for trace clients",
+		" %s agent -p port[-D]\n"
+		"          Creates a vsocket to listen for clients.\n"
+		"          -p port number to listen on.\n"
+		"          -D run in daemon mode.\n"
+	},
+#endif
 	{
 		"list",
 		"list the available events, plugins or options",
Signature · 1 line
2.19.1
#9 lore raw

RFC PATCH v7 9/13 trace-cmd: Add `trace-cmd setup-guest` command

In reply to Slavomir Kaslev

Add trace-cmd setup-guest command that creates the necessary FIFOs for tracing a guest over FIFOs instead of vsockets.

---
 tracecmd/Makefile              |   1 +
 tracecmd/include/trace-local.h |   6 ++
 tracecmd/trace-cmd.c           |   1 +
 tracecmd/trace-setup-guest.c   | 176 +++++++++++++++++++++++++++++++++
 tracecmd/trace-usage.c         |   8 ++
 5 files changed, 192 insertions(+)
 create mode 100644 tracecmd/trace-setup-guest.c
tracecmd/Makefile +1 -0
index 865b1c6..d3e3080 100644
@@ -35,6 +35,7 @@ TRACE_CMD_OBJS += trace-msg.o
 
 ifeq ($(VSOCK_DEFINED), 1)
 TRACE_CMD_OBJS += trace-agent.o
+TRACE_CMD_OBJS += trace-setup-guest.o
 endif
 
 ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o)
tracecmd/include/trace-local.h +6 -0
index 95cf512..d32ba88 100644
@@ -14,6 +14,10 @@
 
 #define TRACE_AGENT_DEFAULT_PORT	823
 
+#define GUEST_PIPE_NAME		"trace-pipe-cpu"
+#define GUEST_DIR_FMT		"/var/lib/trace-cmd/virt/%s"
+#define GUEST_FIFO_FMT		GUEST_DIR_FMT "/" GUEST_PIPE_NAME "%d"
+
 extern int debug;
 extern int quiet;
 
@@ -68,6 +72,8 @@ void trace_listen(int argc, char **argv);
 
 void trace_agent(int argc, char **argv);
 
+void trace_setup_guest(int argc, char **argv);
+
 void trace_restore(int argc, char **argv);
 
 void trace_clear(int argc, char **argv);
tracecmd/trace-cmd.c +1 -0
index 3ae5e2e..4da82b4 100644
@@ -85,6 +85,7 @@ struct command commands[] = {
 	{"listen", trace_listen},
 #ifdef VSOCK
 	{"agent", trace_agent},
+	{"setup-guest", trace_setup_guest},
 #endif
 	{"split", trace_split},
 	{"restore", trace_restore},
tracecmd/trace-setup-guest.c new file +176 -0

176 lines not on this page · raw message

tracecmd/trace-usage.c +8 -0
index e845f50..7a8002f 100644
@@ -246,6 +246,14 @@ static struct usage_help usage_help[] = {
 		"          -p port number to listen on.\n"
 		"          -D run in daemon mode.\n"
 	},
+	{
+		"setup-guest",
+		"create FIFOs for tracing guest VMs",
+		" %s setup-guest -c cpus[-p perm][-g group] guest\n"
+		"          -c number of guest virtual CPUs\n"
+		"          -p FIFOs permissions (default: 0660)\n"
+		"          -g FIFOs group owner\n"
+	},
 #endif
 	{
 		"list",
Signature · 1 line
2.19.1
#10 lore raw

RFC PATCH v7 10/13 trace-cmd: Try to autodetect number of guest CPUs in setup-guest if not specified

In reply to Slavomir Kaslev

When no number of guest CPUs is provided with the -c flag to trace-cmd setup-guest, try to autodetect it using virsh for libvirt managed guests.

---
 tracecmd/trace-setup-guest.c | 21 +++++++++++++++++++++
 tracecmd/trace-usage.c       |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)
tracecmd/trace-setup-guest.c +21 -0
index ded510e..789dc65 100644
@@ -91,6 +91,24 @@ static int make_guest_fifos(const char *guest, int nr_cpus, mode_t mode)
 	return ret;
 }
 
+static int get_guest_cpu_count(const char *guest)
+{
+	const char *cmd_fmt = "virsh vcpucount --maximum '%s' 2>/dev/null";
+	int nr_cpus = -1;
+	char cmd[1024];
+	FILE *f;
+
+	snprintf(cmd, sizeof(cmd), cmd_fmt, guest);
+	f = popen(cmd, "r");
+	if (!f)
+		return -errno;
+
+	fscanf(f, "%d", &nr_cpus);
+	pclose(f);
+
+	return nr_cpus;
+}
+
 static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t gid)
 {
 	gid_t save_egid;
@@ -169,6 +187,9 @@ void trace_setup_guest(int argc, char **argv)
 
 	guest = argv[optind+1];
 
+	if (nr_cpus <= 0)
+		nr_cpus = get_guest_cpu_count(guest);
+
 	if (nr_cpus <= 0)
 		pdie("invalid number of cpus for guest %s", guest);
 
tracecmd/trace-usage.c +1 -1
index 7a8002f..9a13d93 100644
@@ -249,7 +249,7 @@ static struct usage_help usage_help[] = {
 	{
 		"setup-guest",
 		"create FIFOs for tracing guest VMs",
-		" %s setup-guest -c cpus[-p perm][-g group] guest\n"
+		" %s setup-guest [-c cpus][-p perm][-g group] guest\n"
 		"          -c number of guest virtual CPUs\n"
 		"          -p FIFOs permissions (default: 0660)\n"
 		"          -g FIFOs group owner\n"
Signature · 1 line
2.19.1
#11 lore raw

RFC PATCH v7 11/13 trace-cmd: Add setup-guest flag for attaching FIFOs to the guest VM config

In reply to Slavomir Kaslev

This patch adds a flag for attaching the newly created FIFOs for guest tracing as virtio serial devices to libvirt managed guests.

---
 tracecmd/trace-setup-guest.c | 54 ++++++++++++++++++++++++++++++++++--
 tracecmd/trace-usage.c       |  3 +-
 2 files changed, 53 insertions(+), 4 deletions(-)
tracecmd/trace-setup-guest.c +51 -3
index 789dc65..043bca2 100644
@@ -109,7 +109,45 @@ static int get_guest_cpu_count(const char *guest)
 	return nr_cpus;
 }
 
-static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t gid)
+static int attach_guest_fifos(const char *guest, int nr_cpus)
+{
+	const char *cmd_fmt =
+		"virsh attach-device --config '%s' '%s' >/dev/null 2>/dev/null";
+	const char *xml_fmt =
+		"<channel type='pipe'>\n"
+		"  <source path='%s'/>\n"
+		"  <target type='virtio' name='%s%d'/>\n"
+		"</channel>";
+	char tmp_path[PATH_MAX], path[PATH_MAX];
+	char cmd[PATH_MAX], xml[PATH_MAX];
+	int i, fd, ret = 0;
+
+	strcpy(tmp_path, "/tmp/pipexmlXXXXXX");
+	fd = mkstemp(tmp_path);
+	if (fd < 0)
+		return fd;
+
+	for (i = 0; i < nr_cpus; i++) {
+		snprintf(path, sizeof(path), GUEST_FIFO_FMT, guest, i);
+		snprintf(xml, sizeof(xml), xml_fmt, path, GUEST_PIPE_NAME, i);
+		pwrite(fd, xml, strlen(xml), 0);
+
+		snprintf(cmd, sizeof(cmd), cmd_fmt, guest, tmp_path);
+		errno = 0;
+		if (system(cmd) != 0) {
+			ret = -errno;
+			break;
+		}
+	}
+
+	close(fd);
+	unlink(tmp_path);
+
+	return ret;
+}
+
+static void do_setup_guest(const char *guest, int nr_cpus,
+			   mode_t mode, gid_t gid, bool attach)
 {
 	gid_t save_egid;
 	int ret;
@@ -129,6 +167,12 @@ static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t gi
 	if (ret < 0)
 		pdie("failed to create FIFOs for %s", guest);
 
+	if (attach) {
+		ret = attach_guest_fifos(guest, nr_cpus);
+		if (ret < 0)
+			pdie("failed to attach FIFOs to %s", guest);
+	}
+
 	if (gid != -1) {
 		ret = setegid(save_egid);
 		if (ret < 0)
@@ -138,6 +182,7 @@ static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t gi
 
 void trace_setup_guest(int argc, char **argv)
 {
+	bool attach = false;
 	struct group *group;
 	mode_t mode = 0660;
 	int nr_cpus = -1;
@@ -157,7 +202,7 @@ void trace_setup_guest(int argc, char **argv)
 			{NULL, 0, NULL, 0}
 		};
 
-		c = getopt_long(argc-1, argv+1, "+hc:p:g:",
+		c = getopt_long(argc-1, argv+1, "+hc:p:g:a",
 				long_options, &option_index);
 		if (c == -1)
 			break;
@@ -177,6 +222,9 @@ void trace_setup_guest(int argc, char **argv)
 				pdie("group %s does not exist", optarg);
 			gid = group->gr_gid;
 			break;
+		case 'a':
+			attach = true;
+			break;
 		default:
 			usage(argv);
 		}
@@ -193,5 +241,5 @@ void trace_setup_guest(int argc, char **argv)
 	if (nr_cpus <= 0)
 		pdie("invalid number of cpus for guest %s", guest);
 
-	do_setup_guest(guest, nr_cpus, mode, gid);
+	do_setup_guest(guest, nr_cpus, mode, gid, attach);
 }
tracecmd/trace-usage.c +2 -1
index 9a13d93..bcfd3dc 100644
@@ -249,10 +249,11 @@ static struct usage_help usage_help[] = {
 	{
 		"setup-guest",
 		"create FIFOs for tracing guest VMs",
-		" %s setup-guest [-c cpus][-p perm][-g group] guest\n"
+		" %s setup-guest [-c cpus][-p perm][-g group][-a] guest\n"
 		"          -c number of guest virtual CPUs\n"
 		"          -p FIFOs permissions (default: 0660)\n"
 		"          -g FIFOs group owner\n"
+		"          -a Attach FIFOs to guest VM config\n"
 	},
 #endif
 	{
Signature · 1 line
2.19.1
#12 lore raw

RFC PATCH v7 12/13 trace-cmd: Add splice() recording from FIFO without additional pipe buffer

In reply to Slavomir Kaslev

When trace-cmd record is reading tracing data over FIFO we can do a direct splice from the FIFO to the output file descriptor instead of doing two through an additional pipe buffer. This patch implements specialized tracecmd_recorder data transfer version for this case.

---
 include/trace-cmd/trace-cmd.h  |  3 +-
 lib/trace-cmd/trace-recorder.c | 70 ++++++++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 12 deletions(-)
include/trace-cmd/trace-cmd.h +2 -1
index 6a21e66..52962e9 100644
@@ -270,8 +270,9 @@ struct tracecmd_output *tracecmd_get_output_handle_fd(int fd);
 
 enum {
 	TRACECMD_RECORD_NOSPLICE	= (1 << 0),	/* Use read instead of splice */
-	TRACECMD_RECORD_SNAPSHOT	= (1 << 1),	/* extract from snapshot */
+	TRACECMD_RECORD_SNAPSHOT	= (1 << 1),	/* Extract from snapshot */
 	TRACECMD_RECORD_BLOCK		= (1 << 2),	/* Block on splice write */
+	TRACECMD_RECORD_NOBRASS		= (1 << 3),	/* Splice directly without a brass pipe */
 };
 
 void tracecmd_free_recorder(struct tracecmd_recorder *recorder);
lib/trace-cmd/trace-recorder.c +59 -11

137 lines not on this page · raw message

Signature · 1 line
2.19.1
#13 lore raw

RFC PATCH v7 13/13 trace-cmd: Add VM tracing over FIFOs transport

In reply to Slavomir Kaslev

FIFOs offer ~3x the throughput of vsockets. This patch adds support for using FIFOs to stream tracing data back to the host when tracing VMs.

---
 include/trace-cmd/trace-cmd.h  |  8 ++--
 tracecmd/include/trace-local.h |  4 +-
 tracecmd/trace-agent.c         | 40 ++++++++++++++--
 tracecmd/trace-msg.c           | 26 ++++++----
 tracecmd/trace-record.c        | 87 ++++++++++++++++++++++++++++++----
 5 files changed, 137 insertions(+), 28 deletions(-)
include/trace-cmd/trace-cmd.h +4 -4
index 52962e9..27ba89d 100644
@@ -333,16 +333,16 @@ bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);
 void tracecmd_msg_set_done(struct tracecmd_msg_handle *msg_handle);
 
 int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
-				int argc, char **argv);
+				int argc, char **argv, bool use_fifos);
 int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
-				int *argc, char ***argv);
+				int *argc, char ***argv, bool *use_fifos);
 
 int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
 				 int nr_cpus, int page_size,
-				 unsigned int *ports);
+				 unsigned int *ports, bool use_fifos);
 int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
 				 int *nr_cpus, int *page_size,
-				 unsigned int **ports);
+				 unsigned int **ports, bool *use_fifos);
 
 /* --- Plugin handling --- */
 extern struct tep_plugin_option trace_ftrace_options[];
tracecmd/include/trace-local.h +3 -1
index d32ba88..d7bdb1f 100644
@@ -17,6 +17,7 @@
 #define GUEST_PIPE_NAME		"trace-pipe-cpu"
 #define GUEST_DIR_FMT		"/var/lib/trace-cmd/virt/%s"
 #define GUEST_FIFO_FMT		GUEST_DIR_FMT "/" GUEST_PIPE_NAME "%d"
+#define VIRTIO_FIFO_FMT		"/dev/virtio-ports/" GUEST_PIPE_NAME "%d"
 
 extern int debug;
 extern int quiet;
@@ -100,7 +101,7 @@ void trace_usage(int argc, char **argv);
 
 int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 		       int cpus, int *fds,
-		       int argc, char **argv);
+		       int argc, char **argv, bool use_fifos);
 
 struct hook_list;
 
@@ -212,6 +213,7 @@ struct buffer_instance {
 	int			cid;
 	int			port;
 	int			*fds;
+	bool			use_fifos;
 };
 
 extern struct buffer_instance top_instance;
tracecmd/trace-agent.c +36 -4
index 0c0873b..7389f72 100644
@@ -83,12 +83,39 @@ static void make_vsocks(int nr, int *fds, unsigned int *ports)
 	}
 }
 
+static int open_agent_fifos(int nr_cpus, int *fds)
+{
+	char path[PATH_MAX];
+	int i, fd, ret = 0;
+
+	for (i = 0; i < nr_cpus; i++) {
+		snprintf(path, sizeof(path), VIRTIO_FIFO_FMT, i);
+		fd = open(path, O_WRONLY);
+		if (fd < 0) {
+			ret = -errno;
+			break;
+		}
+
+		fds[i] = fd;
+	}
+
+	if (!ret)
+		return ret;
+
+	/* We failed to open all FIFOs so clean up and return error */
+	while (--i >= 0)
+		close(fds[i]);
+
+	return ret;
+}
+
 static void agent_handle(int sd, int nr_cpus, int page_size)
 {
 	struct tracecmd_msg_handle *msg_handle;
 	unsigned int *ports;
 	char **argv = NULL;
 	int argc = 0;
+	bool use_fifos;
 	int *fds;
 	int ret;
 
@@ -101,17 +128,22 @@ static void agent_handle(int sd, int nr_cpus, int page_size)
 	if (!msg_handle)
 		die("Failed to allocate message handle");
 
-	ret = tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv);
+	ret = tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv, &use_fifos);
 	if (ret < 0)
 		die("Failed to receive trace request");
 
-	make_vsocks(nr_cpus, fds, ports);
+	if (use_fifos && open_agent_fifos(nr_cpus, fds))
+		use_fifos = false;
+
+	if (!use_fifos)
+		make_vsocks(nr_cpus, fds, ports);
 
-	ret = tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports);
+	ret = tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size,
+					   ports, use_fifos);
 	if (ret < 0)
 		die("Failed to send trace response");
 
-	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
+	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv, use_fifos);
 
 	free(argv[0]);
 	free(argv);
tracecmd/trace-msg.c +17 -9
index 8ce1f98..065a01e 100644
@@ -159,6 +159,10 @@ static int msg_write(int fd, struct tracecmd_msg *msg)
 	return __do_write_check(fd, msg->buf, data_size);
 }
 
+enum msg_trace_flags {
+	MSG_TRACE_USE_FIFOS = 1 << 0,
+};
+
 enum msg_opt_command {
 	MSGOPT_USETCP = 1,
 };
@@ -740,7 +744,7 @@ error:
 	return ret;
 }
 
-static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv)
+static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv, bool use_fifos)
 {
 	size_t args_size = 0;
 	char *p;
@@ -750,6 +754,7 @@ static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv)
 		args_size += strlen(argv[i]) + 1;
 
 	msg->hdr.size = htonl(ntohl(msg->hdr.size) + args_size);
+	msg->trace_req.flags = use_fifos ? htonl(MSG_TRACE_USE_FIFOS) : htonl(0);
 	msg->trace_req.argc = htonl(argc);
 	msg->buf = calloc(args_size, 1);
 	if (!msg->buf)
@@ -763,13 +768,13 @@ static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv)
 }
 
 int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
-				int argc, char **argv)
+				int argc, char **argv, bool use_fifos)
 {
 	struct tracecmd_msg msg;
 	int ret;
 
 	tracecmd_msg_init(MSG_TRACE_REQ, &msg);
-	ret = make_trace_req(&msg, argc, argv);
+	ret = make_trace_req(&msg, argc, argv, use_fifos);
 	if (ret < 0)
 		return ret;
 
@@ -782,7 +787,7 @@ int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
   *     free(argv);
   */
 int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
-				int *argc, char ***argv)
+				int *argc, char ***argv, bool *use_fifos)
 {
 	struct tracecmd_msg msg;
 	char *p, *buf_end, **args;
@@ -834,6 +839,7 @@ int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
 
 	*argc = nr_args;
 	*argv = args;
+	*use_fifos = ntohl(msg.trace_req.flags) & MSG_TRACE_USE_FIFOS;
 
 	/*
 	 * On success we're passing msg.buf to the caller through argv[0] so we
@@ -853,13 +859,14 @@ out:
 	return ret;
 }
 
-static int make_trace_resp(struct tracecmd_msg *msg,
-			   int page_size, int nr_cpus, unsigned int *ports)
+static int make_trace_resp(struct tracecmd_msg *msg, int page_size, int nr_cpus,
+			   unsigned int *ports, bool use_fifos)
 {
 	int ports_size = nr_cpus * sizeof(*msg->port_array);
 	int i;
 
 	msg->hdr.size = htonl(ntohl(msg->hdr.size) + ports_size);
+	msg->trace_resp.flags = use_fifos ? htonl(MSG_TRACE_USE_FIFOS) : htonl(0);
 	msg->trace_resp.cpus = htonl(nr_cpus);
 	msg->trace_resp.page_size = htonl(page_size);
 
@@ -875,13 +882,13 @@ static int make_trace_resp(struct tracecmd_msg *msg,
 
 int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
 				 int nr_cpus, int page_size,
-				 unsigned int *ports)
+				 unsigned int *ports, bool use_fifos)
 {
 	struct tracecmd_msg msg;
 	int ret;
 
 	tracecmd_msg_init(MSG_TRACE_RESP, &msg);
-	ret = make_trace_resp(&msg, page_size, nr_cpus, ports);
+	ret = make_trace_resp(&msg, page_size, nr_cpus, ports, use_fifos);
 	if (ret < 0)
 		return ret;
 
@@ -890,7 +897,7 @@ int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
 
 int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
 				 int *nr_cpus, int *page_size,
-				 unsigned int **ports)
+				 unsigned int **ports, bool *use_fifos)
 {
 	struct tracecmd_msg msg;
 	ssize_t buf_len;
@@ -912,6 +919,7 @@ int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
 		goto out;
 	}
 
+	*use_fifos = ntohl(msg.trace_resp.flags) & MSG_TRACE_USE_FIFOS;
 	*nr_cpus = ntohl(msg.trace_resp.cpus);
 	*page_size = ntohl(msg.trace_resp.page_size);
 	*ports = calloc(*nr_cpus, sizeof(**ports));
tracecmd/trace-record.c +77 -10

162 lines not on this page · raw message

Signature · 1 line
2.19.1