Add VM guest kernel tracing using vsock sockets for transport
v2 cover Add VM guest kernel tracing using vsock sockets for transport
This patchset adds support for tracing guest kernels to trace-cmd.
Changes in v2:
- rebased on top of protocol V3
- fixed system clock timestamps with the --date flag
Slavomir Kaslev (6):
trace-cmd: Fix indentation
trace-cmd: Fix warnings reported by gcc 8.2
trace-cmd: Add tracecmd_create_recorder_virt function
trace-cmd: Simplify and fix memory leaks in tracecmd_msg_collect_data
trace-cmd: Add TRACE_REQ and TRACE_RESP messages
trace-cmd: Add VM kernel tracing using vsock socket for transport
include/trace-cmd/trace-cmd.h | 14 + lib/trace-cmd/trace-input.c | 2 +- lib/trace-cmd/trace-recorder.c | 53 ++- tracecmd/Makefile | 1 + tracecmd/include/trace-local.h | 19 + tracecmd/trace-agent.c | 228 ++++++++++ tracecmd/trace-cmd.c | 1 + tracecmd/trace-msg.c | 213 +++++++++- tracecmd/trace-read.c | 8 +- tracecmd/trace-record.c | 733 ++++++++++++++++++++++++++++----- tracecmd/trace-usage.c | 11 +- 11 files changed, 1162 insertions(+), 121 deletions(-) create mode 100644 tracecmd/trace-agent.c
Signature · 1 line
2.19.1
PATCH v2 1/6 trace-cmd: Fix indentation
No functional changes in this patch.
--- lib/trace-cmd/trace-input.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/trace-cmd/trace-input.c +1 -1
@@ -2603,7 +2603,7 @@ struct tracecmd_input *tracecmd_alloc_fd(int fd) {struct tracecmd_input *handle;char test[] = {23 ,8 ,68 }; -unsigned int page_size; +unsigned int page_size;char *version;char buf[BUFSIZ ];
Signature · 1 line
2.19.1
PATCH v2 2/6 trace-cmd: Fix warnings reported by gcc 8.2
In reply to Slavomir KaslevCompiling trace-cmd with gcc 8.2 reports:
trace-read.c: In function ‘append_pid_filter’:
trace-read.c:414:3: warning: passing argument 1 to restrict-qualified parameter aliases with argument 3 [-Wrestrict]
sprintf(filter, "%s||" FILTER_FMT, filter, pid, pid, pid);
^~~~~~~
since snprintf declares its first argument as __restrict but buf is also passed as argument #5.
--- tracecmd/trace-read.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
tracecmd/trace-read.c +4 -4
@@ -386,7 +386,7 @@ static void add_pid_filter(const char *arg)static char *append_pid_filter (char *curr_filter,char *pid) {char *filter; -int len; +int len, curr_len;#define FILTER_FMT "(common_pid==" __STR ")||(pid==" __STR ")||(next_pid==" __STR ")"
@@ -405,13 +405,13 @@ static char *append_pid_filter(char *curr_filter, char *pid)die ("Failed to allocate for filter %s" , curr_filter);sprintf (filter,".*:" FILTER_FMT , pid, pid, pid); }else { - - len +=strlen (curr_filter); + curr_len =strlen (curr_filter); + len += curr_len; filter =realloc (curr_filter, len);if (!filter)die ("realloc" ); -sprintf (filter,"%s||" FILTER_FMT , filter, pid, pid, pid); +sprintf (filter + curr_len,"||" FILTER_FMT , pid, pid, pid); }return filter;
Signature · 1 line
2.19.1
Re: [PATCH v2 2/6] trace-cmd: Fix warnings reported by gcc 8.2
Quoted 11 lines · On Tue, 8 Jan 2019 17:00:11 +0200 Slavomir Kaslev <[email protected]> wrote:
Compiling trace-cmd with gcc 8.2 reports:
trace-read.c: In function ‘append_pid_filter’:
trace-read.c:414:3: warning: passing argument 1 to restrict-qualified parameter aliases with argument 3 [-Wrestrict]
sprintf(filter, "%s||" FILTER_FMT, filter, pid, pid, pid);
^~~~~~~since
snprintfdeclares its first argument as__restrictbutbufis also passed as argument #5.Signed-off-by Slavomir Kaslev <[email protected]>
FYI, I took this patch but changed the subject and description to this:
From: Slavomir Kaslev <[email protected]>
Subject: [PATCH] trace-cmd: Do not use target string pointer as parameter to
sprintf()
Compiling trace-cmd with gcc 8.2 reports:
trace-read.c: In function ‘append_pid_filter’:
trace-read.c:414:3: warning: passing argument 1 to restrict-qualified parameter aliases with argument 3 [-Wrestrict]
sprintf(filter, "%s||" FILTER_FMT, filter, pid, pid, pid);
^~~~~~~
since snprintf declares its first argument as __restrict but buf is also passed as argument #5.
The code just concatenates to the filter string. Use the original string length as the position to write to filter.
-- Steve
PATCH v2 3/6 trace-cmd: Add tracecmd_create_recorder_virt function
In reply to Slavomir KaslevAdd tracecmd_create_recorder_virt function that will be used for tracing VM guests.
--- include/trace-cmd/trace-cmd.h | 1 + lib/trace-cmd/trace-recorder.c | 53 ++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-)
include/trace-cmd/trace-cmd.h +1 -0
@@ -277,6 +277,7 @@ enum {void tracecmd_free_recorder (struct tracecmd_recorder *recorder);struct tracecmd_recorder *tracecmd_create_recorder (const char *file,int cpu,unsigned flags);struct tracecmd_recorder *tracecmd_create_recorder_fd (int fd,int cpu,unsigned flags); +struct tracecmd_recorder *tracecmd_create_recorder_virt (const char *file,int cpu,int trace_fd);struct tracecmd_recorder *tracecmd_create_recorder_maxkb (const char *file,int cpu,unsigned flags,int maxkb);struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd (int fd,int cpu,unsigned flags,const char *buffer);struct tracecmd_recorder *tracecmd_create_buffer_recorder (const char *file,int cpu,unsigned flags,const char *buffer);
lib/trace-cmd/trace-recorder.c +38 -15
81 lines not on this page · raw message
Signature · 1 line
2.19.1
PATCH v2 4/6 trace-cmd: Simplify and fix memory leaks in tracecmd_msg_collect_data
In reply to Slavomir KaslevThis patch addresses leaking tracecmd_msg messages in tracecmd_msg_collect_data.
It also splits out tracecmd_msg_read_data from tracecmd_msg_collect_data which only reads incomming data and doesn't wait for a final MSG_CLOSE message and is used by the VM tracing agent.
--- include/trace-cmd/trace-cmd.h | 1 + tracecmd/trace-msg.c | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-)
include/trace-cmd/trace-cmd.h +1 -0
@@ -332,6 +332,7 @@ void tracecmd_msg_send_close_msg(struct tracecmd_msg_handle *msg_handle);int tracecmd_msg_initial_setting (struct tracecmd_msg_handle *msg_handle);int tracecmd_msg_send_port_array (struct tracecmd_msg_handle *msg_handle,int *ports); +int tracecmd_msg_read_data (struct tracecmd_msg_handle *msg_handle,int ofd);int tracecmd_msg_collect_data (struct tracecmd_msg_handle *msg_handle,int ofd);bool tracecmd_msg_done (struct tracecmd_msg_handle *msg_handle);void tracecmd_msg_set_done (struct tracecmd_msg_handle *msg_handle);
tracecmd/trace-msg.c +26 -2
@@ -626,7 +626,7 @@ int tracecmd_msg_finish_sending_data(struct tracecmd_msg_handle *msg_handle)return 0 ; } -int tracecmd_msg_collect_data (struct tracecmd_msg_handle *msg_handle,int ofd) +int tracecmd_msg_read_data (struct tracecmd_msg_handle *msg_handle,int ofd) {struct tracecmd_msg msg;int t, n, cmd;
@@ -659,13 +659,34 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd)if (errno ==EINTR )continue ;warning ("writing to file" ); -return -errno; + ret = -errno; + goto error; } t -= s; s = n - t; } + +msg_free (&msg); } +return 0 ; + +error: +error_operation_for_server (&msg); +msg_free (&msg); +return ret; +} + +int tracecmd_msg_collect_data (struct tracecmd_msg_handle *msg_handle,int ofd) +{ +struct tracecmd_msg msg; +u32 cmd; +int ret; + + ret =tracecmd_msg_read_data (msg_handle, ofd); +if (ret) + goto error; +/* check the finish message of the client */ while (!tracecmd_msg_done (msg_handle)) { ret =tracecmd_msg_recv (msg_handle->fd, &msg);
@@ -683,11 +704,14 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd) ret = -EINVAL ; goto error; } + +msg_free (&msg); }return 0 ; error:error_operation_for_server (&msg); +msg_free (&msg);return ret; }
Signature · 1 line
2.19.1
Re: [PATCH v2 4/6] trace-cmd: Simplify and fix memory leaks in tracecmd_msg_collect_data
Quoted 6 lines · On Tue, 8 Jan 2019 17:00:13 +0200 Slavomir Kaslev <[email protected]> wrote:
This patch addresses leaking
tracecmd_msgmessages intracecmd_msg_collect_data.It also splits out
tracecmd_msg_read_datafromtracecmd_msg_collect_datawhich only reads incomming data and doesn't wait for a finalMSG_CLOSEmessage and is used by the VM tracing agent.
I re-ordered the patch series and made this patch 3/6 and made the original 3/6 the 4/6. The first 3 patches (including this one) are being pulled into the main branch.
I'll keep the virt-server branch separate still.
-- Steve
PATCH v2 5/6 trace-cmd: Add TRACE_REQ and TRACE_RESP messages
In reply to Slavomir KaslevAdd TRACE_REQ and TRACE_RESP messages which are used for initiating guest VM tracing.
--- include/trace-cmd/trace-cmd.h | 12 +++ tracecmd/trace-msg.c | 185 +++++++++++++++++++++++++++++++++- 2 files changed, 196 insertions(+), 1 deletion(-)
include/trace-cmd/trace-cmd.h +12 -0
@@ -337,6 +337,18 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);bool tracecmd_msg_done (struct tracecmd_msg_handle *msg_handle);void tracecmd_msg_set_done (struct tracecmd_msg_handle *msg_handle); +int tracecmd_msg_send_trace_req (struct tracecmd_msg_handle *msg_handle, +int argc,char **argv); +int tracecmd_msg_recv_trace_req (struct tracecmd_msg_handle *msg_handle, +int *argc,char ***argv); + +int tracecmd_msg_send_trace_resp (struct tracecmd_msg_handle *msg_handle, +int nr_cpus,int page_size,int *ports); +int tracecmd_msg_recv_trace_resp (struct tracecmd_msg_handle *msg_handle, +int *nr_cpus,int *page_size,int **ports); + +int tracecmd_msg_wait_close (struct tracecmd_msg_handle *msg_handle); +/* --- Plugin handling --- */ extern struct tep_plugin_option trace_ftrace_options[];
tracecmd/trace-msg.c +184 -1
212 lines not on this page · raw message
Signature · 1 line
2.19.1
Re: [PATCH v2 5/6] trace-cmd: Add TRACE_REQ and TRACE_RESP messages
Quoted 21 lines · On Tue, 8 Jan 2019 17:00:14 +0200 Slavomir Kaslev <[email protected]> wrote:
Add
TRACE_REQandTRACE_RESPmessages which are used for initiating guest VM tracing.+ /*
tracecmd/trace-msg.c +17
-0+* NOTE: On success, the returned `argv` should be freed with: +* free(argv[0]); +* free(argv); +*/ +int tracecmd_msg_recv_trace_req (struct tracecmd_msg_handle *msg_handle, +int *argc,char ***argv) +{ +struct tracecmd_msg msg; +char *p, *buf_end; +size_t buf_len; +int i, ret; + + ret =tracecmd_msg_recv (msg_handle->fd, &msg); +if (ret <0 ) +return ret; + +if (ntohl (msg.hdr.cmd) !=MSG_TRACE_REQ )
tracecmd/trace-msg.c error: · line 797
Quoted 10 lines
tracecmd/trace-msg.c +10
-0+ goto out; + +if (ntohl (msg.trace_req.argc) <0 ) + goto out; + + buf_len =ntohl (msg.hdr.size) -MSG_HDR_LEN -ntohl (msg.hdr.cmd_size); + buf_end = (char *)msg.buf + buf_len; + p = msg.buf; + *argc =ntohl (msg.trace_req.argc); + *argv =calloc (*argc,sizeof (**argv));
Need to test for failed calloc().
tracecmd/trace-msg.c error: · line 810
Quoted 13 lines
tracecmd/trace-msg.c +13
-0+for (i =0 ; i < *argc; i++) { +if (p >= buf_end) { + ret =-1 ; +free (*argv); + goto out; + } + + (*argv)[i] = p; + p =strchr (p,'\0' ); + p++; + } + + ret =0 ;
Should add a comment here saying that we are passing msg.buf as argv[0], and we don't want to have msg_free() free it.
tracecmd/trace-msg.c error: · line 813
Quoted 3 lines
tracecmd/trace-msg.c +3
-0+ msg.buf =NULL ; + +out:
Need to free argv and argv[0] as the description only says that the caller has to free it on success. Make sure to set those pointers to zero (NULL) afterward.
Quoted 54 lines
tracecmd/trace-msg.c +54
-0+msg_free (&msg); +return ret; +} + +static int make_trace_resp (struct tracecmd_msg *msg, +int page_size,int nr_cpus,int *ports) +{ +int ports_size = nr_cpus *sizeof (*msg->port_array); +int i; + + msg->hdr.size =htonl (ntohl (msg->hdr.size) + ports_size); + msg->trace_resp.cpus =htonl (nr_cpus); + msg->trace_resp.page_size =htonl (page_size); + + msg->port_array =malloc (ports_size); +if (!msg->port_array) +return -ENOMEM ; + +for (i =0 ; i < nr_cpus; i++) + msg->port_array[i] =htonl (ports[i]); + +return 0 ; +} + +int tracecmd_msg_send_trace_resp (struct tracecmd_msg_handle *msg_handle, +int nr_cpus,int page_size,int *ports) +{ +struct tracecmd_msg msg; +int ret; + +tracecmd_msg_init (MSG_TRACE_RESP , &msg); + ret =make_trace_resp (&msg, page_size, nr_cpus, ports); +if (ret <0 ) +return ret; + +return tracecmd_msg_send (msg_handle->fd, &msg); +} + +int tracecmd_msg_recv_trace_resp (struct tracecmd_msg_handle *msg_handle, +int *nr_cpus,int *page_size,int **ports) +{ +struct tracecmd_msg msg; +size_t buf_len; +int i, ret; + + ret =tracecmd_msg_recv (msg_handle->fd, &msg); +if (ret <0 ) +return ret; + +if (ntohl (msg.hdr.cmd) !=MSG_TRACE_RESP ) { + ret =-1 ; + goto out; + } +
tracecmd/trace-msg.c error: · line 877
Quoted 10 lines
tracecmd/trace-msg.c +10
-0+ buf_len =ntohl (msg.hdr.size) -MSG_HDR_LEN -ntohl (msg.hdr.cmd_size); +if (buf_len <=0 || + buf_len !=sizeof (*msg.port_array) *ntohl (msg.trace_resp.cpus)) { + ret =-1 ; + goto out; + } + + *nr_cpus =ntohl (msg.trace_resp.cpus); + *page_size =ntohl (msg.trace_resp.page_size); + *ports =calloc (*nr_cpus,sizeof (**ports));
Again, this needs to test the result of calloc().
-- Steve
Quoted 23 lines
tracecmd/trace-msg.c +23
-0+for (i =0 ; i < *nr_cpus; i++) + (*ports)[i] =ntohl (msg.port_array[i]); + + ret =0 ; + +out: +msg_free (&msg); +return ret; +} + +int tracecmd_msg_wait_close (struct tracecmd_msg_handle *msg_handle) +{ +char buf[BUFSIZ ]; +int ret; + +for (;;) { + ret =read (msg_handle->fd, buf,sizeof (buf)); +if (ret <=0 ) +return ret; + } + +return -1 ; +}
PATCH v2 6/6 trace-cmd: Add VM kernel tracing using vsock socket for transport
In reply to Slavomir KaslevTo test, start a VM and assign it a valid (> 2) unused CID:
you@host # qemu-system-x86_64 \
-name guest2 --uuid 7edfdf4a-cb9e-11e8-b38a-173b58342476 \
-m 4096 -boot d -enable-kvm -smp 3 -net nic -net user -hda $HOME/vm/guest2.img \
-device vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=3and start trace-cmd agent on the guest as root:
you@guest2 # trace-cmd agent
Finally, start trace-cmd record on the host (running as root is only necessary if the host will be traced too):
you@host $ trace-cmd record -A guest2 -e irq -e sched
--- tracecmd/Makefile | 1 + tracecmd/include/trace-local.h | 19 + tracecmd/trace-agent.c | 228 ++++++++++ tracecmd/trace-cmd.c | 1 + tracecmd/trace-record.c | 733 ++++++++++++++++++++++++++++----- tracecmd/trace-usage.c | 11 +- 6 files changed, 895 insertions(+), 98 deletions(-) create mode 100644 tracecmd/trace-agent.c
tracecmd/Makefile +1 -0
@@ -17,6 +17,7 @@ TRACE_CMD_OBJS += trace-record.o TRACE_CMD_OBJS += trace-read.o TRACE_CMD_OBJS += trace-split.o TRACE_CMD_OBJS += trace-listen.o +TRACE_CMD_OBJS += trace-agent.o TRACE_CMD_OBJS += trace-stack.o TRACE_CMD_OBJS += trace-hist.o TRACE_CMD_OBJS += trace-mem.o
tracecmd/include/trace-local.h +19 -0
@@ -12,6 +12,8 @@#include "trace-cmd.h" #include "event-utils.h" +#define TRACE_AGENT_DEFAULT_PORT 823 +extern int debug;extern int quiet;
@@ -64,6 +66,8 @@ void trace_split(int argc, char **argv);void trace_listen (int argc,char **argv); +void trace_agent (int argc,char **argv); +void trace_restore (int argc,char **argv);void trace_clear (int argc,char **argv);
@@ -88,6 +92,10 @@ void trace_list(int argc, char **argv);void trace_usage (int argc,char **argv); +int trace_record_agent (struct tracecmd_msg_handle *msg_handle, +int cpus,int *fds, +int argc,char **argv); +struct hook_list ; void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
@@ -149,6 +157,8 @@ char *strstrip(char *str);enum buffer_instance_flags {BUFFER_FL_KEEP =1 <<0 ,BUFFER_FL_PROFILE =1 <<1 , +BUFFER_FL_GUEST =1 <<2 , +BUFFER_FL_AGENT =1 <<3 , };struct func_list {
@@ -174,6 +184,7 @@ struct buffer_instance {struct func_list *notrace_funcs;const char *clock; +int *client_ports;struct trace_seq *s_save;struct trace_seq *s_print;
@@ -188,6 +199,14 @@ struct buffer_instance {int tracing_on_fd;int buffer_size;int cpu_count; + +int argc; +char **argv; +int argv_cap; + +int cid; +int port; +int *fds; };extern struct buffer_instance top_instance;
tracecmd/trace-agent.c new file +228 -0
228 lines not on this page · raw message
tracecmd/trace-cmd.c +1 -0
@@ -83,6 +83,7 @@ struct command commands[] = { {"hist", trace_hist}, {"mem", trace_mem}, {"listen", trace_listen}, + {"agent", trace_agent}, {"split", trace_split}, {"restore", trace_restore}, {"stack", trace_stack},
tracecmd/trace-record.c +636 -97
1,152 lines not on this page · raw message
tracecmd/trace-usage.c +10 -1
@@ -231,11 +231,20 @@ static struct usage_help usage_help[] = {"listen on a network socket for trace clients" ," %s listen -p port[-D][-o file][-d dir][-l logfile]\n" " Creates a socket to listen for clients.\n" -" -D create it in daemon mode.\n" +" -p port number to listen on.\n" +" -D run in daemon mode.\n" " -o file name to use for clients.\n" " -d directory to store client files.\n" " -l logfile to write messages to.\n" }, + { +"agent" , +"listen on a vsock socket for trace clients" , +" %s agent -p port[-D]\n" +" Creates a vsock socket to listen for clients.\n" +" -p port number to listen on.\n" +" -D run in daemon mode.\n" + }, {"list" , "list the available events, plugins or options",
Signature · 1 line
2.19.1
From: Steven Rostedt (VMware) <[email protected]>
BTW, can you add a patch like this too. Add my signed off by along with yours. We need trace-cmd to continue to build with older kernels that do not support vsocks. As I still have some boxes that don't and I use to test on ;-)
Makefile +7 -0
@@ -205,6 +205,13 @@ CFLAGS ?= -g -Wall CPPFLAGS ?= LDFLAGS ?= +VSOCK_DEFINED := $(shell if (echo '\#include <linux/vm_socket.h>' | gcc -E - &> /dev/null) ; then echo 1; else echo 0 ; fi) + +export VSOCK_DEFINED +ifeq ($(VSOCK_DEFINED), 1) +CFLAGS += -DVSOCK +endif + export CFLAGS export INCLUDES
tracecmd/Makefile +5 -2
@@ -17,7 +17,6 @@ TRACE_CMD_OBJS += trace-record.o TRACE_CMD_OBJS += trace-read.o TRACE_CMD_OBJS += trace-split.o TRACE_CMD_OBJS += trace-listen.o -TRACE_CMD_OBJS += trace-agent.o TRACE_CMD_OBJS += trace-stack.o TRACE_CMD_OBJS += trace-hist.o TRACE_CMD_OBJS += trace-mem.o
@@ -34,13 +33,17 @@ TRACE_CMD_OBJS += trace-output.o TRACE_CMD_OBJS += trace-usage.o TRACE_CMD_OBJS += trace-msg.o +ifeq ($(VSOCK_DEFINED), 1) +TRACE_CMD_OBJS += trace-agent.o +endif + ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o) all_objs := $(sort $(ALL_OBJS)) all_deps := $(all_objs:$(bdir)/%.o=$(bdir)/.%.d) CONFIG_INCLUDES = -CONFIG_LIBS = +CONFIG_LIBS = -lrt CONFIG_FLAGS = all: $(TARGETS)
tracecmd/trace-cmd.c +2 -0
@@ -83,7 +83,9 @@ struct command commands[] = { {"hist", trace_hist}, {"mem", trace_mem}, {"listen", trace_listen}, +#ifdef VSOCK {"agent", trace_agent}, +#endif {"split", trace_split}, {"restore", trace_restore}, {"stack", trace_stack},
tracecmd/trace-record.c +11 -0
@@ -33,7 +33,10 @@#include <errno.h> #include <limits.h> #include <libgen.h> + +#ifdef VSOCK #include <linux/vm_sockets.h> +#endif #include "trace-local.h" #include "trace-msg.h"
@@ -2658,6 +2661,7 @@ static int connect_port(const char *host, unsigned port)return sfd; } +#ifdef VSOCK static int open_vsock(unsigned cid, unsigned port) { struct sockaddr_vm addr = {
@@ -2676,6 +2680,13 @@ static int open_vsock(unsigned cid, unsigned port) return sd; } +#else +static inline int open_vsock (unsigned cid,unsigned port) +{ +die ("vsock is not supported" ); +return 0 ; +} +#endif static int do_accept (int sd) {
tracecmd/trace-usage.c +2 -0
@@ -237,6 +237,7 @@ static struct usage_help usage_help[] = { " -d directory to store client files.\n" " -l logfile to write messages to.\n" }, +#ifdef VSOCK {"agent" , "listen on a vsock socket for trace clients",
@@ -245,6 +246,7 @@ static struct usage_help usage_help[] = { " -p port number to listen on.\n" " -D run in daemon mode.\n" }, +#endif { "list", "list the available events, plugins or options",