Add VM kernel tracing over vsockets and FIFOs
v7 cover Add VM kernel tracing over vsockets and FIFOs
This patchset adds support for tracing guest kernels to trace-cmd.
The first two patches in the series are minor cleanups which can be merged separately.
Changes in v7:
- fixed potential memory leak in add_arg()
- call msg_free() in tracecmd_msg_recv_trace_req() eventhough we're stealing the message buffer
- added is_guest and is_agent macros and use throughout tracecmd_record.c instead of testing instance->flags bits
- avoid memory allocation in make_dir()
Changes in v6:
- added specialized data transfer path over FIFOs making single splice() per iteration
- made tracecmd_msg_recv_trace_req/tracecmd_msg_recv_trace_resp more defensive to invalid messages
Changes in v5:
- add FIFOs transport for tracing data
- fixed a bug in tracecmd_msg_wait_close
Changes in v4:
- detect and use splice(2) on vsock sockets if possible
- switch port numbers to unsigned int
- trace-cmd record --date flag is now set for all guests if provided by the user
- removed grow_cap and exponential buffer size growth
Changes in v3:
- addressed Steven's feedback
- detect and disable guest tracing if <linux/vm_sockets.h> is not available
- the --date flag is now treated as global for all guest instances
- fixed a bug that caused --date to be ignored for host tracing data when tracing guests
Changes in v2:
- rebased on top of protocol V3
- fixed system clock timestamps with the --date flag
Slavomir Kaslev (12):
trace-cmd: Minor cleanup in tracecmd_start_recording()
trace-cmd: Minor cleanup in print_stat()
trace-cmd: Add tracecmd_create_recorder_virt function
trace-cmd: Add TRACE_REQ and TRACE_RESP messages
trace-cmd: Add buffer instance flags for tracing in guest and agent
context
trace-cmd: Add VM kernel tracing over vsockets transport
trace-cmd: Use splice(2) for vsockets if available
trace-cmd: Add trace-cmd setup-guest command
trace-cmd: Try to autodetect number of guest CPUs in setup-guest if
not specified
trace-cmd: Add setup-guest flag for attaching FIFOs to the guest VM
config
trace-cmd: Add splice() recording from FIFO without additional pipe
buffer
trace-cmd: Add VM tracing over FIFOs transport
Steven Rostedt (VMware) (1):
trace-cmd: Detect if vsockets are available
Makefile | 7 + include/trace-cmd/trace-cmd.h | 16 +- lib/trace-cmd/trace-recorder.c | 140 ++++-- tracecmd/Makefile | 7 +- tracecmd/include/trace-local.h | 29 ++ tracecmd/trace-agent.c | 257 +++++++++++ tracecmd/trace-cmd.c | 4 + tracecmd/trace-msg.c | 219 ++++++++- tracecmd/trace-record.c | 779 ++++++++++++++++++++++++++++++--- tracecmd/trace-setup-guest.c | 245 +++++++++++ tracecmd/trace-usage.c | 22 +- 11 files changed, 1637 insertions(+), 88 deletions(-) create mode 100644 tracecmd/trace-agent.c create mode 100644 tracecmd/trace-setup-guest.c
--
2.19.1
RFC PATCH v7 1/13 trace-cmd: Minor cleanup in tracecmd_start_recording()
Initialize struct timespec req only once instead of on every loop iteration since sleep is being constant throughout tracecmd_start_recording() execution.
No changes in behavior intended.
--- lib/trace-cmd/trace-recorder.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
lib/trace-cmd/trace-recorder.c +6 -5
@@ -466,7 +466,10 @@ long tracecmd_flush_recording(struct tracecmd_recorder *recorder)int tracecmd_start_recording (struct tracecmd_recorder *recorder,unsigned long sleep) { -struct timespec req; +struct timespec req = { + .tv_sec = sleep /1000000 , + .tv_nsec = (sleep %1000000 ) *1000 , + };long read =1 ;long ret;
@@ -474,11 +477,9 @@ int tracecmd_start_recording(struct tracecmd_recorder *recorder, unsigned long s do { /* Only sleep if we did not read anything last time */ - if (!read && sleep) { - req.tv_sec = sleep / 1000000; - req.tv_nsec = (sleep % 1000000) * 1000; + if (!read && sleep) nanosleep(&req, NULL); - } + read = 0; do { if (recorder->flags & TRACECMD_RECORD_NOSPLICE)
Signature · 1 line
2.19.1
RFC PATCH v7 2/13 trace-cmd: Minor cleanup in print_stat()
In reply to Slavomir KaslevDo an early out in print_stat() when quiet is set instead of rechecking it on each loop iteration.
No changes in behavior intended.
--- tracecmd/trace-record.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
tracecmd/trace-record.c +5 -4
@@ -3106,13 +3106,14 @@ static void print_stat(struct buffer_instance *instance) {int cpu; +if (quiet) +return ; +if (!is_top_instance (instance)) -if (!quiet) -printf ("\nBuffer: %s\n\n" , instance->name); +printf ("\nBuffer: %s\n\n" , instance->name);for (cpu =0 ; cpu < instance->cpu_count; cpu++) -if (!quiet) -trace_seq_do_printf (&instance->s_print[cpu]); +trace_seq_do_printf (&instance->s_print[cpu]); }enum {
Signature · 1 line
2.19.1
RFC PATCH v7 3/13 trace-cmd: Detect if vsockets are available
In reply to Slavomir KaslevFrom: "Steven Rostedt (VMware)" <[email protected]>
Detect and define VSOCK if vsockets are available on the system. This macro is used to disable VM remote tracing features on older kernels.
--- Makefile | 7 +++++++ 1 file changed, 7 insertions(+)
Makefile +7 -0
@@ -204,6 +204,13 @@ CFLAGS ?= -g -Wall CPPFLAGS ?= LDFLAGS ?= +VSOCK_DEFINED := $(shell if (echo "\#include <linux/vm_sockets.h>" | $(CC) -E - >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi) + +export VSOCK_DEFINED +ifeq ($(VSOCK_DEFINED), 1) +CFLAGS += -DVSOCK +endif + export CFLAGS export INCLUDES
Signature · 1 line
2.19.1
RFC PATCH v7 4/13 trace-cmd: Add tracecmd_create_recorder_virt function
In reply to Slavomir KaslevAdd tracecmd_create_recorder_virt function that will be used for tracing VM guests.
--- include/trace-cmd/trace-cmd.h | 1 + lib/trace-cmd/trace-recorder.c | 53 ++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-)
include/trace-cmd/trace-cmd.h +1 -0
@@ -277,6 +277,7 @@ enum {void tracecmd_free_recorder (struct tracecmd_recorder *recorder);struct tracecmd_recorder *tracecmd_create_recorder (const char *file,int cpu,unsigned flags);struct tracecmd_recorder *tracecmd_create_recorder_fd (int fd,int cpu,unsigned flags); +struct tracecmd_recorder *tracecmd_create_recorder_virt (const char *file,int cpu,unsigned flags,int trace_fd);struct tracecmd_recorder *tracecmd_create_recorder_maxkb (const char *file,int cpu,unsigned flags,int maxkb);struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd (int fd,int cpu,unsigned flags,const char *buffer);struct tracecmd_recorder *tracecmd_create_buffer_recorder (const char *file,int cpu,unsigned flags,const char *buffer);
lib/trace-cmd/trace-recorder.c +38 -15
81 lines not on this page · raw message
Signature · 1 line
2.19.1
RFC PATCH v7 5/13 trace-cmd: Add TRACE_REQ and TRACE_RESP messages
In reply to Slavomir KaslevAdd TRACE_REQ and TRACE_RESP messages which are used for initiating guest VM tracing.
--- include/trace-cmd/trace-cmd.h | 12 ++ tracecmd/trace-msg.c | 211 +++++++++++++++++++++++++++++++++- 2 files changed, 222 insertions(+), 1 deletion(-)
include/trace-cmd/trace-cmd.h +12 -0
@@ -331,6 +331,18 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);bool tracecmd_msg_done (struct tracecmd_msg_handle *msg_handle);void tracecmd_msg_set_done (struct tracecmd_msg_handle *msg_handle); +int tracecmd_msg_send_trace_req (struct tracecmd_msg_handle *msg_handle, +int argc,char **argv); +int tracecmd_msg_recv_trace_req (struct tracecmd_msg_handle *msg_handle, +int *argc,char ***argv); + +int tracecmd_msg_send_trace_resp (struct tracecmd_msg_handle *msg_handle, +int nr_cpus,int page_size, +unsigned int *ports); +int tracecmd_msg_recv_trace_resp (struct tracecmd_msg_handle *msg_handle, +int *nr_cpus,int *page_size, +unsigned int **ports); +/* --- Plugin handling --- */ extern struct tep_plugin_option trace_ftrace_options[];
tracecmd/trace-msg.c +210 -1
238 lines not on this page · raw message
Signature · 1 line
2.19.1
RFC PATCH v7 6/13 trace-cmd: Add buffer instance flags for tracing in guest and agent context
In reply to Slavomir KaslevAdd BUFFER_FL_GUEST and BUFFER_FL_AGENT flags to differentiate when trace-record.c is being called to trace guest or the VM tracing agent.
Also disable functions talking to the local tracefs when called in recording guest instances context.
--- tracecmd/include/trace-local.h | 5 ++++ tracecmd/trace-record.c | 55 ++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 2 deletions(-)
tracecmd/include/trace-local.h +5 -0
@@ -149,6 +149,8 @@ char *strstrip(char *str);enum buffer_instance_flags {BUFFER_FL_KEEP =1 <<0 ,BUFFER_FL_PROFILE =1 <<1 , +BUFFER_FL_GUEST =1 <<2 , +BUFFER_FL_AGENT =1 <<3 , };struct func_list {
@@ -198,6 +200,9 @@ extern struct buffer_instance *first_instance;#define for_all_instances (i) for (i = first_instance; i; \ i = i == &top_instance ? buffer_instances : (i)->next) +#define is_agent (instance) ((instance)->flags & BUFFER_FL_AGENT) +#define is_guest (instance) ((instance)->flags & BUFFER_FL_GUEST) +struct buffer_instance *create_instance (const char *name);void add_instance (struct buffer_instance *instance,int cpu_count);char *get_instance_file (struct buffer_instance *instance,const char *file);
tracecmd/trace-record.c +53 -2
169 lines not on this page · raw message
Signature · 1 line
2.19.1
RFC PATCH v7 7/13 trace-cmd: Add VM kernel tracing over vsockets transport
In reply to Slavomir KaslevThis patch adds VM tracing over vsockets. The new trace-cmd agent command needs run on each guest we want to trace:
you@guest2 # trace-cmd agent
Then trace-cmd record on the host can collect data from both the host and several guests simultaneously:
you@host $ trace-cmd record -A guest1 -e irq -e sched \
-A guest2 -p function -e all
--- tracecmd/Makefile | 6 +- tracecmd/include/trace-local.h | 16 + tracecmd/trace-agent.c | 225 +++++++++++++ tracecmd/trace-cmd.c | 3 + tracecmd/trace-record.c | 596 ++++++++++++++++++++++++++++++--- tracecmd/trace-usage.c | 13 +- 6 files changed, 814 insertions(+), 45 deletions(-) create mode 100644 tracecmd/trace-agent.c
tracecmd/Makefile +5 -1
@@ -33,13 +33,17 @@ TRACE_CMD_OBJS += trace-output.o TRACE_CMD_OBJS += trace-usage.o TRACE_CMD_OBJS += trace-msg.o +ifeq ($(VSOCK_DEFINED), 1) +TRACE_CMD_OBJS += trace-agent.o +endif + ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o) all_objs := $(sort $(ALL_OBJS)) all_deps := $(all_objs:$(bdir)/%.o=$(bdir)/.%.d) CONFIG_INCLUDES = -CONFIG_LIBS = +CONFIG_LIBS = -lrt CONFIG_FLAGS = all: $(TARGETS)
tracecmd/include/trace-local.h +16 -0
@@ -12,6 +12,8 @@#include "trace-cmd.h" #include "event-utils.h" +#define TRACE_AGENT_DEFAULT_PORT 823 +extern int debug;extern int quiet;
@@ -64,6 +66,8 @@ void trace_split(int argc, char **argv);void trace_listen (int argc,char **argv); +void trace_agent (int argc,char **argv); +void trace_restore (int argc,char **argv);void trace_clear (int argc,char **argv);
@@ -88,6 +92,10 @@ void trace_list(int argc, char **argv);void trace_usage (int argc,char **argv); +int trace_record_agent (struct tracecmd_msg_handle *msg_handle, +int cpus,int *fds, +int argc,char **argv); +struct hook_list ; void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
@@ -176,6 +184,7 @@ struct buffer_instance {struct func_list *notrace_funcs;const char *clock; +unsigned int *client_ports;struct trace_seq *s_save;struct trace_seq *s_print;
@@ -190,6 +199,13 @@ struct buffer_instance {int tracing_on_fd;int buffer_size;int cpu_count; + +int argc; +char **argv; + +int cid; +int port; +int *fds; };extern struct buffer_instance top_instance;
tracecmd/trace-agent.c new file +225 -0
225 lines not on this page · raw message
tracecmd/trace-cmd.c +3 -0
@@ -83,6 +83,9 @@ struct command commands[] = { {"hist", trace_hist}, {"mem", trace_mem}, {"listen", trace_listen}, +#ifdef VSOCK + {"agent", trace_agent}, +#endif {"split", trace_split}, {"restore", trace_restore}, {"stack", trace_stack},
tracecmd/trace-record.c +553 -43
848 lines not on this page · raw message
tracecmd/trace-usage.c +12 -1
@@ -231,11 +231,22 @@ static struct usage_help usage_help[] = {"listen on a network socket for trace clients" ," %s listen -p port[-D][-o file][-d dir][-l logfile]\n" " Creates a socket to listen for clients.\n" -" -D create it in daemon mode.\n" +" -p port number to listen on.\n" +" -D run in daemon mode.\n" " -o file name to use for clients.\n" " -d directory to store client files.\n" " -l logfile to write messages to.\n" }, +#ifdef VSOCK + { +"agent" , +"listen on a vsocket for trace clients" , +" %s agent -p port[-D]\n" +" Creates a vsocket to listen for clients.\n" +" -p port number to listen on.\n" +" -D run in daemon mode.\n" + }, +#endif {"list" , "list the available events, plugins or options",
Signature · 1 line
2.19.1
RFC PATCH v7 8/13 trace-cmd: Use splice(2) for vsockets if available
In reply to Slavomir KaslevDetect if splice(2) reading is supported for vsockets (Linux 4.20 and later) and use it, or fallback to read/write-ing otherwise.
--- lib/trace-cmd/trace-recorder.c | 6 ++-- tracecmd/trace-record.c | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-)
lib/trace-cmd/trace-recorder.c +2 -4
@@ -386,10 +386,8 @@ static long splice_data(struct tracecmd_recorder *recorder) ret =splice (recorder->brass[0 ],NULL , recorder->fd,NULL , read, recorder->fd_flags);if (ret <0 ) { -if (errno !=EAGAIN && errno !=EINTR ) { -warning ("recorder error in splice output" ); -return -1 ; - } +if (errno !=EAGAIN && errno !=EINTR ) +return ret;return total_read; }else update_fd (recorder, ret);
tracecmd/trace-record.c +54 -2
79 lines not on this page · raw message
Signature · 1 line
2.19.1
RFC PATCH v7 9/13 trace-cmd: Add `trace-cmd setup-guest` command
In reply to Slavomir KaslevAdd trace-cmd setup-guest command that creates the necessary FIFOs for tracing a guest over FIFOs instead of vsockets.
--- tracecmd/Makefile | 1 + tracecmd/include/trace-local.h | 6 ++ tracecmd/trace-cmd.c | 1 + tracecmd/trace-setup-guest.c | 176 +++++++++++++++++++++++++++++++++ tracecmd/trace-usage.c | 8 ++ 5 files changed, 192 insertions(+) create mode 100644 tracecmd/trace-setup-guest.c
tracecmd/Makefile +1 -0
@@ -35,6 +35,7 @@ TRACE_CMD_OBJS += trace-msg.o ifeq ($(VSOCK_DEFINED), 1) TRACE_CMD_OBJS += trace-agent.o +TRACE_CMD_OBJS += trace-setup-guest.o endif ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o)
tracecmd/include/trace-local.h +6 -0
@@ -14,6 +14,10 @@#define TRACE_AGENT_DEFAULT_PORT 823 +#define GUEST_PIPE_NAME "trace-pipe-cpu" +#define GUEST_DIR_FMT "/var/lib/trace-cmd/virt/%s" +#define GUEST_FIFO_FMT GUEST_DIR_FMT "/" GUEST_PIPE_NAME "%d" +extern int debug;extern int quiet;
@@ -68,6 +72,8 @@ void trace_listen(int argc, char **argv);void trace_agent (int argc,char **argv); +void trace_setup_guest (int argc,char **argv); +void trace_restore (int argc,char **argv);void trace_clear (int argc,char **argv);
tracecmd/trace-cmd.c +1 -0
@@ -85,6 +85,7 @@ struct command commands[] = { {"listen", trace_listen}, #ifdef VSOCK {"agent", trace_agent}, + {"setup-guest", trace_setup_guest}, #endif {"split", trace_split}, {"restore", trace_restore},
tracecmd/trace-setup-guest.c new file +176 -0
176 lines not on this page · raw message
tracecmd/trace-usage.c +8 -0
@@ -246,6 +246,14 @@ static struct usage_help usage_help[] = { " -p port number to listen on.\n" " -D run in daemon mode.\n" }, + { + "setup-guest", + "create FIFOs for tracing guest VMs", + " %s setup-guest -c cpus[-p perm][-g group] guest\n" + " -c number of guest virtual CPUs\n" + " -p FIFOs permissions (default: 0660)\n" + " -g FIFOs group owner\n" + }, #endif { "list",
Signature · 1 line
2.19.1
RFC PATCH v7 10/13 trace-cmd: Try to autodetect number of guest CPUs in setup-guest if not specified
In reply to Slavomir KaslevWhen no number of guest CPUs is provided with the -c flag to trace-cmd setup-guest, try to autodetect it using virsh for libvirt managed guests.
--- tracecmd/trace-setup-guest.c | 21 +++++++++++++++++++++ tracecmd/trace-usage.c | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-)
tracecmd/trace-setup-guest.c +21 -0
@@ -91,6 +91,24 @@ static int make_guest_fifos(const char *guest, int nr_cpus, mode_t mode)return ret; } +static int get_guest_cpu_count (const char *guest) +{ +const char *cmd_fmt ="virsh vcpucount --maximum '%s' 2>/dev/null" ; +int nr_cpus =-1 ; +char cmd[1024 ]; +FILE *f; + +snprintf (cmd,sizeof (cmd), cmd_fmt, guest); + f =popen (cmd,"r" ); +if (!f) +return -errno; + +fscanf (f,"%d" , &nr_cpus); +pclose (f); + +return nr_cpus; +} +static void do_setup_guest (const char *guest,int nr_cpus,mode_t mode,gid_t gid) {gid_t save_egid;
@@ -169,6 +187,9 @@ void trace_setup_guest(int argc, char **argv) guest = argv[optind+1 ]; +if (nr_cpus <=0 ) + nr_cpus =get_guest_cpu_count (guest); +if (nr_cpus <=0 )pdie ("invalid number of cpus for guest %s" , guest);
tracecmd/trace-usage.c +1 -1
@@ -249,7 +249,7 @@ static struct usage_help usage_help[] = { {"setup-guest" ,"create FIFOs for tracing guest VMs" , -" %s setup-guest -c cpus[-p perm][-g group] guest\n" +" %s setup-guest [-c cpus][-p perm][-g group] guest\n" " -c number of guest virtual CPUs\n" " -p FIFOs permissions (default: 0660)\n" " -g FIFOs group owner\n"
Signature · 1 line
2.19.1
RFC PATCH v7 11/13 trace-cmd: Add setup-guest flag for attaching FIFOs to the guest VM config
In reply to Slavomir KaslevThis patch adds a flag for attaching the newly created FIFOs for guest tracing as virtio serial devices to libvirt managed guests.
--- tracecmd/trace-setup-guest.c | 54 ++++++++++++++++++++++++++++++++++-- tracecmd/trace-usage.c | 3 +- 2 files changed, 53 insertions(+), 4 deletions(-)
tracecmd/trace-setup-guest.c +51 -3
@@ -109,7 +109,45 @@ static int get_guest_cpu_count(const char *guest)return nr_cpus; } -static void do_setup_guest (const char *guest,int nr_cpus,mode_t mode,gid_t gid) +static int attach_guest_fifos (const char *guest,int nr_cpus) +{ +const char *cmd_fmt = +"virsh attach-device --config '%s' '%s' >/dev/null 2>/dev/null" ; +const char *xml_fmt = +"<channel type='pipe'>\n" +" <source path='%s'/>\n" +" <target type='virtio' name='%s%d'/>\n" +"</channel>" ; +char tmp_path[PATH_MAX ], path[PATH_MAX ]; +char cmd[PATH_MAX ], xml[PATH_MAX ]; +int i, fd, ret =0 ; + +strcpy (tmp_path,"/tmp/pipexmlXXXXXX" ); + fd =mkstemp (tmp_path); +if (fd <0 ) +return fd; + +for (i =0 ; i < nr_cpus; i++) { +snprintf (path,sizeof (path),GUEST_FIFO_FMT , guest, i); +snprintf (xml,sizeof (xml), xml_fmt, path,GUEST_PIPE_NAME , i); +pwrite (fd, xml,strlen (xml),0 ); + +snprintf (cmd,sizeof (cmd), cmd_fmt, guest, tmp_path); + errno =0 ; +if (system (cmd) !=0 ) { + ret = -errno; +break ; + } + } + +close (fd); +unlink (tmp_path); + +return ret; +} + +static void do_setup_guest (const char *guest,int nr_cpus, +mode_t mode,gid_t gid,bool attach) {gid_t save_egid;int ret;
@@ -129,6 +167,12 @@ static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t giif (ret <0 )pdie ("failed to create FIFOs for %s" , guest); +if (attach) { + ret =attach_guest_fifos (guest, nr_cpus); +if (ret <0 ) +pdie ("failed to attach FIFOs to %s" , guest); + } +if (gid !=-1 ) { ret =setegid (save_egid);if (ret <0 )
@@ -138,6 +182,7 @@ static void do_setup_guest(const char *guest, int nr_cpus, mode_t mode, gid_t givoid trace_setup_guest (int argc,char **argv) { +bool attach = false;struct group *group;mode_t mode =0660 ;int nr_cpus =-1 ;
@@ -157,7 +202,7 @@ void trace_setup_guest(int argc, char **argv) {NULL, 0, NULL, 0} }; - c =getopt_long (argc-1 , argv+1 ,"+hc:p:g:" , + c =getopt_long (argc-1 , argv+1 ,"+hc:p:g:a" , long_options, &option_index);if (c ==-1 )break ;
@@ -177,6 +222,9 @@ void trace_setup_guest(int argc, char **argv)pdie ("group %s does not exist" , optarg); gid = group->gr_gid;break ; +case 'a' : + attach = true; +break ;default :usage (argv); }
@@ -193,5 +241,5 @@ void trace_setup_guest(int argc, char **argv)if (nr_cpus <=0 )pdie ("invalid number of cpus for guest %s" , guest); -do_setup_guest (guest, nr_cpus, mode, gid); +do_setup_guest (guest, nr_cpus, mode, gid, attach); }
tracecmd/trace-usage.c +2 -1
@@ -249,10 +249,11 @@ static struct usage_help usage_help[] = { {"setup-guest" ,"create FIFOs for tracing guest VMs" , -" %s setup-guest [-c cpus][-p perm][-g group] guest\n" +" %s setup-guest [-c cpus][-p perm][-g group][-a] guest\n" " -c number of guest virtual CPUs\n" " -p FIFOs permissions (default: 0660)\n" " -g FIFOs group owner\n" +" -a Attach FIFOs to guest VM config\n" }, #endif {
Signature · 1 line
2.19.1
RFC PATCH v7 12/13 trace-cmd: Add splice() recording from FIFO without additional pipe buffer
In reply to Slavomir KaslevWhen trace-cmd record is reading tracing data over FIFO we can do a direct splice from the FIFO to the output file descriptor instead of doing two through an additional pipe buffer. This patch implements specialized tracecmd_recorder data transfer version for this case.
--- include/trace-cmd/trace-cmd.h | 3 +- lib/trace-cmd/trace-recorder.c | 70 ++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 12 deletions(-)
include/trace-cmd/trace-cmd.h +2 -1
@@ -270,8 +270,9 @@ struct tracecmd_output *tracecmd_get_output_handle_fd(int fd);enum {TRACECMD_RECORD_NOSPLICE = (1 <<0 ),/* Use read instead of splice */ -TRACECMD_RECORD_SNAPSHOT = (1 <<1 ),/* extract from snapshot */ +TRACECMD_RECORD_SNAPSHOT = (1 <<1 ),/* Extract from snapshot */ TRACECMD_RECORD_BLOCK = (1 <<2 ),/* Block on splice write */ +TRACECMD_RECORD_NOBRASS = (1 <<3 ),/* Splice directly without a brass pipe */ };void tracecmd_free_recorder (struct tracecmd_recorder *recorder);
lib/trace-cmd/trace-recorder.c +59 -11
137 lines not on this page · raw message
Signature · 1 line
2.19.1
RFC PATCH v7 13/13 trace-cmd: Add VM tracing over FIFOs transport
In reply to Slavomir KaslevFIFOs offer ~3x the throughput of vsockets. This patch adds support for using FIFOs to stream tracing data back to the host when tracing VMs.
--- include/trace-cmd/trace-cmd.h | 8 ++-- tracecmd/include/trace-local.h | 4 +- tracecmd/trace-agent.c | 40 ++++++++++++++-- tracecmd/trace-msg.c | 26 ++++++---- tracecmd/trace-record.c | 87 ++++++++++++++++++++++++++++++---- 5 files changed, 137 insertions(+), 28 deletions(-)
include/trace-cmd/trace-cmd.h +4 -4
@@ -333,16 +333,16 @@ bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);void tracecmd_msg_set_done (struct tracecmd_msg_handle *msg_handle);int tracecmd_msg_send_trace_req (struct tracecmd_msg_handle *msg_handle, -int argc,char **argv); +int argc,char **argv,bool use_fifos);int tracecmd_msg_recv_trace_req (struct tracecmd_msg_handle *msg_handle, -int *argc,char ***argv); +int *argc,char ***argv,bool *use_fifos);int tracecmd_msg_send_trace_resp (struct tracecmd_msg_handle *msg_handle,int nr_cpus,int page_size, -unsigned int *ports); +unsigned int *ports,bool use_fifos);int tracecmd_msg_recv_trace_resp (struct tracecmd_msg_handle *msg_handle,int *nr_cpus,int *page_size, -unsigned int **ports); +unsigned int **ports,bool *use_fifos);/* --- Plugin handling --- */ extern struct tep_plugin_option trace_ftrace_options[];
tracecmd/include/trace-local.h +3 -1
@@ -17,6 +17,7 @@#define GUEST_PIPE_NAME "trace-pipe-cpu"#define GUEST_DIR_FMT "/var/lib/trace-cmd/virt/%s"#define GUEST_FIFO_FMT GUEST_DIR_FMT "/" GUEST_PIPE_NAME "%d" +#define VIRTIO_FIFO_FMT "/dev/virtio-ports/" GUEST_PIPE_NAME "%d"extern int debug;extern int quiet;
@@ -100,7 +101,7 @@ void trace_usage(int argc, char **argv);int trace_record_agent (struct tracecmd_msg_handle *msg_handle,int cpus,int *fds, -int argc,char **argv); +int argc,char **argv,bool use_fifos);struct hook_list ;
@@ -212,6 +213,7 @@ struct buffer_instance {int cid;int port;int *fds; +bool use_fifos; };extern struct buffer_instance top_instance;
tracecmd/trace-agent.c +36 -4
@@ -83,12 +83,39 @@ static void make_vsocks(int nr, int *fds, unsigned int *ports) } } +static int open_agent_fifos (int nr_cpus,int *fds) +{ +char path[PATH_MAX ]; +int i, fd, ret =0 ; + +for (i =0 ; i < nr_cpus; i++) { +snprintf (path,sizeof (path),VIRTIO_FIFO_FMT , i); + fd =open (path,O_WRONLY ); +if (fd <0 ) { + ret = -errno; +break ; + } + + fds[i] = fd; + } + +if (!ret) +return ret; + +/* We failed to open all FIFOs so clean up and return error */ +while (--i >=0 ) +close (fds[i]); + +return ret; +} +static void agent_handle (int sd,int nr_cpus,int page_size) {struct tracecmd_msg_handle *msg_handle;unsigned int *ports;char **argv =NULL ;int argc =0 ; +bool use_fifos;int *fds;int ret;
@@ -101,17 +128,22 @@ static void agent_handle(int sd, int nr_cpus, int page_size)if (!msg_handle)die ("Failed to allocate message handle" ); - ret =tracecmd_msg_recv_trace_req (msg_handle, &argc, &argv); + ret =tracecmd_msg_recv_trace_req (msg_handle, &argc, &argv, &use_fifos);if (ret <0 )die ("Failed to receive trace request" ); -make_vsocks (nr_cpus, fds, ports); +if (use_fifos &&open_agent_fifos (nr_cpus, fds)) + use_fifos = false; + +if (!use_fifos) +make_vsocks (nr_cpus, fds, ports); - ret =tracecmd_msg_send_trace_resp (msg_handle, nr_cpus, page_size, ports); + ret =tracecmd_msg_send_trace_resp (msg_handle, nr_cpus, page_size, + ports, use_fifos);if (ret <0 )die ("Failed to send trace response" ); -trace_record_agent (msg_handle, nr_cpus, fds, argc, argv); +trace_record_agent (msg_handle, nr_cpus, fds, argc, argv, use_fifos);free (argv[0 ]);free (argv);
tracecmd/trace-msg.c +17 -9
@@ -159,6 +159,10 @@ static int msg_write(int fd, struct tracecmd_msg *msg)return __do_write_check (fd, msg->buf, data_size); } +enum msg_trace_flags { +MSG_TRACE_USE_FIFOS =1 <<0 , +}; +enum msg_opt_command {MSGOPT_USETCP =1 , };
@@ -740,7 +744,7 @@ error:return ret; } -static int make_trace_req (struct tracecmd_msg *msg,int argc,char **argv) +static int make_trace_req (struct tracecmd_msg *msg,int argc,char **argv,bool use_fifos) {size_t args_size =0 ;char *p;
@@ -750,6 +754,7 @@ static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv) args_size +=strlen (argv[i]) +1 ; msg->hdr.size =htonl (ntohl (msg->hdr.size) + args_size); + msg->trace_req.flags = use_fifos ?htonl (MSG_TRACE_USE_FIFOS ) :htonl (0 ); msg->trace_req.argc =htonl (argc); msg->buf =calloc (args_size,1 );if (!msg->buf)
@@ -763,13 +768,13 @@ static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv) }int tracecmd_msg_send_trace_req (struct tracecmd_msg_handle *msg_handle, -int argc,char **argv) +int argc,char **argv,bool use_fifos) {struct tracecmd_msg msg;int ret;tracecmd_msg_init (MSG_TRACE_REQ , &msg); - ret =make_trace_req (&msg, argc, argv); + ret =make_trace_req (&msg, argc, argv, use_fifos);if (ret <0 )return ret;
@@ -782,7 +787,7 @@ int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,* free(argv); */ int tracecmd_msg_recv_trace_req (struct tracecmd_msg_handle *msg_handle, -int *argc,char ***argv) +int *argc,char ***argv,bool *use_fifos) {struct tracecmd_msg msg;char *p, *buf_end, **args;
@@ -834,6 +839,7 @@ int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle, *argc = nr_args; *argv = args; + *use_fifos =ntohl (msg.trace_req.flags) &MSG_TRACE_USE_FIFOS ;/* * On success we're passing msg.buf to the caller through argv[0] so we
@@ -853,13 +859,14 @@ out:return ret; } -static int make_trace_resp (struct tracecmd_msg *msg, -int page_size,int nr_cpus,unsigned int *ports) +static int make_trace_resp (struct tracecmd_msg *msg,int page_size,int nr_cpus, +unsigned int *ports,bool use_fifos) {int ports_size = nr_cpus *sizeof (*msg->port_array);int i; msg->hdr.size =htonl (ntohl (msg->hdr.size) + ports_size); + msg->trace_resp.flags = use_fifos ?htonl (MSG_TRACE_USE_FIFOS ) :htonl (0 ); msg->trace_resp.cpus =htonl (nr_cpus); msg->trace_resp.page_size =htonl (page_size);
@@ -875,13 +882,13 @@ static int make_trace_resp(struct tracecmd_msg *msg,int tracecmd_msg_send_trace_resp (struct tracecmd_msg_handle *msg_handle,int nr_cpus,int page_size, -unsigned int *ports) +unsigned int *ports,bool use_fifos) {struct tracecmd_msg msg;int ret;tracecmd_msg_init (MSG_TRACE_RESP , &msg); - ret =make_trace_resp (&msg, page_size, nr_cpus, ports); + ret =make_trace_resp (&msg, page_size, nr_cpus, ports, use_fifos);if (ret <0 )return ret;
@@ -890,7 +897,7 @@ int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,int tracecmd_msg_recv_trace_resp (struct tracecmd_msg_handle *msg_handle,int *nr_cpus,int *page_size, -unsigned int **ports) +unsigned int **ports,bool *use_fifos) {struct tracecmd_msg msg;ssize_t buf_len;
@@ -912,6 +919,7 @@ int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle, goto out; } + *use_fifos =ntohl (msg.trace_resp.flags) &MSG_TRACE_USE_FIFOS ; *nr_cpus =ntohl (msg.trace_resp.cpus); *page_size =ntohl (msg.trace_resp.page_size); *ports =calloc (*nr_cpus,sizeof (**ports));
tracecmd/trace-record.c +77 -10
162 lines not on this page · raw message
Signature · 1 line
2.19.1