trce-cmd: add virtual machine tracing

Hi,

This is the version 6 series of virtio-trace for trace-cmd. The previous series is here; https://lkml.org/lkml/2014/12/22/79

I took over this work from Yoshihiro Yunomae. This version updates previous patch on the latest trace-cmd tree and recover Yoshihiro's signed-off and original authorship.

How to use
==========
1. Run virt-server on a host

   # trace-cmd virt-server --dom guest1 -c 2

2. Set up of virtio-serial pipe of guest1 on the host Add the following tags to domain XML files.

   # virsh edit guest1
   <channel type='unix'>
      <source mode='connect' path='/tmp/trace-cmd/virt/agent-ctl-path'/>
      <target type='virtio' name='agent-ctl-path'/>
   </channel>
   <channel type='pipe'>
      <source path='/tmp/trace-cmd/virt/guest1/trace-path-cpu0'/>
      <target type='virtio' name='trace-path-cpu0'/>
   </channel>
   <channel type='pipe'>
      <source path='/tmp/trace-cmd/virt/guest1/trace-path-cpu1'/>
      <target type='virtio' name='trace-path-cpu1'/>
   </channel>

3. Boot the guest

   # virsh start guest1

4. Run the guest1's client(see trace-cmd-record(1) with the *--virt* option)

   # trace-cmd record -e sched* --virt

If you want to boot another guest sends trace-data via virtio-serial, you will manually make the guest domain directory and trace data I/Fs.

- Make guest domain directory on the host

   # mkdir -p /tmp/trace-cmd/virt/<DOMAIN>
   # chmod 710 /tmp/trace-cmd/virt/<DOMAIN>
   # chgrp qemu /tmp/trace-cmd/virt/<DOMAIN>

- Make FIFO on the host

   # mkfifo /tmp/trace-cmd/virt/<DOMAIN>/trace-path-cpu{0,1,...,X}.{in,out}

TODO
====
- Don't use fixed directory and fifos. Make it flexible.
- Don't depend on the libvirt. We can find fifos in /proc/<pid>/fd/*.
- Cleanup the code. It is not well structured now.

Thank you,

---

Yoshihiro YUNOMAE (7):
trace-cmd: Support -N option for trace-cmd extract
trace-cmd/listen: Introduce trace-msg protocol (protocol v2) trace-cmd/msg: Use poll(2) to wait for a message trace-cmd/virt-server: Add virt-server mode for a virtualization environment trace-cmd/record: Add --virt option for record mode trace-cmd/virt-server: Add --dom option which makes a domain directory to virt-server
trace-cmd: Use pid instead of libvirt virt domain name

 Documentation/Protocol.txt                |  163 +++++
 Documentation/trace-cmd-record.1.txt      |   12 
 Documentation/trace-cmd-virt-server.1.txt |  113 ++++
 Makefile                                  |    3 
 trace-cmd.c                               |    3 
 trace-cmd.h                               |   15 +
 trace-listen.c                            |  687 +++++++++++++++++++----
 trace-msg.c                               |  870 +++++++++++++++++++++++++++++
 trace-msg.h                               |   31 +
 trace-output.c                            |    4 
 trace-record.c                            |  158 +++++
 trace-recorder.c                          |   50 +-
 trace-usage.c                             |   18 +
 13 files changed, 1991 insertions(+), 136 deletions(-)
 create mode 100644 Documentation/Protocol.txt
 create mode 100644 Documentation/trace-cmd-virt-server.1.txt
 create mode 100644 trace-msg.c
 create mode 100644 trace-msg.h

Masami HIRAMATSU
Linux Technology Research Center, System Productivity Research Dept.
Center for Technology Innovation - Systems Engineering 
Hitachi, Ltd., Research & Development Group
E-mail: [email protected]

From: Yoshihiro YUNOMAE <[email protected]>

There is no reason to prohibit supporting -N command in extract mode, since both record and extract read trace logs from ftrace and save it.

---
 trace-record.c |    8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

index 9874a12..c387aff 100644
@@ -520,6 +520,7 @@ static void stop_threads(enum trace_type type)
 
 static int create_recorder(struct buffer_instance *instance, int cpu,
 			   enum trace_type type, int *brass);
+static void setup_network(void);
 
 static void flush_threads(void)
 {

@@ -529,6 +530,9 @@ static void flush_threads(void)
 	if (!cpu_count)
 		return;
 
+	if (host)
+		setup_network();
+
 	for (i = 0; i < cpu_count; i++) {
 		/* Extract doesn't support sub buffers yet */
 		ret = create_recorder(&top_instance, i, TRACE_TYPE_EXTRACT, NULL);

@@ -4047,8 +4051,8 @@ void trace_record (int argc, char **argv)
 			rt_prio = atoi(optarg);
 			break;
 		case 'N':
-			if (!record)
-				die("-N only available with record");
+			if (!record && !extract)
+				die("-N only available with record or extract");
 			if (output)
 				die("-N incompatible with -o");
 			host = optarg;

From: Yoshihiro YUNOMAE <[email protected]>

Introduce new trace-msg protocol (protocol V2) for more flexible messaging. V1 protocol which is currently used by trace-cmd server and client, is based on a simple text messages. It is impossible to extend the protocol without breaking backward compatibility. The V2 protocol introduced by this patch is a binary message-based protocol and it is able to extend by just adding message tags.

<How to test>
[1] Backward compatibility checks
We need to test backward compatibility of this patch for old trace-cmds(client/server). So, this patch was tested for [2] command checks in following 3 types:

	<client>	<server>
	  new		  old
	  old		  new
	  new		  new

[2] Command checks
- server (common)

	# trace-cmd listen -p 12345

1) record
- client

	# trace-cmd record -e sched -N <server IP>:12345
	^C

2) record + multiple buffers
- client

	# trace-cmd record -B foo -e sched -N <server IP>:12345
	^C

3) extract
- client

	# ./trace-cmd start -e sched
	# sleep 5
	# ./trace-cmd stop
	# ./trace-cmd extract -N <server IP>:12345

4) extract + snapshot
- client

	# ./trace-cmd start -e sched
	# sleep 5
	# ./trace-cmd snapshot -s
	# ./trace-cmd stop
	# ./trace-cmd extract -N <server IP>:12345 -s

---
Changes in v6: Update to the latest master.
Fix build errors.
Changes in V5: Client sends "-1V2\0<MAGIC_NUMBER>\00" instead of
"V2\0<MAGIC_NUMBER>\0" not to make old server create zero length file.
Also add a protocol documentation.
Cleanup source code.
Change meaningless loop in tracecmd_msg_collect_metadata().
Changes in V4: Fix some typos, cleanups and rebase for current trace-cmd-v2.4
Change the argument of tracecmd_msg_recv()
Changes in V3: Change the license of trace-msg.c to LGPL v2.1 Changes in V2: Regacy protocol support in order to keep backward compatibility

---
 Documentation/Protocol.txt |  119 ++++++++
 Makefile                   |    3 
 trace-cmd.h                |   11 +
 trace-listen.c             |   84 ++++-
 trace-msg.c                |  682 ++++++++++++++++++++++++++++++++++++++++++++
 trace-msg.h                |   27 ++
 trace-output.c             |    4 
 trace-record.c             |   89 +++++-
 8 files changed, 982 insertions(+), 37 deletions(-)
 create mode 100644 Documentation/Protocol.txt
 create mode 100644 trace-msg.c
 create mode 100644 trace-msg.h

new file mode 100644
index 0000000..49f7766
@@ -0,0 +1,119 @@
+Trace-cmd Protocols
+===================
+
+Index
+=====
+1. What is the trace-cmd protocol?
+2. Trace-cmd Protocol V1 (Obsolete)
+3. Trace-cmd Protocol V2
+
+
+1. What is the trace-cmd protocol?
+==================================
+ Trace-cmd can run as a remote-trace agent(server) and a client, which
+communicate over network and passing the trace data. Trace-cmd protocol
+is used for the communication between the server and the client.
+ There are 2 versions of the trace-cmd protocol. V1 protocol was simple
+text-based but hard to extend. On the other hand, V2 protocol is message
+based and extensible.
+
+
+2. Trace-cmd Protocol V1 (Obsolete)
+========================
+
+The old trace-cmd which supports V1 protocol works as follows;
+
+     <server(local)>          <client(remote)>
+      listen to socket fd
+                              connect to socket fd
+      accept the client
+      send "tracecmd"
+               +------------> receive "tracecmd"
+                              check "tracecmd"
+                              send cpus
+      receive cpus <------------+
+      print "cpus=XXX"
+                              send pagesize
+                                |
+      receive pagesize <--------+
+      print "pagesize=XXX"
+                              send options
+                                |
+      receive options <---------+
+      understand options
+      send port_array
+               +------------> receive port_array
+                              understand port_array
+                              send meta data
+      receive meta data <-------+
+      record meta data
+                        (snip)
+      read block
+     --- start sending trace data on child processes ---
+
+     --- When client finishes sending trace data ---
+                              close(socket fd)
+      read size = 0
+      close(socket fd)
+
+All messages are unstructured character strings and the messaging
+order and contents are fixed. It is impossible to extend the
+protocol without breaking the compatibility.
+
+
+3. Trace-cmd Protocol V2
+========================
+
+From the protocol V2, the structured binary message "trace-msg" is
+introduced as the communication protocol.
+
+     <server>                 <client>
+      listen to socket fd
+                              connect to socket fd
+      accept the client
+      send "tracecmd"
+               +------------> receive "tracecmd"
+                              check "tracecmd"
+                              send "-1V2\0<MAGIC_NUMBER>\0" as the v2 protocol
+      receive "-1V2" <----------+
+      check "-1V2"
+      check <MAGIC_NUMBER>
+      send "V2"
+            +---------------> receive "V2"
+                              check "V2"
+                              send MSG_TINIT with cpus, pagesize and options
+      receive MSG_TINIT <-------+
+      perse the parameters
+      send MSG_RINIT with port_array
+           +----------------> receive MSG_RINIT
+                              get port_array
+                              send meta data(MSG_SENDMETA)
+      receive MSG_SENDMETA <----+
+      record meta data
+                         (snip)
+                              send a message to finish sending meta data
+                                |                           (MSG_FINMETA)
+      receive MSG_FINMETA <-----+
+      read block
+     --- start sending trace data on child processes ---
+
+     --- When client finishes sending trace data ---
+                              send MSG_CLOSE
+      receive MSG_CLOSE <-------+
+      close(socket fd)        close(socket fd)
+
+In this version, after the client checks "tracecmd", it sends
+"-1V2\0<MAGIC_NUMBER>\0". This is for the backward compatibility.
+When the newer client tries to connect to the old server and sends
+this string to the server, the old server parses it to get the
+number of CPUs. Since "-1V2" actually becomes -1 and this is a
+wrong value, the server refuses the client. Then, the client gets
+a connection error because the server is old, so it can try to
+connect with V1 protocol again.
+
+On the other hand, if new server gets a connection from an old
+client, it can easily check whether the client uses V1 protocol
+or not by checking the first message from the client. If client
+sends a positive number, it should be a V1 protocol client.
+
+

index 63f7e79..59a5a0c 100644
@@ -320,7 +320,8 @@ PEVENT_LIB_OBJS = event-parse.o trace-seq.o parse-filter.o parse-utils.o
 TCMD_LIB_OBJS = $(PEVENT_LIB_OBJS) trace-util.o trace-input.o trace-ftrace.o \
 			trace-output.o trace-record.o trace-recorder.o \
 			trace-restore.o trace-usage.o trace-blk-hack.o \
-			kbuffer-parse.o event-plugin.o trace-hooks.o
+			kbuffer-parse.o event-plugin.o trace-hooks.o \
+			trace-msg.o
 
 PLUGIN_OBJS =
 PLUGIN_OBJS += plugin_jbd2.o

index 7bce2a5..1261e23 100644
@@ -263,6 +263,17 @@ void tracecmd_stop_recording(struct tracecmd_recorder *recorder);
 void tracecmd_stat_cpu(struct trace_seq *s, int cpu);
 long tracecmd_flush_recording(struct tracecmd_recorder *recorder);
 
+/* for clients */
+int tracecmd_msg_send_init_data(int fd);
+int tracecmd_msg_metadata_send(int fd, const char *buf, int size);
+int tracecmd_msg_finish_sending_metadata(int fd);
+void tracecmd_msg_send_close_msg(void);
+
+/* for server */
+int tracecmd_msg_initial_setting(int fd, int *cpus, int *pagesize);
+int tracecmd_msg_send_port_array(int fd, int total_cpus, int *ports);
+int tracecmd_msg_collect_metadata(int ifd, int ofd);
+
 /* --- Plugin handling --- */
 extern struct pevent_plugin_option trace_ftrace_options[];
 

new file mode 100644
index 0000000..b23e72b
@@ -0,0 +1,27 @@
+#ifndef _TRACE_MSG_H_
+#define _TRACE_MSG_H_
+
+#include <stdbool.h>
+
+#define UDP_MAX_PACKET	(65536 - 20)
+#define V2_MAGIC	"677768\0"
+
+#define V1_PROTOCOL	1
+#define V2_PROTOCOL	2
+
+/* for both client and server */
+extern bool use_tcp;
+extern int cpu_count;
+
+/* for client */
+extern unsigned int page_size;
+extern int *client_ports;
+extern bool send_metadata;
+
+/* for server */
+extern bool done;
+
+void plog(const char *fmt, ...);
+void pdie(const char *fmt, ...);
+
+#endif /* _TRACE_MSG_H_ */

index 2141d10..11d7827 100644
@@ -37,6 +37,7 @@
 
 #include "trace-cmd-local.h"
 #include "list.h"
+#include "trace-msg.h"
 #include "version.h"
 
 /* We can't depend on the host size for size_t, all must be 64 bit */

@@ -82,6 +83,9 @@ struct list_event_system {
 static stsize_t
 do_write_check(struct tracecmd_output *handle, const void *data, tsize_t size)
 {
+	if (send_metadata)
+		return tracecmd_msg_metadata_send(handle->fd, data, size);
+
 	return __do_write_check(handle->fd, data, size);
 }
 

From: Yoshihiro YUNOMAE <[email protected]>

Use poll(2) to wait for a message. If a client/server cannot send a message for any reasons, the current server/client will wait in a blocking read operation. So, we use poll(2) for avoiding remaining in a blocking state.

---
Changes in V4: Change the argument of tracecmd_msg_recv_wait()
Fix some typos

---
 trace-msg.c |   42 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 6 deletions(-)

index 5669dee..e3d4f3f 100644
@@ -395,6 +395,27 @@ error:
 	return -ENOMSG;
 }
 
+#define MSG_WAIT_MSEC	5000
+
+/*
+ * A return value of 0 indicates time-out
+ */
+static int tracecmd_msg_recv_wait(int fd, struct tracecmd_msg *msg)
+{
+	struct pollfd pfd;
+	int ret;
+
+	pfd.fd = fd;
+	pfd.events = POLLIN;
+	ret = poll(&pfd, 1, MSG_WAIT_MSEC);
+	if (ret < 0)
+		return -errno;
+	else if (ret == 0)
+		return -ETIMEDOUT;
+
+	return tracecmd_msg_recv(fd, msg);
+}
+
 static void *tracecmd_msg_buf_access(struct tracecmd_msg *msg, int offset)
 {
 	return (void *)msg + offset;

@@ -405,9 +426,12 @@ static int tracecmd_msg_wait_for_msg(int fd, struct tracecmd_msg *msg)
 	u32 cmd;
 	int ret;
 
-	ret = tracecmd_msg_recv(fd, msg);
-	if (ret < 0)
+	ret = tracecmd_msg_recv_wait(fd, msg);
+	if (ret < 0) {
+		if (ret == -ETIMEDOUT)
+			warning("Connection timed out\n");
 		return ret;
+	}
 
 	cmd = ntohl(msg->cmd);
 	if (cmd == MSG_CLOSE)

@@ -487,9 +511,12 @@ int tracecmd_msg_initial_setting(int fd, int *cpus, int *pagesize)
 	u32 cmd;
 
 	msg = (struct tracecmd_msg *)buf;
-	ret = tracecmd_msg_recv(fd, msg);
-	if (ret < 0)
+	ret = tracecmd_msg_recv_wait(fd, msg);
+	if (ret < 0) {
+		if (ret == -ETIMEDOUT)
+			warning("Connection timed out\n");
 		return ret;
+	}
 
 	cmd = ntohl(msg->cmd);
 	if (cmd != MSG_TINIT) {

@@ -627,9 +654,12 @@ int tracecmd_msg_collect_metadata(int ifd, int ofd)
 	msg = (struct tracecmd_msg *)buf;
 
 	do {
-		ret = tracecmd_msg_recv(ifd, msg);
+		ret = tracecmd_msg_recv_wait(ifd, msg);
 		if (ret < 0) {
-			warning("reading client");
+			if (ret == -ETIMEDOUT)
+				warning("Connection timed out\n");
+			else
+				warning("reading client");
 			return ret;
 		}
 

From: Yoshihiro YUNOMAE <[email protected]>

Add the virt-server mode for a virtualization environment based on the listen mode. This mode works as a client/server mode over not TCP/UDP but virtio-serial channel. Since the troughput of trace-data can be huge, traditional IP network easily gets higher overhead. Using virtio-serial can reduce overhead because it can skip guest/host TCP/IP network stack.

virt-server uses two kinds of virtio-serial I/Fs:
(1) agent-ctl-path(UNIX domain socket)
=> control path of an agent trace-cmd each guest
(2) trace-path-cpuX(named pipe)
=> trace data path each vcpu

Those I/Fs must be defined as below paths:
(1) /tmp/trace-cmd/virt/agent-ctl-path
(2) /tmp/trace-cmd/virt/<guest domain>/trace-path-cpuX

If we run virt-server, agent-ctl-path I/F is automatically created because virt-server operates as a server mode of UNIX domain socket. However, trace-path-cpuX is not automatically created because we need to separate trace data for each guests.

Over the virtio-serial, V2 protocol is slightly changed since the server can not notice when the client connects. The detail is described in Documentation/Protocol.txt.

NOTE:
This feature requests to disable(or make permissive) selinux since qemu has to open a (non-registered) unix domain socket.

<How to set up>
1. Run virt-server on a host before booting guests

   # trace-cmd virt-server

2. Make guest domain directory

   # mkdir -p /tmp/trace-cmd/virt/<domain>
   # chmod 710 /tmp/trace-cmd/virt/<domain>
   # chgrp qemu /tmp/trace-cmd/virt/<domain>

3. Make FIFO on the host

   # mkfifo /tmp/trace-cmd/virt/<domain>/trace-path-cpu{0,1,...,X}.{in,out}

4. Set up virtio-serial pipes of the guest on the host Add the following tags to domain XML files.

   # virsh edit <domain>
   <channel type='unix'>
      <source mode='connect' path='/tmp/trace-cmd/virt/agent-ctl-path'/>
      <target type='virtio' name='agent-ctl-path'/>
   </channel>
   <channel type='pipe'>
      <source path='/tmp/trace-cmd/virt/<domain>/trace-path-cpu0'/>
      <target type='virtio' name='trace-path-cpu0'/>
   </channel>
   ... (cpu1, cpu2, ...)

5. Boot the guest

   # virsh start <domain>

6. Check I/F of virtio-serial on the guest

   # ls /dev/virtio-ports
     ...
     agent-ctl-path
     ...
     trace-path-cpu0
     ...

Next, the user will run trace-cmd with record --virt options or other options for virtualization on the guest.

This patch adds only minimum features of virt-server as follows:
<Features>
- virt-server subcommand
- Create I/F directory(/tmp/trace-cmd/virt/)
- Use named pipe I/Fs of virtio-serial for trace data paths
- Use UNIX domain socket for connecting clients on guests
- Use splice(2) for collecting trace data of guests

<Restrictions>
- libvirt is required for finding guest domain name
- User must setup fifos by hand
- Do not support hotplug VCPUs
- Interface directory is fixed
- SELinux should be disabled

---
Changes in V5: Change patch description
Update protocol document
Changes in V4: Fix some typos and cleanup
Changes in V3: Change _nw/_NW to _net/_NET

---
 Documentation/Protocol.txt                |   44 +++
 Documentation/trace-cmd-virt-server.1.txt |   89 ++++++
 trace-cmd.c                               |    3 
 trace-cmd.h                               |    2 
 trace-listen.c                            |  467 ++++++++++++++++++++++++-----
 trace-msg.c                               |  105 ++++++-
 trace-recorder.c                          |   50 ++-
 trace-usage.c                             |   10 +
 8 files changed, 667 insertions(+), 103 deletions(-)
 create mode 100644 Documentation/trace-cmd-virt-server.1.txt

index 49f7766..52df89e 100644
@@ -6,6 +6,7 @@ Index
 1. What is the trace-cmd protocol?
 2. Trace-cmd Protocol V1 (Obsolete)
 3. Trace-cmd Protocol V2
+4. Trace-cmd Protocol V2 in virt-server mode
 
 
 1. What is the trace-cmd protocol?

@@ -117,3 +118,46 @@ or not by checking the first message from the client. If client
 sends a positive number, it should be a V1 protocol client.
 
 
+4. Trace-cmd Protocol V2 in virt-server mode
+============================================
+
+In the virt-server mode, trace-cmd uses a control channel and
+trace data channels of virtio-serial to transfar trace data.
+
+Since the virtio-serial channel is just a character device
+on the guest, the server can not notice when a client attaches
+to (means opens) the channel. Thus, the server waits for the
+connection message MSG_TCONNECT from the client on the control
+channel. The protocol flow is as follows;
+
+     <server>                 <client>
+      Open a control channel
+      wait for MSG_TCONNECT
+                              open a virtio-serial channel
+                              send MSG_TCONNECT
+      receive MSG_TCONNECT <----+
+      send MSG_RCONNECT
+            +---------------> receive MSG_RCONNECT
+                              check "tracecmd-V2"
+                              send MSG_TINIT with cpus, pagesize and options
+      receive MSG_TINIT <-------+
+      perse the parameters
+      send MSG_RINIT with port_array
+           +----------------> receive MSG_RINIT
+                              get port_array
+                              send meta data(MSG_SENDMETA)
+      receive MSG_SENDMETA <----+
+      record meta data
+                         (snip)
+                              send a message to finish sending meta data
+                                |                           (MSG_FINMETA)
+      receive MSG_FINMETA <-----+
+      read block
+     --- start sending trace data on child processes ---
+
+     --- When client finishes sending trace data ---
+                              send MSG_CLOSE
+      receive MSG_CLOSE <-------+
+                              close the virtio-serial channel
+
+

index 4c5b564..29a2bb8 100644
@@ -425,7 +425,8 @@ int main (int argc, char **argv)
 	} else if (strcmp(argv[1], "mem") == 0) {
 		trace_mem(argc, argv);
 		exit(0);
-	} else if (strcmp(argv[1], "listen") == 0) {
+	} else if (strcmp(argv[1], "listen") == 0 ||
+		   strcmp(argv[1], "virt-server") == 0) {
 		trace_listen(argc, argv);
 		exit(0);
 	} else if (strcmp(argv[1], "split") == 0) {

index 1261e23..a93920f 100644
@@ -257,6 +257,7 @@ struct tracecmd_recorder *tracecmd_create_recorder_maxkb(const char *file, int c
 struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *buffer);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer);
 struct tracecmd_recorder *tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags, const char *buffer, int maxkb);
+struct tracecmd_recorder *tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd);
 
 int tracecmd_start_recording(struct tracecmd_recorder *recorder, unsigned long sleep);
 void tracecmd_stop_recording(struct tracecmd_recorder *recorder);

@@ -270,6 +271,7 @@ int tracecmd_msg_finish_sending_metadata(int fd);
 void tracecmd_msg_send_close_msg(void);
 
 /* for server */
+int tracecmd_msg_set_connection(int fd, const char *domain);
 int tracecmd_msg_initial_setting(int fd, int *cpus, int *pagesize);
 int tracecmd_msg_send_port_array(int fd, int total_cpus, int *ports);
 int tracecmd_msg_collect_metadata(int ifd, int ofd);

index 520b14b..3d9b821 100644
@@ -212,6 +212,16 @@ static struct usage_help usage_help[] = {
 		"	   -l logfile to write messages to.\n"
 	},
 	{
+		"virt-server",
+		"listen on a virtio-serial for trace clients",
+		" %s virt-server [-o file][-d dir][-l logfile]\n"
+		"          Creates a socket to listen for clients.\n"
+		"          -D create it in daemon mode.\n"
+		"          -o file name to use for clients.\n"
+		"          -d diretory to store client files.\n"
+		"	   -l logfile to write messages to.\n"
+	},
+	{
 		"list",
 		"list the available events, plugins or options",
 		" %s list [-e [regex]][-t][-o][-f [regex]]\n"

From: Yoshihiro YUNOMAE <[email protected]>

Add --virt option for record mode for a virtualization environment. If we use this option on a guest, we can send trace data in low-overhead. This is because guests can send trace data to a host without copying the data by using splice(2).

The format is:

   trace-cmd record --virt -e sched*

<Note>
The client using virtio-serial does not wait for the connection message "tracecmd" from the server. The client sends the connection message MSG_TCONNECT first.

<Restriction>
This feature can use from kernel-3.6 which supports splice_read for ftrace and splice_write for virtio-serial.

---
Changes in V4: Rebase for current trace-cmd-v2.4
Add usage of --virt for record in trace-usage.c Divide tracecmd_msg_connect_to_server() into two functions
(tracecmd_msg_connect_to_server() and
tracecmd_msg_send_init_data_virt(fd))
Changes in V3: Change _nw/_NW to _net/_NET

---
 Documentation/trace-cmd-record.1.txt |   12 +++++
 trace-cmd.h                          |    4 +-
 trace-msg.c                          |   79 +++++++++++++++++++++++++++++++---
 trace-msg.h                          |    4 ++
 trace-record.c                       |   71 ++++++++++++++++++++++++++++---
 trace-usage.c                        |    3 +
 6 files changed, 160 insertions(+), 13 deletions(-)

index 2a368fe..7b9981a 100644
@@ -290,6 +290,14 @@ OPTIONS
     Have output go to stderr instead of stdout, but the output of the command
     executed will not be changed. This is useful if you want to monitor the
     output of the command being executed, but not see the output from trace-cmd.
+*--virt*::
+    This option is usded on a guest in a virtualization environment. If a host
+    is running "trace-cmd virt-server", this option is used to have the data
+    sent to the host with virtio-serial like *-N* option. (see also
+    trace-cmd-virt-server(1))
+
+    Note: This option is not supported with latency tracer plugins:
+      wakeup, wakeup_rt, irqsoff, preemptoff and preemptirqsoff
 
 EXAMPLES
 --------

@@ -390,7 +398,11 @@ SEE ALSO
 --------
 trace-cmd(1), trace-cmd-report(1), trace-cmd-start(1), trace-cmd-stop(1),
 trace-cmd-extract(1), trace-cmd-reset(1), trace-cmd-split(1),
+<<<<<<< current
 trace-cmd-list(1), trace-cmd-listen(1), trace-cmd-profile(1)
+=======
+trace-cmd-list(1), trace-cmd-listen(1), trace-cmd-virt-server(1)
+>>>>>>> patched
 
 AUTHOR
 ------

index a93920f..fbfe3bf 100644
@@ -265,7 +265,9 @@ void tracecmd_stat_cpu(struct trace_seq *s, int cpu);
 long tracecmd_flush_recording(struct tracecmd_recorder *recorder);
 
 /* for clients */
-int tracecmd_msg_send_init_data(int fd);
+int tracecmd_msg_connect_to_server(int fd);
+int tracecmd_msg_send_init_data_net(int fd);
+int tracecmd_msg_send_init_data_virt(int fd);
 int tracecmd_msg_metadata_send(int fd, const char *buf, int size);
 int tracecmd_msg_finish_sending_metadata(int fd);
 void tracecmd_msg_send_close_msg(void);

index b23e72b..502c1bf 100644
@@ -2,6 +2,9 @@
 #define _TRACE_MSG_H_
 
 #include <stdbool.h>
+#define VIRTIO_PORTS	"/dev/virtio-ports/"
+#define AGENT_CTL_PATH	VIRTIO_PORTS "agent-ctl-path"
+#define TRACE_PATH_CPU	VIRTIO_PORTS "trace-path-cpu%d"
 
 #define UDP_MAX_PACKET	(65536 - 20)
 #define V2_MAGIC	"677768\0"

@@ -17,6 +20,7 @@ extern int cpu_count;
 extern unsigned int page_size;
 extern int *client_ports;
 extern bool send_metadata;
+extern int *virt_sfds;
 
 /* for server */
 extern bool done;

index 3d9b821..23cb124 100644
@@ -19,7 +19,7 @@ static struct usage_help usage_help[] = {
 		" %s record [-v][-e event [-f filter]][-p plugin][-F][-d][-D][-o file] \\\n"
 		"           [-s usecs][-O option ][-l func][-g func][-n func] \\\n"
 		"           [-P pid][-N host:port][-t][-r prio][-b size][-B buf][command ...]\n"
-		"           [-m max][-C clock]\n"
+		"           [-m max][-C clock][--virt]\n"
 		"          -e run command with event enabled\n"
 		"          -f filter for previous -e event\n"
 		"          -R trigger for previous -e event\n"

@@ -51,6 +51,7 @@ static struct usage_help usage_help[] = {
 		"          --profile enable tracing options needed for report --profile\n"
 		"          --func-stack perform a stack trace for function tracer\n"
 		"             (use with caution)\n"
+		"	   --virt to connect to virt-server\n"
 	},
 	{
 		"start",

From: Yoshihiro YUNOMAE <[email protected]>

Add --dom option which makes a domain directory to virt-server. When a user already knows domain name of a guest before running virt-server, trace-cmd should automatically set up I/Fs of the guest. By adding --dom option, trace-cmd creates a domain directory with 0710 and qemu group.

This patch adds additional options for --dom as follows:

-m <permission>
This option changes the permission of domain directory. If you don't use this option, the default permission is 0710.

-g <group>
This option changes group of domain directory. If you don't use this option, the default group is qemu.

-c <cpu>
This option creates trace data I/Fs(trace-path-cpu*.{in,out}) for each CPU of 'domain'. If you don't use this option, those files are not created.

Here, an example you use this option is written as follows:

- trace-cmd creates a guest1 directory with trace data I/Fs of 2 CPUs.

   # trace-cmd virt-server --dom guest1 -c 2

- trace-cmd creates guest2 and guest3 directories

   # trace-cmd virt-server --dom guest2 -c 3 --dom guest3 -c 1

---
Changes in V5: Update document.
Changes in V4: Introduce parse_args_virt()
Add usage of virt-server in trace-usage.c

---
 Documentation/trace-cmd-virt-server.1.txt |   58 ++++++++---
 trace-listen.c                            |  151 ++++++++++++++++++++++++++---
 trace-usage.c                             |    5 +
 3 files changed, 179 insertions(+), 35 deletions(-)

index b775745..19f2b23 100644
@@ -34,40 +34,64 @@ OPTIONS
 *-l* 'filename'::
     This option writes the output messages to a log file instead of standard output.
 
+*--dom* 'domain'::
+    This option makes a directory for the 'domain'. You can use additional options
+    *-m*, *-g*, *-c* after this option for the 'domain'. If you don't use these
+    additional options, the directory is made as 0710 and qemu group and
+    trace data I/Fs(trace-path-cpu*.{in,out}) are not created.
+
+*-m* 'permission'::
+    This option changes the permission of 'domain' directory. If you don't use
+    this option, the default permission is 0710.
+
+*-g* 'group'::
+    This option changes group of 'domain' directory. If you don't use this option,
+    the default group is qemu.
+
+*-c* 'cpu'::
+    This option creates trace data I/Fs(trace-path-cpu*.{in,out}) for each CPU
+    of 'domain'. If you don't use this option, those files are not created.
+
 SETTING
 -------
 Here, an example is written as follows:
 
-1. Run virt-server on a host
-   # trace-cmd virt-server
-
-2. Make guest domain directory
-   # mkdir -p /tmp/trace-cmd/virt/<DOMAIN>
-   # chmod 710 /tmp/trace-cmd/virt/<DOMAIN>
-   # chgrp qemu /tmp/trace-cmd/virt/<DOMAIN>
-
-3. Make FIFO on the host
-   # mkfifo /tmp/trace-cmd/virt/<DOMAIN>/trace-path-cpu{0,1,...,X}.{in,out}
+1. Run virt-server with initializing guest interfaces on a host
+   # trace-cmd virt-server --dom "GUEST" -c 2
 
-4. Set up of virtio-serial pipe of a guest on the host
+2. Set up of virtio-serial pipe of GUEST on the host
    Add the following tags to domain XML files.
-   # virsh edit <guest domain>
+   # virsh edit "GUEST"
    <channel type='unix'>
       <source mode='connect' path='/tmp/trace-cmd/virt/agent-ctl-path'/>
       <target type='virtio' name='agent-ctl-path'/>
    </channel>
    <channel type='pipe'>
-      <source path='/tmp/trace-cmd/virt/<DOMAIN>/trace-path-cpu0'/>
+      <source path='/tmp/trace-cmd/virt/guest1/trace-path-cpu0'/>
       <target type='virtio' name='trace-path-cpu0'/>
    </channel>
-   ... (cpu1, cpu2, ...)
+   <channel type='pipe'>
+      <source path='/tmp/trace-cmd/virt/guest1/trace-path-cpu1'/>
+      <target type='virtio' name='trace-path-cpu1'/>
+   </channel>
 
-5. Boot the guest
-   # virsh start <DOMAIN>
+3. Boot the guest
+   # virsh start "GUEST"
 
-6. Run the guest's client(see trace-cmd-record(1) with the *--virt* option)
+4. Run the guest1's client(see trace-cmd-record(1) with the *--virt* option)
    # trace-cmd record -e sched* --virt
 
+If you want to boot another guest sends trace-data via virtio-serial,
+you will manually make the guest domain directory and trace data I/Fs.
+
+- Make guest domain directory on the host
+   # mkdir -p /tmp/trace-cmd/virt/<DOMAIN>
+   # chmod 710 /tmp/trace-cmd/virt/<DOMAIN>
+   # chgrp qemu /tmp/trace-cmd/virt/<DOMAIN>
+
+- Make FIFO on the host
+   # mkfifo /tmp/trace-cmd/virt/<DOMAIN>/trace-path-cpu{0,1,...,X}.{in,out}
+
 SEE ALSO
 --------
 trace-cmd(1), trace-cmd-record(1), trace-cmd-report(1), trace-cmd-start(1),

index 23cb124..caba1f9 100644
@@ -216,11 +216,16 @@ static struct usage_help usage_help[] = {
 		"virt-server",
 		"listen on a virtio-serial for trace clients",
 		" %s virt-server [-o file][-d dir][-l logfile]\n"
+		"                [--dom domain [-m permisson] [-g group] [-c cpu]]\n"
 		"          Creates a socket to listen for clients.\n"
 		"          -D create it in daemon mode.\n"
 		"          -o file name to use for clients.\n"
 		"          -d diretory to store client files.\n"
 		"	   -l logfile to write messages to.\n"
+		"          --dom create domain direcroty in /tmp/trace-cmd/virt and folling directory permissions/group names and FIFO files will be changed here\n"
+		"	   -m changes the permission of domain directory.\n"
+		"	   -g changes group of domain directory.\n"
+		"	   -c creates trace data I/F(trace-path-cpu*.{in, out} files) in domain directory.\n"
 	},
 	{
 		"list",

From: Yoshihiro YUNOMAE <[email protected]>

Use pid instead of domain name if libvirt is not found. This also fix the trace fifo lookup routine to find it from /proc/<PID>/fd.
With this change, we can use trace-cmd virt-server for directly running qemu virtual machines.

---
 trace-listen.c |   47 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 13 deletions(-)

index f6d53d2..f40b3a5 100644
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <signal.h>
 #include <errno.h>
+#include <fnmatch.h>
 
 #include "trace-local.h"
 #include "trace-msg.h"

@@ -396,20 +397,41 @@ static int open_udp(const char *node, const char *port, int *pid,
 #define TRACE_PATH_DOMAIN_CPU_O	VIRT_DOMAIN_DIR "trace-path-cpu%d.out"
 #define TRACE_PATH_DOMAIN_CPU_I	VIRT_DOMAIN_DIR "trace-path-cpu%d.in"
 
+#define TRACE_PATH_PATTERN	VIRT_DIR "*/trace-path-cpu%d.out"
+
 static int open_virtio_serial_pipe(int *pid, int cpu, int pagesize,
 				   const char *domain, int virtpid)
 {
+	char path[PATH_MAX];
 	char buf[PATH_MAX];
-	int fd;
-
-	snprintf(buf, PATH_MAX, TRACE_PATH_DOMAIN_CPU_O, domain, cpu);
-	fd = open(buf, O_RDONLY | O_NONBLOCK);
-	if (fd < 0) {
-		warning("open %s", buf);
-		return fd;
+	int fd = -ENOENT;
+	DIR *dir;
+	struct dirent *ent;
+	struct stat st;
+
+	snprintf(path, PATH_MAX, "/proc/%d/fd", virtpid);
+	dir = opendir(path);
+	if (!dir)
+		return -errno;
+	while ((ent = readdir(dir)) != NULL) {
+		snprintf(path, PATH_MAX, "/proc/%d/fd/%s",
+			 virtpid, ent->d_name);
+		if (readlink(path, buf, PATH_MAX) < 0)
+			continue;
+		snprintf(path, PATH_MAX, TRACE_PATH_PATTERN, cpu);
+		/* Find the pipe which matchs pattern */
+		if ((fnmatch(path, buf, FNM_PATHNAME) == 0) &&
+		    (stat(buf, &st) == 0) && (S_ISFIFO(st.st_mode))) {
+			fd = open(buf, O_RDONLY | O_NONBLOCK);
+			break;
+		}
 	}
+	closedir(dir);
 
-	fork_virt_reader(fd, pid, cpu, pagesize, domain, virtpid);
+	if (fd < 0)
+		warning("open %s: %d", buf, fd);
+	else
+		fork_virt_reader(fd, pid, cpu, pagesize, domain, virtpid);
 
 	return fd;
 }

@@ -762,6 +784,7 @@ static int do_fork(int cfd)
 	return 0;
 }
 
+/* Get client (VM) pid from unix domain socket */
 static int get_virtpid(int cfd)
 {
 	struct ucred cr;

@@ -790,11 +813,8 @@ static char *get_guest_domain_from_pid(int pid)
 	int fd;
 
 	dir = opendir(LIBVIRT_DOMAIN_PATH);
-	if (!dir) {
-		if (errno == ENOENT)
-			warning("Only support for using libvirt");
+	if (!dir)
 		return NULL;
-	}
 
 	for (dirent = readdir(dir); dirent != NULL; dirent = readdir(dir)) {
 		snprintf(file_name, NAME_MAX, LIBVIRT_DOMAIN_PATH"%s",

@@ -837,7 +857,8 @@ static int do_connection(int cfd, struct sockaddr *peer_addr,
 
 		domain = get_guest_domain_from_pid(virtpid);
 		if (!domain)
-			return -1;
+			if (asprintf(&domain, "%d", virtpid) < 0)
+				return -ENOMEM;
 	}
 
 	ret = do_fork(cfd);