Skip to contentSkip to conversation

Tracing patches

GIT PULL In review Stefan Hajnoczi 3 participants 8 messages May 3, 2013 – May 8, 2013

lore
Share & tools
lore
#0 lore raw

This tracing pull request is long overdue for QEMU 1.5.

Eiichi Tsukata's ftrace backend makes it easy to correlate QEMU events with host kernel events. He also reports good performance.

Kazuya Saito's trace events make it easier to observe the KVM run loop.

The following changes since commit 8ca27ce2e1150486ea2db4116a03706b28294f16:

Merge remote-tracking branch 'afaerber/qom-cpu' into staging (2013-05-02 10:57:01 -0500)

are available in the git repository at:

git://github.com/stefanha/qemu.git tracing

for you to fetch changes up to e64dd5efb2c6d522a3bc9d096cd49a4e53f0ae10:

trace: document ftrace backend (2013-05-03 13:58:09 +0200)

----------------------------------------------------------------
Eiichi Tsukata (2):
trace: Add ftrace tracing backend
trace: document ftrace backend

Kazuya Saito (2):
kvm-all: add kvm_ioctl, kvm_vm_ioctl, kvm_vcpu_ioctl tracepoints
kvm-all: add kvm_run_exit tracepoint

 configure                           |   8 +++
 docs/tracing.txt                    |  16 ++++++
 kvm-all.c                           |   5 ++
 scripts/tracetool/backend/ftrace.py |  54 +++++++++++++++++++
 trace-events                        |   7 +++
 trace/Makefile.objs                 |   1 +
 trace/ftrace.c                      | 102 ++++++++++++++++++++++++++++++++++++
 trace/ftrace.h                      |  10 ++++
 8 files changed, 203 insertions(+)
 create mode 100644 scripts/tracetool/backend/ftrace.py
 create mode 100644 trace/ftrace.c
 create mode 100644 trace/ftrace.h
Signature · 1 line
1.8.1.4
#1 lore raw

PATCH 1/4 kvm-all: add kvm_ioctl, kvm_vm_ioctl, kvm_vcpu_ioctl tracepoints

From: Kazuya Saito <[email protected]>

This patch adds tracepoints at ioctl to kvm. Tracing these ioctl is useful for clarification whether the cause of troubles is qemu or kvm.

---
 kvm-all.c    | 4 ++++
 trace-events | 5 +++++
 2 files changed, 9 insertions(+)
kvm-all.c +4 -0
index f6c0f4a..4f73b98 100644
@@ -33,6 +33,7 @@
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "qemu/event_notifier.h"
+#include "trace.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
@@ -1687,6 +1688,7 @@ int kvm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_ioctl(type, arg);
     ret = ioctl(s->fd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1704,6 +1706,7 @@ int kvm_vm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vm_ioctl(type, arg);
     ret = ioctl(s->vmfd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1721,6 +1724,7 @@ int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
     ret = ioctl(cpu->kvm_fd, type, arg);
     if (ret == -1) {
         ret = -errno;
trace-events +5 -0
index 55e80be..d5bc7a5 100644
@@ -1153,3 +1153,8 @@ virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev
 
 # migration.c
 migrate_set_state(int new_state) "new state %d"
+
+# kvm-all.c
+kvm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"
Signature · 1 line
1.8.1.4
#2 lore raw

Re: [Qemu-devel] [PATCH 1/4] kvm-all: add kvm_ioctl, kvm_vm_ioctl, kvm_vcpu_ioctl tracepoints

Am 03.05.2013 14:01, schrieb Stefan Hajnoczi:

Quoted 54 lines

From: Kazuya Saito <[email protected]>

This patch adds tracepoints at ioctl to kvm. Tracing these ioctl is useful for clarification whether the cause of troubles is qemu or kvm.

---
 kvm-all.c    | 4 ++++
 trace-events | 5 +++++
 2 files changed, 9 insertions(+)
kvm-all.c +4 -0
index f6c0f4a..4f73b98 100644
@@ -33,6 +33,7 @@
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "qemu/event_notifier.h"
+#include "trace.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
@@ -1687,6 +1688,7 @@ int kvm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_ioctl(type, arg);
     ret = ioctl(s->fd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1704,6 +1706,7 @@ int kvm_vm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vm_ioctl(type, arg);
     ret = ioctl(s->vmfd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1721,6 +1724,7 @@ int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
     ret = ioctl(cpu->kvm_fd, type, arg);
     if (ret == -1) {
         ret = -errno;
trace-events +0 -0
index 55e80be..d5bc7a5 100644

trace-events virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev · line 1160

Quoted 8 lines
trace-events +5 -0
@@ -1153,3 +1153,8 @@ virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev
 
 # migration.c
 migrate_set_state(int new_state) "new state %d"
+
+# kvm-all.c
+kvm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"

Sorry that I'm just seeing this patch now (wasn't CC'ed), but I wonder whether cpu_index is the best thing to trace here? Can we still change trace event API or would we have to nack/change now?

CC'ing Igor since he just introduced a cpu_get_arch_id() and there's also a kvm_arch_vcpu_id() introduced earlier by Eduardo.

Regards,
Andreas

Signature · 2 lines
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg, Germany
GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer; HRB 16746 AG Nürnberg
#3 lore raw

Re: [Qemu-devel] [PATCH 1/4] kvm-all: add kvm_ioctl, kvm_vm_ioctl, kvm_vcpu_ioctl tracepoints

Quoted 68 lines · Andreas Färber

Am 03.05.2013 14:01, schrieb Stefan Hajnoczi:

Quoted 54 lines

From: Kazuya Saito <[email protected]>

This patch adds tracepoints at ioctl to kvm. Tracing these ioctl is useful for clarification whether the cause of troubles is qemu or kvm.

---
 kvm-all.c    | 4 ++++
 trace-events | 5 +++++
 2 files changed, 9 insertions(+)
kvm-all.c +4 -0
index f6c0f4a..4f73b98 100644
@@ -33,6 +33,7 @@
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "qemu/event_notifier.h"
+#include "trace.h"
 
 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
@@ -1687,6 +1688,7 @@ int kvm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_ioctl(type, arg);
     ret = ioctl(s->fd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1704,6 +1706,7 @@ int kvm_vm_ioctl(KVMState *s, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vm_ioctl(type, arg);
     ret = ioctl(s->vmfd, type, arg);
     if (ret == -1) {
         ret = -errno;
@@ -1721,6 +1724,7 @@ int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
     arg = va_arg(ap, void *);
     va_end(ap);
 
+    trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
     ret = ioctl(cpu->kvm_fd, type, arg);
     if (ret == -1) {
         ret = -errno;
trace-events +0 -0
index 55e80be..d5bc7a5 100644

trace-events virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev · line 1160

Quoted 8 lines
trace-events +5 -0
@@ -1153,3 +1153,8 @@ virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev
 
 # migration.c
 migrate_set_state(int new_state) "new state %d"
+
+# kvm-all.c
+kvm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"

Sorry that I'm just seeing this patch now (wasn't CC'ed), but I wonder whether cpu_index is the best thing to trace here? Can we still change trace event API or would we have to nack/change now?

Quoted 2 lines

CC'ing Igor since he just introduced a cpu_get_arch_id() and there's also a kvm_arch_vcpu_id() introduced earlier by Eduardo.

Being kvm_vcpu_ioctl() a very low-level KVM function, I believe the KVM VCPU "id" (the argument passed to KVM_CREATE_VCPU, that needs to be the APIC ID on x86, and is returned by kvm_arch_vcpu_id()) is the best CPU identifier to be included here.

cpu_index is the most ambiguous and least reliable CPU identifier we have today. I wouldn't use it in any new code.

Signature · 1 line
Eduardo
#4 lore raw

Re: [Qemu-devel] [PATCH 1/4] kvm-all: add kvm_ioctl, kvm_vm_ioctl, kvm_vcpu_ioctl tracepoints

In reply to Andreas Färber
Quoted 8 lines · Andreas Färber

trace-events virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *dev · line 1160

Quoted 4 lines
trace-events +4 -0
+# kvm-all.c
+kvm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"

Sorry that I'm just seeing this patch now (wasn't CC'ed), but I wonder whether cpu_index is the best thing to trace here? Can we still change trace event API or would we have to nack/change now?

Trace events are not stable. We can change them.

STefan

#5 lore raw

PATCH 2/4 kvm-all: add kvm_run_exit tracepoint

In reply to Stefan Hajnoczi

From: Kazuya Saito <[email protected]>

This patch enable us to know exit reason of KVM_RUN. It will help us know where the trouble is caused.

---
 kvm-all.c    | 1 +
 trace-events | 2 ++
 2 files changed, 3 insertions(+)
kvm-all.c +1 -0
index 4f73b98..3a31602 100644
@@ -1627,6 +1627,7 @@ int kvm_cpu_exec(CPUArchState *env)
             abort();
         }
 
+        trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
         switch (run->exit_reason) {
         case KVM_EXIT_IO:
             DPRINTF("handle_io\n");
trace-events +2 -0
index d5bc7a5..17d75ab 100644
@@ -1158,3 +1158,5 @@ migrate_set_state(int new_state) "new state %d"
 kvm_ioctl(int type, void *arg) "type %d, arg %p"
 kvm_vm_ioctl(int type, void *arg) "type %d, arg %p"
 kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type %d, arg %p"
+kvm_run_exit(int cpu_index, uint32_t reason) "cpu_index %d, reason %d"
+
Signature · 1 line
1.8.1.4
#6 lore raw

PATCH 3/4 trace: Add ftrace tracing backend

In reply to Stefan Hajnoczi

From: Eiichi Tsukata <[email protected]>

This patch adds a ftrace tracing backend which sends trace event to ftrace marker file. You can effectively compare qemu trace data and kernel(especially, kvm.ko when using KVM) trace data. The ftrace backend is restricted to Linux only.

To try out the ftrace backend:

 $ ./configure --trace-backend=ftrace
 $ make

if you use KVM, enable kvm events in ftrace:

 # sudo echo 1 > /sys/kernel/debug/tracing/events/kvm/enable

After running qemu by root user, you can get the trace:

 # cat /sys/kernel/debug/tracing/trace
---
 configure                           |   8 +++
 scripts/tracetool/backend/ftrace.py |  54 +++++++++++++++++++
 trace/Makefile.objs                 |   1 +
 trace/ftrace.c                      | 102 ++++++++++++++++++++++++++++++++++++
 trace/ftrace.h                      |  10 ++++
 5 files changed, 175 insertions(+)
 create mode 100644 scripts/tracetool/backend/ftrace.py
 create mode 100644 trace/ftrace.c
 create mode 100644 trace/ftrace.h
configure +8 -0
index c4d85ba..e818e8b 100755
@@ -4004,6 +4004,14 @@ if test "$trace_backend" = "dtrace"; then
     echo "CONFIG_TRACE_SYSTEMTAP=y" >> $config_host_mak
   fi
 fi
+if test "$trace_backend" = "ftrace"; then
+  if test "$linux" = "yes" ; then
+    echo "CONFIG_TRACE_FTRACE=y" >> $config_host_mak
+    trace_default=no
+  else
+    feature_not_found "ftrace(trace backend)"
+  fi
+fi
 echo "CONFIG_TRACE_FILE=$trace_file" >> $config_host_mak
 if test "$trace_default" = "yes"; then
   echo "CONFIG_TRACE_DEFAULT=y" >> $config_host_mak
scripts/tracetool/backend/ftrace.py new file +54 -0

54 lines not on this page · raw message

trace/Makefile.objs +1 -0
index a043072..3b88e49 100644
@@ -76,5 +76,6 @@ endif
 util-obj-$(CONFIG_TRACE_DEFAULT) += default.o
 util-obj-$(CONFIG_TRACE_SIMPLE) += simple.o
 util-obj-$(CONFIG_TRACE_STDERR) += stderr.o
+util-obj-$(CONFIG_TRACE_FTRACE) += ftrace.o
 util-obj-y += control.o
 util-obj-y += generated-tracers.o
trace/ftrace.c new file +102 -0

102 lines not on this page · raw message

trace/ftrace.h new file +10 -0
new file mode 100644
index 0000000..94cb8d5
@@ -0,0 +1,10 @@
+#ifndef TRACE_FTRACE_H
+#define TRACE_FTRACE_H
+
+#define MAX_TRACE_STRLEN 512
+#define _STR(x) #x
+#define STR(x) _STR(x)
+
+extern int trace_marker_fd;
+
+#endif /* ! TRACE_FTRACE_H */
Signature · 1 line
1.8.1.4
#7 lore raw

PATCH 4/4 trace: document ftrace backend

In reply to Stefan Hajnoczi

From: Eiichi Tsukata <[email protected]>

Add documentation of ftrace backend.

---
 docs/tracing.txt | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
docs/tracing.txt +16 -0
index cf53c17..60ff9c5 100644
@@ -175,6 +175,22 @@ unless you have specific needs for more advanced backends.
 The "simple" backend currently does not capture string arguments, it simply
 records the char* pointer value instead of the string that is pointed to.
 
+=== Ftrace ===
+
+The "ftrace" backend writes trace data to ftrace marker. This effectively
+sends trace events to ftrace ring buffer, and you can compare qemu trace
+data and kernel(especially kvm.ko when using KVM) trace data.
+
+if you use KVM, enable kvm events in ftrace:
+
+   # echo 1 > /sys/kernel/debug/tracing/events/kvm/enable
+
+After running qemu by root user, you can get the trace:
+
+   # cat /sys/kernel/debug/tracing/trace
+
+Restriction: "ftrace" backend is restricted to Linux only.
+
 ==== Monitor commands ====
 
 * trace-file on|off|flush|set <path>
Signature · 1 line
1.8.1.4