Skip to contentSkip to conversation

Integrating qemu to Linux Perf

PATCH In review vandersonmr 3 participants 6 messages Aug 30, 2019 – Oct 2, 2019

lore
Share & tools
lore
#0 lore raw

v2 cover Integrating qemu to Linux Perf

This patch is part of Google Summer of Code (GSoC) 2019. More about the project can be found in:
https://wiki.qemu.org/Internships/ProjectIdeas/TCGCodeQuality

This adds --perf command-line option to dump Linux Perf jitdump files. These files are used to enhant Perf report and to be able to analyze and dump JITed code with perf.

Example of use:
perf record -k 1 qemu-x86_64 -perf ./a.out
perf inject -j -i perf.data -o perf.data.jitted
perf report -i perf.data.jitted

vandersonmr (2):
accel/tcg: adding integration with linux perf
tb-stats: adding TBStatistics info into perf dump

 accel/tcg/Makefile.objs      |   1 +
 accel/tcg/perf/Makefile.objs |   1 +
 accel/tcg/perf/jitdump.c     | 206 +++++++++++++++++++++++++++++++++++
 accel/tcg/perf/jitdump.h     |  36 ++++++
 accel/tcg/translate-all.c    |  14 +++
 include/qemu-common.h        |   3 +
 linux-user/main.c            |   7 ++
 os-posix.c                   |   3 +
 qemu-options.hx              |  11 ++
 9 files changed, 282 insertions(+)
 create mode 100644 accel/tcg/perf/Makefile.objs
 create mode 100644 accel/tcg/perf/jitdump.c
 create mode 100644 accel/tcg/perf/jitdump.h
Signature · 1 line
2.22.0
#1 lore raw

PATCH v2 1/2 accel/tcg: adding integration with linux perf

This commit adds support to Linux Perf in order
to be able to analyze qemu jitted code and
also to able to see the TBs PC in it.

When using "-perf" qemu creates a jitdump file in
the current working directory. The file format
specification can be found in:
https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jitdump-specification.tx

Example of use:
perf record -k 1 qemu-x86_64 -perf ./a.out
perf inject -j -i perf.data -o perf.data.jitted
perf report -i perf.data.jitted

---
 accel/tcg/Makefile.objs      |   1 +
 accel/tcg/perf/Makefile.objs |   1 +
 accel/tcg/perf/jitdump.c     | 192 +++++++++++++++++++++++++++++++++++
 accel/tcg/perf/jitdump.h     |  36 +++++++
 accel/tcg/translate-all.c    |  14 +++
 include/qemu-common.h        |   3 +
 linux-user/main.c            |   7 ++
 os-posix.c                   |   3 +
 qemu-options.hx              |  11 ++
 9 files changed, 268 insertions(+)
 create mode 100644 accel/tcg/perf/Makefile.objs
 create mode 100644 accel/tcg/perf/jitdump.c
 create mode 100644 accel/tcg/perf/jitdump.h
accel/tcg/Makefile.objs +1 -0
index d381a02f34..f393a7438f 100644
@@ -3,6 +3,7 @@ obj-$(CONFIG_SOFTMMU) += cputlb.o
 obj-y += tcg-runtime.o tcg-runtime-gvec.o
 obj-y += cpu-exec.o cpu-exec-common.o translate-all.o
 obj-y += translator.o
+obj-y += perf/
 
 obj-$(CONFIG_USER_ONLY) += user-exec.o
 obj-$(call lnot,$(CONFIG_SOFTMMU)) += user-exec-stub.o
accel/tcg/perf/Makefile.objs new file +1 -0
new file mode 100644
index 0000000000..ca9abb4f48
@@ -0,0 +1 @@
+obj-$(CONFIG_LINUX) += jitdump.o
accel/tcg/perf/jitdump.c new file +192 -0

192 lines not on this page · raw message

accel/tcg/perf/jitdump.h new file +36 -0
new file mode 100644
index 0000000000..5d6df3ec91
@@ -0,0 +1,36 @@
+/*
+ * QEMU Linux Perf Support
+ *
+ * Copyright (c) 2019 Vanderson M. do Rosario ([email protected])
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef JITDUMP_H
+#define JITDUMP_H
+
+#include "exec/exec-all.h"
+
+void start_jitdump_file(void);
+
+void append_load_in_jitdump_file(TranslationBlock *tb);
+void append_move_in_jitdump_file(TranslationBlock *tb);
+
+void close_jitdump_file(void);
+
+#endif
accel/tcg/translate-all.c +14 -0
index 5d1e08b169..9b30d96af5 100644
@@ -58,6 +58,10 @@
 #include "sysemu/cpus.h"
 #include "sysemu/tcg.h"
 
+#ifdef __linux__
+#include "perf/jitdump.h"
+#endif
+
 /* #define DEBUG_TB_INVALIDATE */
 /* #define DEBUG_TB_FLUSH */
 /* make various TB consistency checks */
@@ -1148,6 +1152,11 @@ void tcg_exec_init(unsigned long tb_size)
     cpu_gen_init();
     page_init();
     tb_htable_init();
+#ifdef __linux__
+    if (jitdump_enabled()) {
+        start_jitdump_file();
+    }
+#endif
     code_gen_alloc(tb_size);
 #if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and
@@ -1877,6 +1886,11 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
         return existing_tb;
     }
     tcg_tb_insert(tb);
+#ifdef __linux__
+    if (jitdump_enabled()) {
+        append_load_in_jitdump_file(tb);
+    }
+#endif
     return tb;
 }
 
include/qemu-common.h +3 -0
index 0235cd3b91..7ab796e4a3 100644
@@ -130,4 +130,7 @@ void page_size_init(void);
  * returned. */
 bool dump_in_progress(void);
 
+void enable_jitdump(void);
+bool jitdump_enabled(void);
+
 #endif
linux-user/main.c +7 -0
index 47917bbb20..f6618dd060 100644
@@ -379,6 +379,11 @@ static void handle_arg_strace(const char *arg)
     do_strace = 1;
 }
 
+static void handle_arg_perf(const char *arg)
+{
+    enable_jitdump();
+}
+
 static void handle_arg_version(const char *arg)
 {
     printf("qemu-" TARGET_NAME " version " QEMU_FULL_VERSION
@@ -444,6 +449,8 @@ static const struct qemu_argument arg_table[] = {
      "",           "Seed for pseudo-random number generator"},
     {"trace",      "QEMU_TRACE",       true,  handle_arg_trace,
      "",           "[[enable=]<pattern>][,events=<file>][,file=<file>]"},
+    {"perf",      "QEMU_PERF",         false, handle_arg_perf,
+     "",           "dump jitdump files to help linux perf JIT code visualization"},
     {"version",    "QEMU_VERSION",     false, handle_arg_version,
      "",           "display version information and exit"},
     {NULL, NULL, false, NULL, NULL, NULL}
os-posix.c +3 -0
index 86cffd2c7d..2ddb1e6393 100644
@@ -191,6 +191,9 @@ int os_parse_cmd_args(int index, const char *optarg)
     case QEMU_OPTION_enablefips:
         fips_set_state(true);
         break;
+    case QEMU_OPTION_perf:
+        enable_jitdump();
+        break;
 #endif
     default:
         return -1;
qemu-options.hx +11 -0
index ea0638e92d..24ebfb910c 100644
@@ -4156,6 +4156,17 @@ STEXI
 Enable FIPS 140-2 compliance mode.
 ETEXI
 
+#ifdef __linux__
+DEF("perf", 0, QEMU_OPTION_perf,
+    "-perf  dump jitdump files to help linux perf JIT code visualization\n",
+    QEMU_ARCH_ALL)
+#endif
+STEXI
+@item -perf
+@findex -perf
+Dumps jitdump files to help linux perf JIT code visualization
+ETEXI
+
 HXCOMM Deprecated by -machine accel=tcg property
 DEF("no-kvm", 0, QEMU_OPTION_no_kvm, "", QEMU_ARCH_I386)
 
Signature · 1 line
2.22.0
#2 lore raw

Re: [Qemu-devel] [PATCH v2 1/2] accel/tcg: adding integration with linux perf

Quoted 8 lines · vandersonmr

This commit adds support to Linux Perf in order to be able to analyze qemu jitted code and also to able to see the TBs PC in it.

When using "-perf" qemu creates a jitdump file in the current working directory. The file format specification can be found in:
https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jitdump-specification.tx

Oops, the link is broken: .txt

accel/tcg/perf/jitdump.c line 65

Quoted 3 lines
accel/tcg/perf/jitdump.c new file +3 -0
+struct jr_code_close {
+    struct jr_prefix p;
+};

Unused?

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+struct jr_code_move {

Unused?

accel/tcg/perf/jitdump.c line 102

Quoted 11 lines
accel/tcg/perf/jitdump.c new file +11 -0
+static uint32_t get_e_machine(void)
+{
+    uint32_t e_machine = EM_NONE;
+    Elf64_Ehdr elf_header;
+    FILE *exe = fopen("/proc/self/exe", "r");
+
+    if (exe == NULL) {
+        return e_machine;
+    }
+
+    if (fread(&elf_header, sizeof(Elf64_Ehdr), 1, exe) != 1) {

What if this is a 32-bit binary because QEMU was built for a 32-bit host?

accel/tcg/perf/jitdump.c line 115

Quoted 13 lines
accel/tcg/perf/jitdump.c new file +13 -0
+        goto end;
+    }
+
+    e_machine = elf_header.e_machine;
+
+end:
+    fclose(exe);
+    return e_machine;
+}
+
+void start_jitdump_file(void)
+{
+    gchar *dumpfile_name = g_strdup_printf("./jit-%d.dump", getpid());

You can now use g_autofree:

  g_autofree gchar *dumpfile_name = g_strdup_printf(...);

and then the explicit g_free() isn't necessary anymore (and the memory leak in the mmap error case is also solved).

Quoted 13 lines
accel/tcg/perf/jitdump.c new file +13 -0
+void append_load_in_jitdump_file(TranslationBlock *tb)
+{
+    gchar *func_name = g_strdup_printf("TB virt:0x"TARGET_FMT_lx, tb->pc);
+
+    struct jr_code_load load_event;
+    load_event.p.id = JIT_CODE_LOAD;
+    load_event.p.total_size =
+        sizeof(struct jr_code_load) + strlen(func_name) + 1 + tb->tc.size;
+    load_event.p.timestamp = get_timestamp();
+    load_event.pid = getpid();
+    load_event.tid = syscall(SYS_gettid);
+    load_event.vma = tb->pc;
+    load_event.code_addr = (uint64_t) tb->tc.ptr;

accel/tcg/perf/jitdump.c line 172

Quoted 10 lines
accel/tcg/perf/jitdump.c new file +10 -0
+    load_event.code_size = tb->tc.size;
+    load_event.code_index = tb->pc;
+
+    fwrite(&load_event, sizeof(struct jr_code_load), 1, dumpfile);
+    fwrite(func_name, strlen(func_name) + 1, 1, dumpfile);
+    fwrite(tb->tc.ptr, tb->tc.size, 1, dumpfile);
+
+    g_free(func_name);
+    fflush(dumpfile);
+}

I didn't see a reply to my question on the previous patch series:

"append_load_in_jitdump_file() calls fwrite() multiple times. What guarantees they will not be interleaved when multiple threads call this function?"

Does TCG ever throw away TBs and is it necessary to record this in the file so perf knows about it?

#3 lore raw

Re: [Qemu-devel] [PATCH v2 1/2] accel/tcg: adding integration with linux perf

Quoted 85 lines · Stefan Hajnoczi
Quoted 8 lines · vandersonmr

This commit adds support to Linux Perf in order to be able to analyze qemu jitted code and also to able to see the TBs PC in it.

When using "-perf" qemu creates a jitdump file in the current working directory. The file format specification can be found in:
https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jitdump-specification.tx

Oops, the link is broken: .txt

accel/tcg/perf/jitdump.c line 65

Quoted 3 lines
accel/tcg/perf/jitdump.c new file +3 -0
+struct jr_code_close {
+    struct jr_prefix p;
+};

Unused?

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+struct jr_code_move {

Unused?

accel/tcg/perf/jitdump.c line 102

Quoted 11 lines
accel/tcg/perf/jitdump.c new file +11 -0
+static uint32_t get_e_machine(void)
+{
+    uint32_t e_machine = EM_NONE;
+    Elf64_Ehdr elf_header;
+    FILE *exe = fopen("/proc/self/exe", "r");
+
+    if (exe == NULL) {
+        return e_machine;
+    }
+
+    if (fread(&elf_header, sizeof(Elf64_Ehdr), 1, exe) != 1) {

What if this is a 32-bit binary because QEMU was built for a 32-bit host?

accel/tcg/perf/jitdump.c line 115

Quoted 13 lines
accel/tcg/perf/jitdump.c new file +13 -0
+        goto end;
+    }
+
+    e_machine = elf_header.e_machine;
+
+end:
+    fclose(exe);
+    return e_machine;
+}
+
+void start_jitdump_file(void)
+{
+    gchar *dumpfile_name = g_strdup_printf("./jit-%d.dump", getpid());

You can now use g_autofree:

  g_autofree gchar *dumpfile_name = g_strdup_printf(...);

and then the explicit g_free() isn't necessary anymore (and the memory leak in the mmap error case is also solved).

Quoted 13 lines
accel/tcg/perf/jitdump.c new file +13 -0
+void append_load_in_jitdump_file(TranslationBlock *tb)
+{
+    gchar *func_name = g_strdup_printf("TB virt:0x"TARGET_FMT_lx, tb->pc);
+
+    struct jr_code_load load_event;
+    load_event.p.id = JIT_CODE_LOAD;
+    load_event.p.total_size =
+        sizeof(struct jr_code_load) + strlen(func_name) + 1 + tb->tc.size;
+    load_event.p.timestamp = get_timestamp();
+    load_event.pid = getpid();
+    load_event.tid = syscall(SYS_gettid);
+    load_event.vma = tb->pc;
+    load_event.code_addr = (uint64_t) tb->tc.ptr;

accel/tcg/perf/jitdump.c line 172

Quoted 10 lines
accel/tcg/perf/jitdump.c new file +10 -0
+    load_event.code_size = tb->tc.size;
+    load_event.code_index = tb->pc;
+
+    fwrite(&load_event, sizeof(struct jr_code_load), 1, dumpfile);
+    fwrite(func_name, strlen(func_name) + 1, 1, dumpfile);
+    fwrite(tb->tc.ptr, tb->tc.size, 1, dumpfile);
+
+    g_free(func_name);
+    fflush(dumpfile);
+}

I didn't see a reply to my question on the previous patch series:

Quoted 3 lines

"append_load_in_jitdump_file() calls fwrite() multiple times. What guarantees they will not be interleaved when multiple threads call this function?"

Ahh that was exactly the problem I was debugging. Fixed with a lock now.

Quoted 2 lines

Does TCG ever throw away TBs and is it necessary to record this in the file so perf knows about it?

Yes - I suspect that's what the currently unused headers are for.

--
Alex Bennée

#4 lore raw

PATCH v2 2/2 tb-stats: adding TBStatistics info into perf dump

In reply to vandersonmr

Adding TBStatistics information to linux perf TB's symbol names.

This commit depends on the following PATCH:
[PATCH v5 00/10] Measure Tiny Code Generation Quality

---
 accel/tcg/perf/jitdump.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)
accel/tcg/perf/jitdump.c +15 -1
index 6fb464039d..6a7ca4cf06 100644
@@ -21,6 +21,7 @@
 #include "disas/disas.h"
 #include "jitdump.h"
 #include "qemu-common.h"
+#include "exec/tb-stats.h"
 
 struct jitheader {
     uint32_t magic;     /* characters "jItD" */
@@ -149,7 +150,20 @@ void start_jitdump_file(void)
 
 void append_load_in_jitdump_file(TranslationBlock *tb)
 {
-    gchar *func_name = g_strdup_printf("TB virt:0x"TARGET_FMT_lx, tb->pc);
+    gchar *func_name;
+    if (tb->tb_stats) {
+        TBStatistics *tbs = tb->tb_stats;
+        unsigned g = stat_per_translation(tbs, code.num_guest_inst);
+        unsigned ops = stat_per_translation(tbs, code.num_tcg_ops);
+        unsigned ops_opt = stat_per_translation(tbs, code.num_tcg_ops_opt);
+        unsigned spills = stat_per_translation(tbs, code.spills);
+
+        func_name = g_strdup_printf(func_name,
+                "TB virt:0x"TARGET_FMT_lx" (g:%u op:%u opt:%u spills:%d)",
+                tb->pc, g, ops, ops_opt, spills);
+    } else {
+        func_name = g_strdup_printf("TB virt:0x"TARGET_FMT_lx, tb->pc);
+    }
 
     struct jr_code_load load_event;
     load_event.p.id = JIT_CODE_LOAD;
Signature · 1 line
2.22.0
#5 lore raw
In reply to vandersonmr
Quoted 4 lines · vandersonmr

This patch is part of Google Summer of Code (GSoC) 2019. More about the project can be found in:
https://wiki.qemu.org/Internships/ProjectIdeas/TCGCodeQuality

Quoted 8 lines

This adds --perf command-line option to dump Linux Perf jitdump files. These files are used to enhant Perf report and to be able to analyze and dump JITed code with perf.

Example of use:
perf record -k 1 qemu-x86_64 -perf ./a.out
perf inject -j -i perf.data -o perf.data.jitted
perf report -i perf.data.jitted

Please include a changelog in the future so reviewers know what to look out for in this new revision. For example:

v2:
* Fix foo in bar() [Bob]

(where Bob is the reviewer who requested the change)

Quoted 22 lines

vandersonmr (2):
accel/tcg: adding integration with linux perf
tb-stats: adding TBStatistics info into perf dump

 accel/tcg/Makefile.objs      |   1 +
 accel/tcg/perf/Makefile.objs |   1 +
 accel/tcg/perf/jitdump.c     | 206 +++++++++++++++++++++++++++++++++++
 accel/tcg/perf/jitdump.h     |  36 ++++++
 accel/tcg/translate-all.c    |  14 +++
 include/qemu-common.h        |   3 +
 linux-user/main.c            |   7 ++
 os-posix.c                   |   3 +
 qemu-options.hx              |  11 ++
 9 files changed, 282 insertions(+)
 create mode 100644 accel/tcg/perf/Makefile.objs
 create mode 100644 accel/tcg/perf/jitdump.c
 create mode 100644 accel/tcg/perf/jitdump.h
Signature · 1 line
2.22.0