Skip to contentSkip to conversation

perf kvm: Add kvm stat support on riscv

PATCH Superseded Shenlin Liang 3 participants 8 messages Mar 28, 2024 – Apr 11, 2024

Subsystems (4)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover perf kvm: Add kvm stat support on riscv

'perf kvm stat report/record' generates a statistical analysis of KVM events and can be used to analyze guest exit reasons. This patch tries to add stat support on riscv.

Map the return value of trace_kvm_exit() to the specific cause of the exception, and export it to userspace.

It records on two available KVM tracepoints for riscv: "kvm:kvm_entry" and "kvm:kvm_exit", and reports statistical data which includes events handles time, samples, and so on.

Simple tests go below:

# ./perf kvm record -e "kvm:kvm_entry" -e "kvm:kvm_exit"
Lowering default frequency rate from 4000 to 2500.
Please consider tweaking /proc/sys/kernel/perf_event_max_sample_rate.
[ perf record: Woken up 18 times to write data ]
[ perf record: Captured and wrote 5.433 MB perf.data.guest (62519 samples) 
# ./perf kvm report
31K kvm:kvm_entry
31K kvm:kvm_exit
# ./perf kvm stat record -a
[ perf record: Woken up 3 times to write data ]
[ perf record: Captured and wrote 8.502 MB perf.data.guest (99338 samples) ]
# ./perf kvm stat report --event=vmexit
Event name                Samples   Sample%    Time (ns)     Time%   Max Time (ns)   Min Time (ns)  Mean Time (ns)
STORE_GUEST_PAGE_FAULT     26968     54.00%    2003031800    40.00%     3361400         27600          74274
LOAD_GUEST_PAGE_FAULT      17645     35.00%    1153338100    23.00%     2513400         30800          65363
VIRTUAL_INST_FAULT         1247      2.00%     340820800     6.00%      1190800         43300          273312
INST_GUEST_PAGE_FAULT      1128      2.00%     340645800     6.00%      2123200         30200          301990
SUPERVISOR_SYSCALL         1019      2.00%     245989900     4.00%      1851500         29300          241403
LOAD_ACCESS                986       1.00%     671556200     13.00%     4180200         100700         681091
INST_ACCESS                655       1.00%     170054800     3.00%      1808300         54600          259625
HYPERVISOR_SYSCALL         21        0.00%     4276400       0.00%      716500          116000         203638 

Shenlin Liang (2):
RISCV: KVM: add tracepoints for entry and exit events perf kvm/riscv: Port perf kvm stat to RISC-V

 arch/riscv/kvm/trace_riscv.h                  | 60 ++++++++++++++
 arch/riscv/kvm/vcpu.c                         |  7 ++
 tools/perf/arch/riscv/Makefile                |  1 +
 tools/perf/arch/riscv/util/Build              |  1 +
 tools/perf/arch/riscv/util/kvm-stat.c         | 78 +++++++++++++++++++
 .../arch/riscv/util/riscv_exception_types.h   | 41 ++++++++++
 6 files changed, 188 insertions(+)
 create mode 100644 arch/riscv/kvm/trace_riscv.h
 create mode 100644 tools/perf/arch/riscv/util/kvm-stat.c
 create mode 100644 tools/perf/arch/riscv/util/riscv_exception_types.h
Signature · 1 line
2.37.2
#1 lore raw

PATCH 1/2 RISCV: KVM: add tracepoints for entry and exit events

Like other architectures, RISCV KVM also needs to add these event tracepoints to count the number of times kvm guest entry/exit.

---
 arch/riscv/kvm/trace_riscv.h | 60 ++++++++++++++++++++++++++++++++++++
 arch/riscv/kvm/vcpu.c        |  7 +++++
 2 files changed, 67 insertions(+)
 create mode 100644 arch/riscv/kvm/trace_riscv.h
arch/riscv/kvm/trace_riscv.h new file +60 -0
new file mode 100644
index 000000000000..5848083c7a5e
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Tracepoints for RISC-V KVM
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *
+ */
+#if !defined(_TRACE_RSICV_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RSICV_KVM_H
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+
+TRACE_EVENT(kvm_entry,
+	TP_PROTO(struct kvm_vcpu *vcpu),
+	TP_ARGS(vcpu),
+
+	TP_STRUCT__entry(
+		__field(unsigned long, pc)
+	),
+
+	TP_fast_assign(
+		__entry->pc	= vcpu->arch.guest_context.sepc;
+	),
+
+	TP_printk("PC: 0x%016lx", __entry->pc)
+);
+
+TRACE_EVENT(kvm_exit,
+	TP_PROTO(struct kvm_vcpu *vcpu, unsigned long exit_reason,
+			unsigned long scause),
+	TP_ARGS(vcpu, exit_reason, scause),
+
+	TP_STRUCT__entry(
+		__field(unsigned long, pc)
+		__field(unsigned long, exit_reason)
+		__field(unsigned long, scause)
+	),
+
+	TP_fast_assign(
+		__entry->pc		= vcpu->arch.guest_context.sepc;
+		__entry->exit_reason	= exit_reason;
+		__entry->scause		= scause;
+	),
+
+	TP_printk("EXIT_REASON:0x%lx,PC: 0x%016lx,SCAUSE:0x%lx",
+			__entry->exit_reason, __entry->pc, __entry->scause)
+);
+
+#endif /* _TRACE_RSICV_KVM_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace_riscv
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
arch/riscv/kvm/vcpu.c +7 -0
index b5ca9f2e98ac..ed0932f0d514 100644
@@ -21,6 +21,9 @@
 #include <asm/cacheflush.h>
 #include <asm/kvm_vcpu_vector.h>
 
+#define CREATE_TRACE_POINTS
+#include "trace_riscv.h"
+
 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
 	KVM_GENERIC_VCPU_STATS(),
 	STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
@@ -782,6 +785,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		 */
 		kvm_riscv_local_tlb_sanitize(vcpu);
 
+		trace_kvm_entry(vcpu);
+
 		guest_timing_enter_irqoff();
 
 		kvm_riscv_vcpu_enter_exit(vcpu);
@@ -820,6 +825,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
 		local_irq_enable();
 
+		trace_kvm_exit(vcpu, run->exit_reason, trap.scause);
+
 		preempt_enable();
 
 		kvm_vcpu_srcu_read_lock(vcpu);
Signature · 1 line
2.37.2
maintainer
#2 lore raw

Re: [PATCH 1/2] RISCV: KVM: add tracepoints for entry and exit events

Quoted 17 lines · On Thu, Mar 28, 2024 at 8:49?AM Shenlin Liang…

Like other architectures, RISCV KVM also needs to add these event tracepoints to count the number of times kvm guest entry/exit.

---
 arch/riscv/kvm/trace_riscv.h | 60 ++++++++++++++++++++++++++++++++++++
 arch/riscv/kvm/vcpu.c        |  7 +++++
 2 files changed, 67 insertions(+)
 create mode 100644 arch/riscv/kvm/trace_riscv.h
arch/riscv/kvm/trace_riscv.h new file +0 -0
new file mode 100644
index 000000000000..5848083c7a5e

arch/riscv/kvm/trace_riscv.h line 9

Quoted 9 lines
arch/riscv/kvm/trace_riscv.h new file +9 -0
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Tracepoints for RISC-V KVM
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *
+ */
+#if !defined(_TRACE_RSICV_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RSICV_KVM_H

s/_RSICV_/_RISCV_/

Quoted 20 lines
arch/riscv/kvm/trace_riscv.h new file +20 -0
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+
+TRACE_EVENT(kvm_entry,
+       TP_PROTO(struct kvm_vcpu *vcpu),
+       TP_ARGS(vcpu),
+
+       TP_STRUCT__entry(
+               __field(unsigned long, pc)
+       ),
+
+       TP_fast_assign(
+               __entry->pc     = vcpu->arch.guest_context.sepc;
+       ),
+
+       TP_printk("PC: 0x%016lx", __entry->pc)
+);

arch/riscv/kvm/trace_riscv.h line 39

Quoted 10 lines
arch/riscv/kvm/trace_riscv.h new file +10 -0
+
+TRACE_EVENT(kvm_exit,
+       TP_PROTO(struct kvm_vcpu *vcpu, unsigned long exit_reason,
+                       unsigned long scause),
+       TP_ARGS(vcpu, exit_reason, scause),
+
+       TP_STRUCT__entry(
+               __field(unsigned long, pc)
+               __field(unsigned long, exit_reason)
+               __field(unsigned long, scause)

This is not the right contents describing a KVM exit.

The fields over here should be aligned with "struct kvm_cpu_trap" so we should have following fields:
__field(unsigned long, sepc)
__field(unsigned long, scause)
__field(unsigned long, stval)
__field(unsigned long, htval)
__field(unsigned long, htinst)

Quoted 54 lines
arch/riscv/kvm/trace_riscv.h new file +21 -0
+       ),
+
+       TP_fast_assign(
+               __entry->pc             = vcpu->arch.guest_context.sepc;
+               __entry->exit_reason    = exit_reason;
+               __entry->scause         = scause;
+       ),
+
+       TP_printk("EXIT_REASON:0x%lx,PC: 0x%016lx,SCAUSE:0x%lx",
+                       __entry->exit_reason, __entry->pc, __entry->scause)
+);
+
+#endif /* _TRACE_RSICV_KVM_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace_riscv
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
arch/riscv/kvm/vcpu.c +7 -1
index b5ca9f2e98ac..ed0932f0d514 100644
@@ -21,6 +21,9 @@
 #include <asm/cacheflush.h>
 #include <asm/kvm_vcpu_vector.h>
 
+#define CREATE_TRACE_POINTS
+#include "trace_riscv.h"
+
 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
        KVM_GENERIC_VCPU_STATS(),
        STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
@@ -782,6 +785,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                 */
                kvm_riscv_local_tlb_sanitize(vcpu);
 
+               trace_kvm_entry(vcpu);
+
                guest_timing_enter_irqoff();
 
                kvm_riscv_vcpu_enter_exit(vcpu);
@@ -820,6 +825,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
                local_irq_enable();
 
+               trace_kvm_exit(vcpu, run->exit_reason, trap.scause);
+
                preempt_enable();
 
                kvm_vcpu_srcu_read_lock(vcpu);
--
Quoted 2 lines

2.37.2

Regards,
Anup

#3 lore raw

Re: [PATCH 1/2] RISCV: KVM: add tracepoints for entry and exit events

Quoted 125 lines · Anup Patel
Quoted 17 lines · On Thu, Mar 28, 2024 at 8:49?AM Shenlin Liang…

Like other architectures, RISCV KVM also needs to add these event tracepoints to count the number of times kvm guest entry/exit.

---
 arch/riscv/kvm/trace_riscv.h | 60 ++++++++++++++++++++++++++++++++++++
 arch/riscv/kvm/vcpu.c        |  7 +++++
 2 files changed, 67 insertions(+)
 create mode 100644 arch/riscv/kvm/trace_riscv.h
arch/riscv/kvm/trace_riscv.h new file +0 -0
new file mode 100644
index 000000000000..5848083c7a5e

arch/riscv/kvm/trace_riscv.h line 9

Quoted 9 lines
arch/riscv/kvm/trace_riscv.h new file +9 -0
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Tracepoints for RISC-V KVM
+ *
+ * Copyright 2024 Beijing ESWIN Computing Technology Co., Ltd.
+ *
+ */
+#if !defined(_TRACE_RSICV_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RSICV_KVM_H

s/_RSICV_/_RISCV_/

Quoted 20 lines
arch/riscv/kvm/trace_riscv.h new file +20 -0
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+
+TRACE_EVENT(kvm_entry,
+       TP_PROTO(struct kvm_vcpu *vcpu),
+       TP_ARGS(vcpu),
+
+       TP_STRUCT__entry(
+               __field(unsigned long, pc)
+       ),
+
+       TP_fast_assign(
+               __entry->pc     = vcpu->arch.guest_context.sepc;
+       ),
+
+       TP_printk("PC: 0x%016lx", __entry->pc)
+);

arch/riscv/kvm/trace_riscv.h line 39

Quoted 10 lines
arch/riscv/kvm/trace_riscv.h new file +10 -0
+
+TRACE_EVENT(kvm_exit,
+       TP_PROTO(struct kvm_vcpu *vcpu, unsigned long exit_reason,
+                       unsigned long scause),
+       TP_ARGS(vcpu, exit_reason, scause),
+
+       TP_STRUCT__entry(
+               __field(unsigned long, pc)
+               __field(unsigned long, exit_reason)
+               __field(unsigned long, scause)

This is not the right contents describing a KVM exit.

The fields over here should be aligned with "struct kvm_cpu_trap" so we should have following fields:
__field(unsigned long, sepc)
__field(unsigned long, scause)
__field(unsigned long, stval)
__field(unsigned long, htval)
__field(unsigned long, htinst)

Quoted 52 lines
arch/riscv/kvm/trace_riscv.h new file +21 -0
+       ),
+
+       TP_fast_assign(
+               __entry->pc             = vcpu->arch.guest_context.sepc;
+               __entry->exit_reason    = exit_reason;
+               __entry->scause         = scause;
+       ),
+
+       TP_printk("EXIT_REASON:0x%lx,PC: 0x%016lx,SCAUSE:0x%lx",
+                       __entry->exit_reason, __entry->pc, __entry->scause)
+);
+
+#endif /* _TRACE_RSICV_KVM_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace_riscv
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
arch/riscv/kvm/vcpu.c +7 -0
index b5ca9f2e98ac..ed0932f0d514 100644
@@ -21,6 +21,9 @@
 #include <asm/cacheflush.h>
 #include <asm/kvm_vcpu_vector.h>
 
+#define CREATE_TRACE_POINTS
+#include "trace_riscv.h"
+
 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
        KVM_GENERIC_VCPU_STATS(),
        STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
@@ -782,6 +785,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                 */
                kvm_riscv_local_tlb_sanitize(vcpu);
 
+               trace_kvm_entry(vcpu);
+
                guest_timing_enter_irqoff();
 
                kvm_riscv_vcpu_enter_exit(vcpu);
@@ -820,6 +825,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
                local_irq_enable();
 
+               trace_kvm_exit(vcpu, run->exit_reason, trap.scause);
+
                preempt_enable();
 
Quoted 7 lines
Quoted 4 lines
arch/riscv/kvm/vcpu.c +0 -1
                kvm_vcpu_srcu_read_lock(vcpu);
--

2.37.2

Regards,
Anup

Thank you for the review. I will send the v2 version.

#4 lore raw

Re: [PATCH 1/2] RISCV: KVM: add tracepoints for entry and exit events

In reply to Shenlin Liang
Quoted 6 lines · Shenlin Liang

Like other architectures, RISCV KVM also needs to add these event tracepoints to count the number of times kvm guest entry/exit.

---
  arch/riscv/kvm/trace_riscv.h | 60 ++++++++++++++++++++++++++++++++++++

Conventionally, it should be named to trace.h, especially if only one. Refer to other arch's.

#5 lore raw

Re: [PATCH 1/2] RISCV: KVM: add tracepoints for entry and exit events

Quoted 11 lines · Eric Cheng
Quoted 6 lines · Shenlin Liang

Like other architectures, RISCV KVM also needs to add these event tracepoints to count the number of times kvm guest entry/exit.

---
  arch/riscv/kvm/trace_riscv.h | 60 ++++++++++++++++++++++++++++++++++++

Conventionally, it should be named to trace.h, especially if only one. Refer to other arch's.

Thank you for the review. I will send the v2 version.

#6 lore raw

PATCH 2/2 perf kvm/riscv: Port perf kvm stat to RISC-V

In reply to Shenlin Liang

'perf kvm stat report/record' generates a statistical analysis of KVM events and can be used to analyze guest exit reasons.

"report" reports statistical analysis of guest exit events.

To record kvm events on the host:

 # perf kvm stat record -a

To report kvm VM EXIT events:

 # perf kvm stat report --event=vmexit
---
 tools/perf/arch/riscv/Makefile                |  1 +
 tools/perf/arch/riscv/util/Build              |  1 +
 tools/perf/arch/riscv/util/kvm-stat.c         | 78 +++++++++++++++++++
 .../arch/riscv/util/riscv_exception_types.h   | 41 ++++++++++
 4 files changed, 121 insertions(+)
 create mode 100644 tools/perf/arch/riscv/util/kvm-stat.c
 create mode 100644 tools/perf/arch/riscv/util/riscv_exception_types.h
tools/perf/arch/riscv/Makefile +1 -0
index a8d25d005207..e1e445615536 100644
@@ -3,3 +3,4 @@ PERF_HAVE_DWARF_REGS := 1
 endif
 PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1
 PERF_HAVE_JITDUMP := 1
+HAVE_KVM_STAT_SUPPORT := 1
\ No newline at end of file
tools/perf/arch/riscv/util/Build +1 -0
index 603dbb5ae4dc..d72b04f8d32b 100644
@@ -1,5 +1,6 @@
 perf-y += perf_regs.o
 perf-y += header.o
 
+perf-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o
 perf-$(CONFIG_DWARF) += dwarf-regs.o
 perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o
tools/perf/arch/riscv/util/kvm-stat.c new file +78 -0

78 lines not on this page · raw message

tools/perf/arch/riscv/util/riscv_exception_types.h new file +41 -0

41 lines not on this page · raw message

Signature · 1 line
2.37.2
#7 lore raw
In reply to Shenlin Liang

Gentle ping...

Quoted 56 lines

'perf kvm stat report/record' generates a statistical analysis of KVM events and can be used to analyze guest exit reasons. This patch tries to add stat support on riscv.

Map the return value of trace_kvm_exit() to the specific cause of the exception, and export it to userspace.

It records on two available KVM tracepoints for riscv: "kvm:kvm_entry" and "kvm:kvm_exit", and reports statistical data which includes events handles time, samples, and so on.

Simple tests go below:

# ./perf kvm record -e "kvm:kvm_entry" -e "kvm:kvm_exit"
Lowering default frequency rate from 4000 to 2500.
Please consider tweaking /proc/sys/kernel/perf_event_max_sample_rate.
[ perf record: Woken up 18 times to write data ]
[ perf record: Captured and wrote 5.433 MB perf.data.guest (62519 samples) 
# ./perf kvm report
31K kvm:kvm_entry
31K kvm:kvm_exit
# ./perf kvm stat record -a
[ perf record: Woken up 3 times to write data ]
[ perf record: Captured and wrote 8.502 MB perf.data.guest (99338 samples) ]
# ./perf kvm stat report --event=vmexit
Event name                Samples   Sample%    Time (ns)     Time%   Max Time (ns)   Min Time (ns)  Mean Time (ns)
STORE_GUEST_PAGE_FAULT     26968     54.00%    2003031800    40.00%     3361400         27600          74274
LOAD_GUEST_PAGE_FAULT      17645     35.00%    1153338100    23.00%     2513400         30800          65363
VIRTUAL_INST_FAULT         1247      2.00%     340820800     6.00%      1190800         43300          273312
INST_GUEST_PAGE_FAULT      1128      2.00%     340645800     6.00%      2123200         30200          301990
SUPERVISOR_SYSCALL         1019      2.00%     245989900     4.00%      1851500         29300          241403
LOAD_ACCESS                986       1.00%     671556200     13.00%     4180200         100700         681091
INST_ACCESS                655       1.00%     170054800     3.00%      1808300         54600          259625
HYPERVISOR_SYSCALL         21        0.00%     4276400       0.00%      716500          116000         203638 

Shenlin Liang (2):
RISCV: KVM: add tracepoints for entry and exit events perf kvm/riscv: Port perf kvm stat to RISC-V

 arch/riscv/kvm/trace_riscv.h                  | 60 ++++++++++++++
 arch/riscv/kvm/vcpu.c                         |  7 ++
 tools/perf/arch/riscv/Makefile                |  1 +
 tools/perf/arch/riscv/util/Build              |  1 +
 tools/perf/arch/riscv/util/kvm-stat.c         | 78 +++++++++++++++++++
 .../arch/riscv/util/riscv_exception_types.h   | 41 ++++++++++
 6 files changed, 188 insertions(+)
 create mode 100644 arch/riscv/kvm/trace_riscv.h
 create mode 100644 tools/perf/arch/riscv/util/kvm-stat.c
 create mode 100644 tools/perf/arch/riscv/util/riscv_exception_types.h
Signature · 1 line
2.37.2