Skip to contentSkip to conversation

KVM: arm64: Count guest exit due to various reasons

PATCH Superseded Amit 3 participants 6 messages Oct 18, 2015 – Oct 26, 2015

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Amit

author
#0 lore raw

PATCH 1/1 KVM: arm64: Count guest exit due to various reasons

From: Amit Singh Tomar <[email protected]>

This patch adds guest exit statistics to debugfs, This can be helpful while measuring KVM performance.

---
 arch/arm/include/asm/kvm_host.h   | 2 ++
 arch/arm/kvm/arm.c                | 1 +
 arch/arm/kvm/guest.c              | 2 ++
 arch/arm/kvm/mmio.c               | 1 +
 arch/arm64/include/asm/kvm_host.h | 5 +++++
 arch/arm64/kvm/guest.c            | 8 ++++++++
 arch/arm64/kvm/handle_exit.c      | 3 +++
 7 files changed, 22 insertions(+)
arch/arm/include/asm/kvm_host.h +2 -0
index c4072d9..d1f3159 100644
@@ -147,6 +147,8 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm/kvm/arm.c +1 -0
index 78b2869..5d0a2c5 100644
@@ -569,6 +569,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 
 		vcpu->mode = OUTSIDE_GUEST_MODE;
+		vcpu->stat.exits++;
 		/*
 		 * Back from guest
 		 *************************************************************/
arch/arm/kvm/guest.c +2 -0
index 96e935b..0eea5fc 100644
@@ -33,6 +33,8 @@
 #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm/kvm/mmio.c +1 -0
index 974b1c6..51d518e 100644
@@ -184,6 +184,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	}
 
 	rt = vcpu->arch.mmio_decode.rt;
+	vcpu->stat.mmio_exit_stat++;
 
 	if (is_write) {
 		data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
arch/arm64/include/asm/kvm_host.h +5 -0
index ed03968..b32c938 100644
@@ -194,6 +194,11 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 hvc_exit_stat;
+	u32 wfx_exit_stat;
+	u32 wfi_exit_stat;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm64/kvm/guest.c +8 -0
index d250160..b1943cd 100644
@@ -34,7 +34,15 @@
 
 #include "trace.h"
 
+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfx_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm64/kvm/handle_exit.c +3 -0
index 68a0759..6e38c50 100644
@@ -39,6 +39,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 	trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
 			    kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {
@@ -71,9 +72,11 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfx_exit_stat++;
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}
 
Signature · 1 line
1.9.1
#1 lore raw
Quoted 129 lines · Amit

From: Amit Singh Tomar <[email protected]>

This patch adds guest exit statistics to debugfs, This can be helpful while measuring KVM performance.

---
 arch/arm/include/asm/kvm_host.h   | 2 ++
 arch/arm/kvm/arm.c                | 1 +
 arch/arm/kvm/guest.c              | 2 ++
 arch/arm/kvm/mmio.c               | 1 +
 arch/arm64/include/asm/kvm_host.h | 5 +++++
 arch/arm64/kvm/guest.c            | 8 ++++++++
 arch/arm64/kvm/handle_exit.c      | 3 +++
 7 files changed, 22 insertions(+)
arch/arm/include/asm/kvm_host.h +2 -0
index c4072d9..d1f3159 100644
@@ -147,6 +147,8 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm/kvm/arm.c +1 -0
index 78b2869..5d0a2c5 100644
@@ -569,6 +569,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 
 		vcpu->mode = OUTSIDE_GUEST_MODE;
+		vcpu->stat.exits++;
 		/*
 		 * Back from guest
 		 *************************************************************/
arch/arm/kvm/guest.c +2 -0
index 96e935b..0eea5fc 100644
@@ -33,6 +33,8 @@
 #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm/kvm/mmio.c +1 -0
index 974b1c6..51d518e 100644
@@ -184,6 +184,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	}
 
 	rt = vcpu->arch.mmio_decode.rt;
+	vcpu->stat.mmio_exit_stat++;
 
 	if (is_write) {
 		data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
arch/arm64/include/asm/kvm_host.h +5 -0
index ed03968..b32c938 100644
@@ -194,6 +194,11 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 hvc_exit_stat;
+	u32 wfx_exit_stat;
+	u32 wfi_exit_stat;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm64/kvm/guest.c +8 -0
index d250160..b1943cd 100644
@@ -34,7 +34,15 @@
 
 #include "trace.h"
 
+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfx_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm64/kvm/handle_exit.c +3 -0
index 68a0759..6e38c50 100644
@@ -39,6 +39,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 	trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
 			    kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {
@@ -71,9 +72,11 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfx_exit_stat++;
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}
 
Signature · 1 line
1.9.1

I thought the idea was to not use this infrastructure anymore and instead use tracepoints to count exit reasons?

I'm cc'ing Wei here who sent some patches to QEMU for this purpose recently.

Thanks,
-Christoffer

#2 lore raw
Quoted 134 lines · Christoffer Dall
Quoted 129 lines · Amit

From: Amit Singh Tomar <[email protected]>

This patch adds guest exit statistics to debugfs, This can be helpful while measuring KVM performance.

---
 arch/arm/include/asm/kvm_host.h   | 2 ++
 arch/arm/kvm/arm.c                | 1 +
 arch/arm/kvm/guest.c              | 2 ++
 arch/arm/kvm/mmio.c               | 1 +
 arch/arm64/include/asm/kvm_host.h | 5 +++++
 arch/arm64/kvm/guest.c            | 8 ++++++++
 arch/arm64/kvm/handle_exit.c      | 3 +++
 7 files changed, 22 insertions(+)
arch/arm/include/asm/kvm_host.h +2 -0
index c4072d9..d1f3159 100644
@@ -147,6 +147,8 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm/kvm/arm.c +1 -0
index 78b2869..5d0a2c5 100644
@@ -569,6 +569,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 
 		vcpu->mode = OUTSIDE_GUEST_MODE;
+		vcpu->stat.exits++;
 		/*
 		 * Back from guest
 		 *************************************************************/
arch/arm/kvm/guest.c +2 -0
index 96e935b..0eea5fc 100644
@@ -33,6 +33,8 @@
 #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm/kvm/mmio.c +1 -0
index 974b1c6..51d518e 100644
@@ -184,6 +184,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	}
 
 	rt = vcpu->arch.mmio_decode.rt;
+	vcpu->stat.mmio_exit_stat++;
 
 	if (is_write) {
 		data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
arch/arm64/include/asm/kvm_host.h +5 -0
index ed03968..b32c938 100644
@@ -194,6 +194,11 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 hvc_exit_stat;
+	u32 wfx_exit_stat;
+	u32 wfi_exit_stat;
+	u32 mmio_exit_stat;
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm64/kvm/guest.c +8 -0
index d250160..b1943cd 100644
@@ -34,7 +34,15 @@
 
 #include "trace.h"
 
+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfx_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm64/kvm/handle_exit.c +3 -0
index 68a0759..6e38c50 100644
@@ -39,6 +39,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 	trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
 			    kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {
@@ -71,9 +72,11 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfx_exit_stat++;
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}
 
Signature · 1 line
1.9.1

I thought the idea was to not use this infrastructure anymore and instead use tracepoints to count exit reasons?

Quoted 2 lines

I'm cc'ing Wei here who sent some patches to QEMU for this purpose recently.

The patches I submitted were for fixing architecture-related issues while running kvm_stat on ARM platforms. I don't think tracepoint is meant to replace debugfs for kvm profiling. Both methods have their own benefits; and they are supported on x86 & ppc platforms. From this perspective I don't mind taking this patch (after review).

Thanks,
-Wei

Quoted 4 lines

Thanks,
-Christoffer

#3 lore raw
In reply to Amit
Quoted 21 lines · Amit

From: Amit Singh Tomar <[email protected]>

This patch adds guest exit statistics to debugfs, This can be helpful while measuring KVM performance.

---
 arch/arm/include/asm/kvm_host.h   | 2 ++
 arch/arm/kvm/arm.c                | 1 +
 arch/arm/kvm/guest.c              | 2 ++
 arch/arm/kvm/mmio.c               | 1 +
 arch/arm64/include/asm/kvm_host.h | 5 +++++
 arch/arm64/kvm/guest.c            | 8 ++++++++
 arch/arm64/kvm/handle_exit.c      | 3 +++
 7 files changed, 22 insertions(+)
arch/arm/include/asm/kvm_host.h +0 -0
index c4072d9..d1f3159 100644

arch/arm/include/asm/kvm_host.h struct kvm_vcpu_stat { · line 150

Quoted 4 lines
arch/arm/include/asm/kvm_host.h +1 -0
@@ -147,6 +147,8 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 mmio_exit_stat;

I think it would be even more useful to distinguish between mmio exits handled in the kernel vs. those going to userspace.

Quoted 46 lines
arch/arm/include/asm/kvm_host.h +1 -0
+	u32 exits;
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm/kvm/arm.c +1 -0
index 78b2869..5d0a2c5 100644
@@ -569,6 +569,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 
 		vcpu->mode = OUTSIDE_GUEST_MODE;
+		vcpu->stat.exits++;
 		/*
 		 * Back from guest
 		 *************************************************************/
arch/arm/kvm/guest.c +2 -0
index 96e935b..0eea5fc 100644
@@ -33,6 +33,8 @@
 #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm/kvm/mmio.c +1 -0
index 974b1c6..51d518e 100644
@@ -184,6 +184,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	}
 
 	rt = vcpu->arch.mmio_decode.rt;
+	vcpu->stat.mmio_exit_stat++;
 
 	if (is_write) {
 		data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
arch/arm64/include/asm/kvm_host.h +0 -0
index ed03968..b32c938 100644

arch/arm64/include/asm/kvm_host.h struct kvm_vcpu_stat { · line 201

Quoted 8 lines
arch/arm64/include/asm/kvm_host.h +5 -0
@@ -194,6 +194,11 @@ struct kvm_vcpu_stat {
 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 hvc_exit_stat;
+	u32 wfx_exit_stat;
+	u32 wfi_exit_stat;
+	u32 mmio_exit_stat;
+	u32 exits;

why more exit stats on arm64 than on arm?

why u32? Would it not make sense to make these u64 or just unsigned long?

Quoted 36 lines
arch/arm64/include/asm/kvm_host.h +0 -0
 };
 
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
arch/arm64/kvm/guest.c +8 -0
index d250160..b1943cd 100644
@@ -34,7 +34,15 @@
 
 #include "trace.h"
 
+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfx_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_stat),
+	VCPU_STAT(exits),
 	{ NULL }
 };
 
arch/arm64/kvm/handle_exit.c +1 -0
index 68a0759..6e38c50 100644
@@ -39,6 +39,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 	trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
 			    kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {

arch/arm64/kvm/handle_exit.c static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) · line 75

Quoted 4 lines
arch/arm64/kvm/handle_exit.c +1 -0
@@ -71,9 +72,11 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfx_exit_stat++;

either you should use one called wfx_exit_stat or you should distinguish between wfi and wfe; this is just a weird way of doing it.

Quoted 10 lines
arch/arm64/kvm/handle_exit.c +1 -0
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}
 
Signature · 1 line
1.9.1

Thanks,
-Christoffer

author
#4 lore raw

Thanks for the Review.

Quoted 2 lines

I think it would be even more useful to distinguish between mmio exits handled in the kernel vs. those going to userspace.

Ok, I would work on it.

Quoted 1 line

why more exit stats on arm64 than on arm?

Intention is to just add these stats only for ARM64 but I had to do it for arm, so that things don't break.

Quoted 2 lines

why u32? Would it not make sense to make these u64 or just unsigned long?

Ok, You mean to say, for ARM we should use u32 and for ARM64 we should u64, right?

Quoted 2 lines

either you should use one called wfx_exit_stat or you should distinguish between wfi and wfe; this is just a weird way of doing it.

Sorry, didn't just get your point here :(

Thanks
Amit.

#5 lore raw
Quoted 6 lines · Amit Tomer

Thanks for the Review.

Quoted 2 lines

I think it would be even more useful to distinguish between mmio exits handled in the kernel vs. those going to userspace.

Quoted 7 lines

Ok, I would work on it.

Quoted 1 line

why more exit stats on arm64 than on arm?

Intention is to just add these stats only for ARM64 but I had to do it for arm, so that things don't break.

please keep both architectures up to date as much as possible for these kinds of changes.

Quoted 6 lines
Quoted 3 lines

why u32? Would it not make sense to make these u64 or just unsigned long?

Ok, You mean to say, for ARM we should use u32 and for ARM64 we should u64, right?

loss of context here, so I'm not sure. Is there a reason why you cannot/shouldn't use u64? I'm just thinking that overflows could happen fairly easy on u32, but otoh. there are some existing defines using u32 for some reason...

Quoted 6 lines
Quoted 3 lines

either you should use one called wfx_exit_stat or you should distinguish between wfi and wfe; this is just a weird way of doing it.

Sorry, didn't just get your point here :(

wfx means "either a wfi or wfe happened", so you either define two separate trace points, one for wfi and one for wfe, or you define a single tracepoint for wfx.

I'll look forward to v2.

-Christoffer