Skip to contentSkip to conversation

kvmppc: add detailed exit timing statistic v2

PATCH New [email protected] 1 participant 1 message Oct 15, 2008

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 kvmppc: add detailed exit timing statistic v2

From: Christian Ehrhardt <[email protected]>

Since relay based statistic approaches slow us down to much in some scenarios this patch adds a configurable exit timing statistic.
It is powerpc only and holds timing stats of exit/reenter per exit type read via debugfs.

updates in v2:
* moved statistic code to a separate header
* changed code to be at least basically multi cpu/guest safe
* reworked old noop stubs used if exit timing is not configured
* combined kvm-stat based accounting with this (less code in our main code flow)
* drop internal debug print_exit_timing and it's callers
* changed access from sysrq to debugfs using seq_file
* introduced per vm & per vcpu debugfs reports
* a lot of renaming/cleaning

The issue behind this patch is that trace based approaches generate too much overhead in our hot path, while the existng kvm_stat interface is not able to report per vm/vcpu (or would become unusable by the high number of files I would need). Therefore this path uses the elso existing seq_file helpers to extend the data presented in the %debugfs%/kvm directory.

This emerged from a debug patch I once created. I reworked and polished it to allow it to be applied to our powerpc upstrema code, but it can't deny it's heritage so this patch v2 is my rfc to see if/how I could/should do different.

---

[diffstat]

 arch/powerpc/kernel/asm-offsets.c      |   11 ++
 arch/powerpc/kvm/Kconfig               |    9 ++
 arch/powerpc/kvm/booke_guest.c         |   39 +++++----
 arch/powerpc/kvm/booke_interrupts.S    |   24 +++++
 arch/powerpc/kvm/powerpc.c             |   93 ++++++++++++++++++++++
 include/asm-powerpc/kvm_host.h         |   50 ++++++++++++
 include/asm-powerpc/kvm_timing_stats.h |  136 +++++++++++++++++++++++++++++++++
 7 files changed, 347 insertions(+), 15 deletions(-)

[diff]

arch/powerpc/kernel/asm-offsets.c +11 -0
@@ -348,5 +348,16 @@
 	DEFINE(VCPU_FAULT_ESR, offsetof(struct kvm_vcpu, arch.fault_esr));
 #endif
 
+#ifdef CONFIG_KVM_BOOKE_EXIT_TIMING
+	DEFINE(VCPU_TIMING_EXIT_TBU, offsetof(struct kvm_vcpu,
+						arch.timing_exit.tv32.tbu));
+	DEFINE(VCPU_TIMING_EXIT_TBL, offsetof(struct kvm_vcpu,
+						arch.timing_exit.tv32.tbl));
+	DEFINE(VCPU_TIMING_LAST_ENTER_TBU, offsetof(struct kvm_vcpu,
+					arch.timing_last_enter.tv32.tbu));
+	DEFINE(VCPU_TIMING_LAST_ENTER_TBL, offsetof(struct kvm_vcpu,
+					arch.timing_last_enter.tv32.tbl));
+#endif
+
 	return 0;
 }
arch/powerpc/kvm/Kconfig +9 -0
@@ -37,6 +37,15 @@
 	  Provides host support for KVM on Book E PowerPC processors. Currently
 	  this works on 440 processors only.
 
+config KVM_BOOKE_EXIT_TIMING
+	bool "Trace detailed exit Timing"
+	depends on KVM_BOOKE_HOST
+	---help---
+	  Inserts code to trace timestamps for every exit/enter cycle. A per vcpu
+	  report is available in debugfs kvm/VM_###/VPCU_###_exit_timing.
+	  The overhead is relatively small, however it is not recommended for
+	  production environments.
+
 config KVM_TRACE
 	bool "KVM trace support"
 	depends on KVM && MARKERS && SYSFS
arch/powerpc/kvm/booke_guest.c +24 -15
@@ -29,6 +29,7 @@
 #include <asm/cputable.h>
 #include <asm/uaccess.h>
 #include <asm/kvm_ppc.h>
+#include <asm/kvm_timing_stats.h>
 
 #include "44x_tlb.h"
 
@@ -228,6 +229,9 @@
 	enum emulation_result er;
 	int r = RESUME_HOST;
 
+	/* update before a new last_exit_type is rewritten */
+	update_timing_stats(vcpu);
+
 	local_irq_enable();
 
 	run->exit_reason = KVM_EXIT_UNKNOWN;
@@ -241,7 +245,7 @@
 		break;
 
 	case BOOKE_INTERRUPT_EXTERNAL:
-		vcpu->stat.ext_intr_exits++;
+		account_exit(vcpu, EXT_INTR_EXITS);
 		if (need_resched())
 			cond_resched();
 		r = RESUME_GUEST;
@@ -256,7 +260,7 @@
 		 * we do reschedule the host will fault over it. Perhaps we
 		 * should politely restore the host's entries to minimize
 		 * misses before ceding control. */
-		vcpu->stat.dec_exits++;
+		account_exit(vcpu, DEC_EXITS);
 		if (need_resched())
 			cond_resched();
 		r = RESUME_GUEST;
@@ -269,6 +273,7 @@
 			vcpu->arch.esr = vcpu->arch.fault_esr;
 			kvmppc_queue_exception(vcpu, BOOKE_INTERRUPT_PROGRAM);
 			r = RESUME_GUEST;
+			account_exit(vcpu, USR_PR_INST);
 			break;
 		}
 
@@ -277,12 +282,12 @@
 		case EMULATE_DONE:
 			/* Future optimization: only reload non-volatiles if
 			 * they were actually modified by emulation. */
-			vcpu->stat.emulated_inst_exits++;
+			account_exit(vcpu, EMULATED_INST_EXITS);
 			r = RESUME_GUEST_NV;
 			break;
 		case EMULATE_DO_DCR:
 			run->exit_reason = KVM_EXIT_DCR;
-			vcpu->stat.dcr_exits++;
+			account_exit(vcpu, DCR_EXITS);
 			r = RESUME_HOST;
 			break;
 		case EMULATE_FAIL:
@@ -302,6 +307,7 @@
 
 	case BOOKE_INTERRUPT_FP_UNAVAIL:
 		kvmppc_queue_exception(vcpu, exit_nr);
+		account_exit(vcpu, FP_UNAVAIL);
 		r = RESUME_GUEST;
 		break;
 
@@ -309,24 +315,24 @@
 		vcpu->arch.dear = vcpu->arch.fault_dear;
 		vcpu->arch.esr = vcpu->arch.fault_esr;
 		kvmppc_queue_exception(vcpu, exit_nr);
-		vcpu->stat.dsi_exits++;
+		account_exit(vcpu, DSI_EXITS);
 		r = RESUME_GUEST;
 		break;
 
 	case BOOKE_INTERRUPT_INST_STORAGE:
 		vcpu->arch.esr = vcpu->arch.fault_esr;
 		kvmppc_queue_exception(vcpu, exit_nr);
-		vcpu->stat.isi_exits++;
+		account_exit(vcpu, ISI_EXITS);
 		r = RESUME_GUEST;
 		break;
 
 	case BOOKE_INTERRUPT_SYSCALL:
 		if (vcpu->arch.last_inst = KVM_HYPERCALL_BIN) {
 			kvmppc_do_hypercall(vcpu);
-			vcpu->stat.hcall_exits++;
+			account_exit(vcpu, HCALL_EXITS);
 		} else {
 			kvmppc_queue_exception(vcpu, exit_nr);
-			vcpu->stat.syscall_exits++;
+			account_exit(vcpu, SYSCALL_EXITS);
 		}
 		r = RESUME_GUEST;
 		break;
@@ -340,7 +346,7 @@
 			kvmppc_mmu_map(vcpu, eaddr,
 			 vcpu->arch.pvmem_gpaddr >> KVM_PPCPV_MAGIC_PAGE_SHIFT,
 			 0, KVM_PPCPV_MAGIC_PAGE_FLAGS);
-			vcpu->stat.dtlb_pvmem_miss_exits++;
+			account_exit(vcpu, DTLB_PVMEM_MISS_EXITS);
 			r = RESUME_GUEST;
 			break;
 		}
@@ -352,7 +358,7 @@
 			kvmppc_queue_exception(vcpu, exit_nr);
 			vcpu->arch.dear = vcpu->arch.fault_dear;
 			vcpu->arch.esr = vcpu->arch.fault_esr;
-			vcpu->stat.dtlb_real_miss_exits++;
+			account_exit(vcpu, DTLB_REAL_MISS_EXITS);
 			r = RESUME_GUEST;
 			break;
 		}
@@ -369,13 +375,13 @@
 			 * invoking the guest. */
 			kvmppc_mmu_map(vcpu, eaddr, gfn, gtlbe->tid,
 			               gtlbe->word2);
-			vcpu->stat.dtlb_virt_miss_exits++;
+			account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
 			r = RESUME_GUEST;
 		} else {
 			/* Guest has mapped and accessed a page which is not
 			 * actually RAM. */
 			r = kvmppc_emulate_mmio(run, vcpu);
-			vcpu->stat.mmio_exits++;
+			account_exit(vcpu, MMIO_EXITS);
 		}
 
 		break;
@@ -393,11 +399,11 @@
 		if (!gtlbe) {
 			/* The guest didn't have a mapping for it. */
 			kvmppc_queue_exception(vcpu, exit_nr);
-			vcpu->stat.itlb_real_miss_exits++;
+			account_exit(vcpu, ITLB_REAL_MISS_EXITS);
 			break;
 		}
 
-		vcpu->stat.itlb_virt_miss_exits++;
+		account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
 
 		gfn = tlb_xlate(gtlbe, eaddr) >> PAGE_SHIFT;
 
@@ -430,6 +436,7 @@
 		mtspr(SPRN_DBSR, dbsr);
 
 		run->exit_reason = KVM_EXIT_DEBUG;
+		account_exit(vcpu, DEBUG_EXITS);
 		r = RESUME_HOST;
 		break;
 	}
@@ -450,7 +457,7 @@
 		if (signal_pending(current)) {
 			run->exit_reason = KVM_EXIT_INTR;
 			r = (-EINTR << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
-			vcpu->stat.signal_exits++;
+			account_exit(vcpu, SIGNAL_EXITS);
 		}
 	}
 
@@ -490,6 +497,8 @@
 	 * real timebase frequency. Accordingly, it must see the state of
 	 * CCR1[TCS]. */
 	vcpu->arch.ccr1 = mfspr(SPRN_CCR1);
+
+	init_timing_stats(vcpu);
 
 	return 0;
 }
arch/powerpc/kvm/booke_interrupts.S +24 -0
@@ -109,6 +109,18 @@
 
 	li	r6, 1
 	slw	r6, r6, r5
+
+#ifdef CONFIG_KVM_BOOKE_EXIT_TIMING
+	/* save exit time */
+..exit_tbu_overflow_loop:
+	mfspr	r7, SPRN_TBRU
+	mfspr	r8, SPRN_TBRL
+	mfspr	r9, SPRN_TBRU
+	cmpw	r9, r7
+	bne	..exit_tbu_overflow_loop
+	stw	r8, VCPU_TIMING_EXIT_TBL(r4)
+	stw	r9, VCPU_TIMING_EXIT_TBU(r4)
+#endif
 
 	/* Save the faulting instruction and all GPRs for emulation. */
 	andi.	r7, r6, NEED_INST_MASK
@@ -415,6 +427,18 @@
 	lwz	r3, VCPU_SPRG7(r4)
 	mtspr	SPRN_SPRG7, r3
 
+#ifdef CONFIG_KVM_BOOKE_EXIT_TIMING
+	/* save enter time */
+..enter_tbu_overflow_loop:
+	mfspr	r6, SPRN_TBRU
+	mfspr	r7, SPRN_TBRL
+	mfspr	r8, SPRN_TBRU
+	cmpw	r8, r6
+	bne	..enter_tbu_overflow_loop
+	stw	r7, VCPU_TIMING_LAST_ENTER_TBL(r4)
+	stw	r8, VCPU_TIMING_LAST_ENTER_TBU(r4)
+#endif
+
 	/* Finish loading guest volatiles and jump to guest. */
 	lwz	r3, VCPU_CTR(r4)
 	mtctr	r3
arch/powerpc/kvm/powerpc.c +106 -0

142 lines not on this page · raw message

include/asm-powerpc/kvm_host.h +50 -0

63 lines not on this page · raw message

include/asm-powerpc/kvm_timing_stats.h new file +136 -0

136 lines not on this page · raw message