KVM: Implement nested TSC scaling
v3 cover KVM: Implement nested TSC scaling
KVM currently supports hardware-assisted TSC scaling but only for L1; the feature is not exposed to nested guests. This patch series adds support for nested TSC scaling and allows both L1 and L2 to be scaled with different scaling factors. That is achieved by "merging" the 01 and 02 values together.
Most of the logic in this series is implemented in common code (by doing the necessary restructurings), however the patches add support for VMX only. Adding support for SVM should be easy at this point and Maxim Levitsky has volunteered to do this (thanks!).
Changelog:
v3:
- Applied Sean's feedback
- Refactored patches 7 to 10
v2:
- Applied all of Maxim's feedback
- Added a mul_s64_u64_shr function in math64.h
- Added a separate kvm_scale_tsc_l1 function instead of passing an argument to kvm_scale_tsc
- Implemented the 02 fields calculations in common code
- Moved all of write_l1_tsc_offset's logic to common code
- Added a check for whether the TSC is stable in patch 10
- Used a random L1 factor and a negative offset in patch 10
Ilias Stamatis (12):
math64.h: Add mul_s64_u64_shr()
KVM: X86: Store L1's TSC scaling ratio in 'struct kvm_vcpu_arch'
KVM: X86: Rename kvm_compute_tsc_offset() to
kvm_compute_tsc_offset_l1()
KVM: X86: Add a ratio parameter to kvm_scale_tsc()
KVM: VMX: Add a TSC multiplier field in VMCS12
KVM: X86: Add functions for retrieving L2 TSC fields from common code
KVM: X86: Add functions that calculate L2's TSC offset and multiplier
KVM: X86: Move write_l1_tsc_offset() logic to common code and rename
it
KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
KVM: VMX: Set the TSC offset and multiplier on nested entry and exit
KVM: VMX: Expose TSC scaling to L2
KVM: selftests: x86: Add vmx_nested_tsc_scaling_test
arch/x86/include/asm/kvm-x86-ops.h | 4 +- arch/x86/include/asm/kvm_host.h | 14 +- arch/x86/kvm/svm/svm.c | 29 ++- arch/x86/kvm/vmx/nested.c | 33 ++- arch/x86/kvm/vmx/vmcs12.c | 1 + arch/x86/kvm/vmx/vmcs12.h | 4 +- arch/x86/kvm/vmx/vmx.c | 49 ++-- arch/x86/kvm/vmx/vmx.h | 11 +- arch/x86/kvm/x86.c | 91 +++++-- include/linux/math64.h | 19 ++ tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/vmx_nested_tsc_scaling_test.c | 242 ++++++++++++++++++ 13 files changed, 417 insertions(+), 82 deletions(-) create mode 100644 tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c
--
2.17.1
PATCH v3 1/12 math64.h: Add mul_s64_u64_shr()
This function is needed for KVM's nested virtualization. The nested TSC scaling implementation requires multiplying the signed TSC offset with the unsigned TSC multiplier.
--- include/linux/math64.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+)
include/linux/math64.h +19 -0
@@ -3,6 +3,7 @@#define _LINUX_MATH64_H#include <linux/types.h> +#include <linux/math.h> #include <vdso/math64.h> #include <asm/div64.h>
@@ -234,6 +235,24 @@ static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)#endif +#ifndef mul_s64_u64_shr +static inline u64 mul_s64_u64_shr (s64 a,u64 b,unsigned int shift) +{ +u64 ret; + +/* +* Extract the sign before the multiplication and put it back +* afterwards if needed. +*/ + ret =mul_u64_u64_shr (abs (a), b, shift); + +if (a <0 ) + ret = -((s64 ) ret); + +return ret; +} +#endif /* mul_s64_u64_shr */ +#ifndef mul_u64_u32_divstatic inline u64 mul_u64_u32_div (u64 a,u32 mul,u32 divisor) {
Signature · 1 line
2.17.1
PATCH v3 2/12 KVM: X86: Store L1's TSC scaling ratio in 'struct kvm_vcpu_arch'
In reply to Ilias StamatisStore L1's scaling ratio in the kvm_vcpu_arch struct like we already do for L1's TSC offset. This allows for easy save/restore when we enter and then exit the nested guest.
--- arch/x86/include/asm/kvm_host.h | 5 +++-- arch/x86/kvm/vmx/vmx.c | 4 ++-- arch/x86/kvm/x86.c | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -2
@@ -707,7 +707,7 @@ struct kvm_vcpu_arch { } st;u64 l1_tsc_offset; -u64 tsc_offset; +u64 tsc_offset;/* current tsc offset */ u64 last_guest_tsc;u64 last_host_tsc;u64 tsc_offset_adjustment;
@@ -721,7 +721,8 @@ struct kvm_vcpu_arch {u32 virtual_tsc_khz;s64 ia32_tsc_adjust_msr;u64 msr_ia32_power_ctl; -u64 tsc_scaling_ratio; +u64 l1_tsc_scaling_ratio; +u64 tsc_scaling_ratio;/* current scaling ratio */ atomic_t nmi_queued;/* unprocessed asynchronous NMIs */ unsigned nmi_pending;/* NMI queued after currently running handler */
arch/x86/kvm/vmx/vmx.c +2 -2
@@ -7453,10 +7453,10 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, delta_tsc =0 ;/* Convert to host delta tsc if tsc scaling is enabled */ -if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && +if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && delta_tsc &&u64_shl_div_u64 (delta_tsc, kvm_tsc_scaling_ratio_frac_bits, - vcpu->arch.tsc_scaling_ratio, &delta_tsc)) + vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc))return -ERANGE ;/*
arch/x86/kvm/x86.c +4 -2
@@ -2185,6 +2185,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)/* Guest TSC same frequency as host TSC? */ if (!scale) { + vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;return 0 ; }
@@ -2211,7 +2212,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)return -1 ; } - vcpu->arch.tsc_scaling_ratio = ratio; + vcpu->arch.l1_tsc_scaling_ratio = vcpu->arch.tsc_scaling_ratio = ratio;return 0 ; }
@@ -2223,6 +2224,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)/* tsc_khz can be zero if TSC calibration fails */ if (user_tsc_khz ==0 ) {/* set tsc_scaling_ratio to a safe value */ + vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;return -1 ; }
@@ -2459,7 +2461,7 @@ static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,static inline void adjust_tsc_offset_host (struct kvm_vcpu *vcpu,s64 adjustment) { -if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) +if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)WARN_ON (adjustment <0 ); adjustment =kvm_scale_tsc (vcpu, (u64 ) adjustment);adjust_tsc_offset_guest (vcpu, adjustment);
Signature · 1 line
2.17.1
PATCH v3 3/12 KVM: X86: Rename kvm_compute_tsc_offset() to kvm_compute_tsc_offset_l1()
In reply to Ilias StamatisAll existing code uses kvm_compute_tsc_offset() passing L1 TSC values to it. Let's document this by renaming it to kvm_compute_tsc_offset_l1().
--- arch/x86/kvm/x86.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
arch/x86/kvm/x86.c +5 -5
@@ -2319,7 +2319,7 @@ u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc) }EXPORT_SYMBOL_GPL (kvm_scale_tsc); -static u64 kvm_compute_tsc_offset (struct kvm_vcpu *vcpu,u64 target_tsc) +static u64 kvm_compute_tsc_offset_l1 (struct kvm_vcpu *vcpu,u64 target_tsc) {u64 tsc;
@@ -2363,7 +2363,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)bool synchronizing = false;raw_spin_lock_irqsave (&kvm->arch.tsc_write_lock, flags); - offset =kvm_compute_tsc_offset (vcpu, data); + offset =kvm_compute_tsc_offset_l1 (vcpu, data); ns =get_kvmclock_base_ns (); elapsed = ns - kvm->arch.last_tsc_nsec;
@@ -2402,7 +2402,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) }else {u64 delta =nsec_to_cycles (vcpu, elapsed); data += delta; - offset =kvm_compute_tsc_offset (vcpu, data); + offset =kvm_compute_tsc_offset_l1 (vcpu, data); } matched = true; already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
@@ -3235,7 +3235,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)if (msr_info->host_initiated) {kvm_synchronize_tsc (vcpu, data); }else { -u64 adj =kvm_compute_tsc_offset (vcpu, data) - vcpu->arch.l1_tsc_offset; +u64 adj =kvm_compute_tsc_offset_l1 (vcpu, data) - vcpu->arch.l1_tsc_offset;adjust_tsc_offset_guest (vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; }
@@ -4123,7 +4123,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)mark_tsc_unstable ("KVM discovered backwards TSC" );if (kvm_check_tsc_unstable ()) { -u64 offset =kvm_compute_tsc_offset (vcpu, +u64 offset =kvm_compute_tsc_offset_l1 (vcpu, vcpu->arch.last_guest_tsc);kvm_vcpu_write_tsc_offset (vcpu, offset); vcpu->arch.tsc_catchup =1 ;
Signature · 1 line
2.17.1
PATCH v3 4/12 KVM: X86: Add a ratio parameter to kvm_scale_tsc()
In reply to Ilias StamatisSometimes kvm_scale_tsc() needs to use the current scaling ratio and other times (like when reading the TSC from user space) it needs to use L1's scaling ratio. Have the caller specify this by passing the ratio as a parameter.
--- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/x86.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
@@ -1788,7 +1788,7 @@ static inline bool kvm_is_supported_user_return_msr(u32 msr)return kvm_find_user_return_msr (msr) >=0 ; } -u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc); +u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc,u64 ratio);u64 kvm_read_l1_tsc (struct kvm_vcpu *vcpu,u64 host_tsc);unsigned long kvm_get_linear_rip (struct kvm_vcpu *vcpu);
arch/x86/kvm/x86.c +16 -10
@@ -2307,10 +2307,9 @@ static inline u64 __scale_tsc(u64 ratio, u64 tsc)return mul_u64_u64_shr (tsc, ratio, kvm_tsc_scaling_ratio_frac_bits); } -u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc) +u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc,u64 ratio) {u64 _tsc = tsc; -u64 ratio = vcpu->arch.tsc_scaling_ratio;if (ratio != kvm_default_tsc_scaling_ratio) _tsc =__scale_tsc (ratio, tsc);
@@ -2323,14 +2322,15 @@ static u64 kvm_compute_tsc_offset_l1(struct kvm_vcpu *vcpu, u64 target_tsc) {u64 tsc; - tsc =kvm_scale_tsc (vcpu,rdtsc ()); + tsc =kvm_scale_tsc (vcpu,rdtsc (), vcpu->arch.l1_tsc_scaling_ratio);return target_tsc - tsc; }u64 kvm_read_l1_tsc (struct kvm_vcpu *vcpu,u64 host_tsc) { -return vcpu->arch.l1_tsc_offset +kvm_scale_tsc (vcpu, host_tsc); +return vcpu->arch.l1_tsc_offset + +kvm_scale_tsc (vcpu, host_tsc, vcpu->arch.l1_tsc_scaling_ratio); }EXPORT_SYMBOL_GPL (kvm_read_l1_tsc);
@@ -2463,7 +2463,8 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) {if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)WARN_ON (adjustment <0 ); - adjustment =kvm_scale_tsc (vcpu, (u64 ) adjustment); + adjustment =kvm_scale_tsc (vcpu, (u64 ) adjustment, + vcpu->arch.l1_tsc_scaling_ratio);adjust_tsc_offset_guest (vcpu, adjustment); }
@@ -2846,7 +2847,8 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)/* With all the info we got, fill in the values */ if (kvm_has_tsc_control) - tgt_tsc_khz =kvm_scale_tsc (v, tgt_tsc_khz); + tgt_tsc_khz =kvm_scale_tsc (v, tgt_tsc_khz, + v->arch.l1_tsc_scaling_ratio);if (unlikely (vcpu->hw_tsc_khz != tgt_tsc_khz)) { kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
@@ -3537,10 +3539,14 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)* return L1's TSC value to ensure backwards-compatible * behavior for migration. */ -u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset : - vcpu->arch.tsc_offset; - - msr_info->data =kvm_scale_tsc (vcpu,rdtsc ()) + tsc_offset; +if (msr_info->host_initiated) + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.l1_tsc_scaling_ratio + ) + vcpu->arch.l1_tsc_offset; +else + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.tsc_scaling_ratio + ) + vcpu->arch.tsc_offset;break ; }case MSR_MTRRcap:
Signature · 1 line
2.17.1
PATCH v3 6/12 KVM: X86: Add functions for retrieving L2 TSC fields from common code
In reply to Ilias StamatisIn order to implement as much of the nested TSC scaling logic as possible in common code, we need these vendor callbacks for retrieving the TSC offset and the TSC multiplier that L1 has set for L2.
--- arch/x86/include/asm/kvm-x86-ops.h | 2 ++ arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/svm/svm.c | 14 ++++++++++++++ arch/x86/kvm/vmx/vmx.c | 23 +++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 3 +++ 5 files changed, 44 insertions(+)
arch/x86/include/asm/kvm-x86-ops.h +2 -0
@@ -87,6 +87,8 @@ KVM_X86_OP(set_identity_map_addr) KVM_X86_OP(get_mt_mask) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_NULL(has_wbinvd_exit) +KVM_X86_OP(get_l2_tsc_offset) +KVM_X86_OP(get_l2_tsc_multiplier) KVM_X86_OP(write_l1_tsc_offset) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept)
arch/x86/include/asm/kvm_host.h +2 -0
@@ -1305,6 +1305,8 @@ struct kvm_x86_ops {bool (*has_wbinvd_exit)(void ); +u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); +u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);/* Returns actual tsc_offset set in active VMCS */ u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset);
arch/x86/kvm/svm/svm.c +14 -0
@@ -1082,6 +1082,18 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) seg->base =0 ; } +static u64 svm_get_l2_tsc_offset (struct kvm_vcpu *vcpu) +{ +struct vcpu_svm *svm =to_svm (vcpu); + +return svm->nested.ctl.tsc_offset; +} + +static u64 svm_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu) +{ +return kvm_default_tsc_scaling_ratio; +} +static u64 svm_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {struct vcpu_svm *svm =to_svm (vcpu);
@@ -4526,6 +4538,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .has_wbinvd_exit = svm_has_wbinvd_exit, + .get_l2_tsc_offset = svm_get_l2_tsc_offset, + .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, .write_l1_tsc_offset = svm_write_l1_tsc_offset, .load_mmu_pgd = svm_load_mmu_pgd,
arch/x86/kvm/vmx/vmx.c +23 -0
@@ -1787,6 +1787,27 @@ static void setup_msrs(struct vcpu_vmx *vmx) vmx->guest_uret_msrs_loaded = false; } +u64 vmx_get_l2_tsc_offset (struct kvm_vcpu *vcpu) +{ +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); + +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING )) +return vmcs12->tsc_offset; + +return 0 ; +} + +u64 vmx_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu) +{ +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); + +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING ) && +nested_cpu_has2 (vmcs12,SECONDARY_EXEC_TSC_SCALING )) +return vmcs12->tsc_multiplier; + +return kvm_default_tsc_scaling_ratio; +} +static u64 vmx_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {struct vmcs12 *vmcs12 =get_vmcs12 (vcpu);
@@ -7700,6 +7721,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, + .get_l2_tsc_offset = vmx_get_l2_tsc_offset, + .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, .write_l1_tsc_offset = vmx_write_l1_tsc_offset, .load_mmu_pgd = vmx_load_mmu_pgd,
arch/x86/kvm/vmx/vmx.h +3 -0
@@ -404,6 +404,9 @@ void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);void vmx_disable_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type);void vmx_enable_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type); +u64 vmx_get_l2_tsc_offset (struct kvm_vcpu *vcpu); +u64 vmx_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu); +static inline void vmx_set_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type,bool value) {
Signature · 1 line
2.17.1
PATCH v3 5/12 KVM: VMX: Add a TSC multiplier field in VMCS12
In reply to Ilias StamatisThis is required for supporting nested TSC scaling.
--- arch/x86/kvm/vmx/vmcs12.c | 1 + arch/x86/kvm/vmx/vmcs12.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-)
arch/x86/kvm/vmx/vmcs12.c +1 -0
@@ -37,6 +37,7 @@ const unsigned short vmcs_field_to_offset_table[] = {FIELD64 (VM_ENTRY_MSR_LOAD_ADDR , vm_entry_msr_load_addr),FIELD64 (PML_ADDRESS , pml_address),FIELD64 (TSC_OFFSET , tsc_offset), +FIELD64 (TSC_MULTIPLIER , tsc_multiplier),FIELD64 (VIRTUAL_APIC_PAGE_ADDR , virtual_apic_page_addr),FIELD64 (APIC_ACCESS_ADDR , apic_access_addr), FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
arch/x86/kvm/vmx/vmcs12.h +3 -1
@@ -70,7 +70,8 @@ struct __packed vmcs12 {u64 eptp_list_address;u64 pml_address;u64 encls_exiting_bitmap; -u64 padding64[2 ];/* room for future expansion */ +u64 tsc_multiplier; +u64 padding64[1 ];/* room for future expansion */ /* * To allow migration of L1 (complete with its L2 guests) between * machines of different natural widths (32 or 64 bit), we cannot have
@@ -258,6 +259,7 @@ static inline void vmx_check_vmcs12_offsets(void)CHECK_OFFSET (eptp_list_address,304 );CHECK_OFFSET (pml_address,312 );CHECK_OFFSET (encls_exiting_bitmap,320 ); +CHECK_OFFSET (tsc_multiplier,328 );CHECK_OFFSET (cr0_guest_host_mask,344 );CHECK_OFFSET (cr4_guest_host_mask,352 );CHECK_OFFSET (cr0_read_shadow,360 );
Signature · 1 line
2.17.1
PATCH v3 7/12 KVM: X86: Add functions that calculate L2's TSC offset and multiplier
In reply to Ilias StamatisWhen L2 is entered we need to "merge" the TSC multiplier and TSC offset values of 01 and 12 together.
The merging is done using the following equations:
offset_02 = ((offset_01 * mult_12) >> shift_bits) + offset_12 mult_02 = (mult_01 * mult_12) >> shift_bits
Where shift_bits is kvm_tsc_scaling_ratio_frac_bits.
--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/x86.c | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+)
arch/x86/include/asm/kvm_host.h +2 -0
@@ -1792,6 +1792,8 @@ static inline bool kvm_is_supported_user_return_msr(u32 msr)u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc,u64 ratio);u64 kvm_read_l1_tsc (struct kvm_vcpu *vcpu,u64 host_tsc); +u64 kvm_calc_nested_tsc_offset (u64 l1_offset,u64 l2_offset,u64 l2_multiplier); +u64 kvm_calc_nested_tsc_multiplier (u64 l1_multiplier,u64 l2_multiplier);unsigned long kvm_get_linear_rip (struct kvm_vcpu *vcpu);bool kvm_is_linear_rip (struct kvm_vcpu *vcpu,unsigned long linear_rip);
arch/x86/kvm/x86.c +25 -0
@@ -2334,6 +2334,31 @@ u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) }EXPORT_SYMBOL_GPL (kvm_read_l1_tsc); +u64 kvm_calc_nested_tsc_offset (u64 l1_offset,u64 l2_offset,u64 l2_multiplier) +{ +u64 nested_offset; + +if (l2_multiplier == kvm_default_tsc_scaling_ratio) + nested_offset = l1_offset; +else + nested_offset =mul_s64_u64_shr ((s64 ) l1_offset, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + + nested_offset += l2_offset; +return nested_offset; +} +EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_offset); + +u64 kvm_calc_nested_tsc_multiplier (u64 l1_multiplier,u64 l2_multiplier) +{ +if (l2_multiplier != kvm_default_tsc_scaling_ratio) +return mul_u64_u64_shr (l1_multiplier, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + +return l1_multiplier; +} +EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_multiplier); +static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) { vcpu->arch.l1_tsc_offset = offset;
Signature · 1 line
2.17.1
PATCH v3 9/12 KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Ilias StamatisThe vmx->current_tsc_ratio field is redundant as vcpu->arch.tsc_scaling_ratio already tracks the current TSC scaling ratio. Removing this field makes decache_tsc_multiplier() an one-liner so remove that too and do a vmcs_write64() directly in order to be more consistent with surrounding code.
--- arch/x86/kvm/vmx/nested.c | 9 ++++----- arch/x86/kvm/vmx/vmx.c | 5 ++--- arch/x86/kvm/vmx/vmx.h | 8 -------- 3 files changed, 6 insertions(+), 16 deletions(-)
arch/x86/kvm/vmx/nested.c +4 -5
@@ -2533,9 +2533,8 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, }vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset); -if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);nested_vmx_transition_tlb_flush (vcpu, vmcs12, true);
@@ -4501,12 +4500,12 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,vmcs_write32 (VM_EXIT_MSR_LOAD_COUNT , vmx->msr_autoload.host.nr);vmcs_write32 (VM_ENTRY_MSR_LOAD_COUNT , vmx->msr_autoload.guest.nr);vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio); +if (vmx->nested.l1_tpr_threshold !=-1 )vmcs_write32 (TPR_THRESHOLD , vmx->nested.l1_tpr_threshold); -if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); -if (vmx->nested.change_vmcs01_virtual_apic_mode) { vmx->nested.change_vmcs01_virtual_apic_mode = false;vmx_set_virtual_apic_mode (vcpu);
arch/x86/kvm/vmx/vmx.c +2 -3
@@ -1392,9 +1392,8 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, }/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio); }/*
arch/x86/kvm/vmx/vmx.h +0 -8
@@ -322,8 +322,6 @@ struct vcpu_vmx {/* apic deadline value in host tsc */ u64 hv_deadline_tsc; -u64 current_tsc_ratio; -unsigned long host_debugctlmsr;/*
@@ -532,12 +530,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow) GFP_KERNEL_ACCOUNT); } -static inline void decache_tsc_multiplier (struct vcpu_vmx *vmx) -{ - vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio; -vmcs_write64 (TSC_MULTIPLIER , vmx->current_tsc_ratio); -} -static inline bool vmx_has_waitpkg (struct vcpu_vmx *vmx) { return vmx->secondary_exec_control &
Signature · 1 line
2.17.1
PATCH v3 8/12 KVM: X86: Move write_l1_tsc_offset() logic to common code and rename it
In reply to Ilias StamatisThe write_l1_tsc_offset() callback has a misleading name. It does not set L1's TSC offset, it rather updates the current TSC offset which might be different if a nested guest is executing. Additionally, both the vmx and svm implementations use the same logic for calculating the current TSC before writing it to hardware.
Rename the function and move the common logic to the caller. The vmx/svm specific code now merely sets the given offset to the corresponding hardware structure.
--- arch/x86/include/asm/kvm-x86-ops.h | 2 +- arch/x86/include/asm/kvm_host.h | 3 +-- arch/x86/kvm/svm/svm.c | 21 ++++----------------- arch/x86/kvm/vmx/vmx.c | 23 +++-------------------- arch/x86/kvm/x86.c | 24 +++++++++++++++++++++--- 5 files changed, 30 insertions(+), 43 deletions(-)
arch/x86/include/asm/kvm-x86-ops.h +1 -1
@@ -89,7 +89,7 @@ KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_NULL(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) -KVM_X86_OP(write_l1_tsc_offset) +KVM_X86_OP(write_tsc_offset) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff)
arch/x86/include/asm/kvm_host.h +1 -2
@@ -1307,8 +1307,7 @@ struct kvm_x86_ops {u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); -/* Returns actual tsc_offset set in active VMCS */ -u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset); +void (*write_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset);/* * Retrieve somewhat arbitrary exit information. Intended to be used
arch/x86/kvm/svm/svm.c +4 -17
@@ -1094,26 +1094,13 @@ static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)return kvm_default_tsc_scaling_ratio; } -static u64 svm_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void svm_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {struct vcpu_svm *svm =to_svm (vcpu); -u64 g_tsc_offset =0 ; - -if (is_guest_mode (vcpu)) { -/* Write L1's TSC offset. */ - g_tsc_offset = svm->vmcb->control.tsc_offset - - svm->vmcb01.ptr->control.tsc_offset; - svm->vmcb01.ptr->control.tsc_offset = offset; - } - -trace_kvm_write_tsc_offset (vcpu->vcpu_id, - svm->vmcb->control.tsc_offset - g_tsc_offset, - offset); - - svm->vmcb->control.tsc_offset = offset + g_tsc_offset; + svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; + svm->vmcb->control.tsc_offset = offset;vmcb_mark_dirty (svm->vmcb,VMCB_INTERCEPTS ); -return svm->vmcb->control.tsc_offset; }/* Evaluate instruction intercepts that depend on guest CPUID features. */
@@ -4540,7 +4527,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_l2_tsc_offset = svm_get_l2_tsc_offset, .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, - .write_l1_tsc_offset = svm_write_l1_tsc_offset, + .write_tsc_offset = svm_write_tsc_offset, .load_mmu_pgd = svm_load_mmu_pgd,
arch/x86/kvm/vmx/vmx.c +3 -20
@@ -1808,26 +1808,9 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)return kvm_default_tsc_scaling_ratio; } -static u64 vmx_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void vmx_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) { -struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); -u64 g_tsc_offset =0 ; - -/* -* We're here if L1 chose not to trap WRMSR to TSC. According -* to the spec, this should set L1's TSC; The offset that L1 -* set for L2 remains unchanged, and still needs to be added -* to the newly set TSC to get L2's TSC. -*/ -if (is_guest_mode (vcpu) && - (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING )) - g_tsc_offset = vmcs12->tsc_offset; - -trace_kvm_write_tsc_offset (vcpu->vcpu_id, - vcpu->arch.tsc_offset - g_tsc_offset, - offset); -vmcs_write64 (TSC_OFFSET , offset + g_tsc_offset); -return offset + g_tsc_offset; +vmcs_write64 (TSC_OFFSET , offset); }/*
@@ -7723,7 +7706,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .get_l2_tsc_offset = vmx_get_l2_tsc_offset, .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, - .write_l1_tsc_offset = vmx_write_l1_tsc_offset, + .write_tsc_offset = vmx_write_tsc_offset, .load_mmu_pgd = vmx_load_mmu_pgd,
arch/x86/kvm/x86.c +21 -3
@@ -2359,10 +2359,28 @@ u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) }EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_multiplier); -static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 l1_offset) { - vcpu->arch.l1_tsc_offset = offset; - vcpu->arch.tsc_offset =static_call (kvm_x86_write_l1_tsc_offset)(vcpu, offset); +trace_kvm_write_tsc_offset (vcpu->vcpu_id, + vcpu->arch.l1_tsc_offset, + l1_offset); + + vcpu->arch.l1_tsc_offset = l1_offset; + +/* +* If we are here because L1 chose not to trap WRMSR to TSC then +* according to the spec this should set L1's TSC (as opposed to +* setting L1's offset for L2). +*/ +if (is_guest_mode (vcpu)) + vcpu->arch.tsc_offset =kvm_calc_nested_tsc_offset ( + l1_offset, +static_call (kvm_x86_get_l2_tsc_offset)(vcpu), +static_call (kvm_x86_get_l2_tsc_multiplier)(vcpu)); +else + vcpu->arch.tsc_offset = l1_offset; + +static_call (kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); }static inline bool kvm_check_tsc_unstable (void )
Signature · 1 line
2.17.1
PATCH v3 10/12 KVM: VMX: Set the TSC offset and multiplier on nested entry and exit
In reply to Ilias StamatisCalculate the nested TSC offset and multiplier on entering L2 using the corresponding functions. Restore the L1 values on L2's exit.
--- arch/x86/kvm/vmx/nested.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)
arch/x86/kvm/vmx/nested.c +14 -4
@@ -2532,6 +2532,15 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,vmcs_write64 (GUEST_IA32_PAT , vmx->vcpu.arch.pat); } + vcpu->arch.tsc_offset =kvm_calc_nested_tsc_offset ( + vcpu->arch.l1_tsc_offset, +vmx_get_l2_tsc_offset (vcpu), +vmx_get_l2_tsc_multiplier (vcpu)); + + vcpu->arch.tsc_scaling_ratio =kvm_calc_nested_tsc_multiplier ( + vcpu->arch.l1_tsc_scaling_ratio, +vmx_get_l2_tsc_multiplier (vcpu)); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset);if (kvm_has_tsc_control)vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);
@@ -3353,8 +3362,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, }enter_guest_mode (vcpu); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset += vmcs12->tsc_offset;if (prepare_vmcs02 (vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic =EXIT_REASON_INVALID_STATE ;
@@ -4462,8 +4469,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,if (nested_cpu_has_preemption_timer (vmcs12))hrtimer_cancel (&to_vmx (vcpu)->nested.preemption_timer); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset -= vmcs12->tsc_offset; +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING )) { + vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; +if (nested_cpu_has2 (vmcs12,SECONDARY_EXEC_TSC_SCALING )) + vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; + }if (likely (!vmx->fail)) {sync_vmcs02_to_vmcs12 (vcpu, vmcs12);
Signature · 1 line
2.17.1
PATCH v3 12/12 KVM: selftests: x86: Add vmx_nested_tsc_scaling_test
In reply to Ilias StamatisTest that nested TSC scaling works as expected with both L1 and L2 scaled.
--- tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/vmx_nested_tsc_scaling_test.c | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c
tools/testing/selftests/kvm/.gitignore +1 -0
@@ -29,6 +29,7 @@ /x86_64/vmx_preemption_timer_test /x86_64/vmx_set_nested_state_test /x86_64/vmx_tsc_adjust_test +/x86_64/vmx_nested_tsc_scaling_test /x86_64/xapic_ipi_test /x86_64/xen_shinfo_test /x86_64/xen_vmcall_test
tools/testing/selftests/kvm/Makefile +1 -0
@@ -60,6 +60,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test +TEST_GEN_PROGS_x86_64 += x86_64/vmx_nested_tsc_scaling_test TEST_GEN_PROGS_x86_64 += x86_64/xapic_ipi_test TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test TEST_GEN_PROGS_x86_64 += x86_64/debug_regs
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file +242 -0
242 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 11/12 KVM: VMX: Expose TSC scaling to L2
In reply to Ilias StamatisExpose the TSC scaling feature to nested guests.
--- arch/x86/kvm/vmx/nested.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
arch/x86/kvm/vmx/nested.c +4 -2
@@ -2277,7 +2277,8 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |SECONDARY_EXEC_APIC_REGISTER_VIRT | - SECONDARY_EXEC_ENABLE_VMFUNC); +SECONDARY_EXEC_ENABLE_VMFUNC | + SECONDARY_EXEC_TSC_SCALING);if (nested_cpu_has (vmcs12,CPU_BASED_ACTIVATE_SECONDARY_CONTROLS )) exec_control |= vmcs12->secondary_vm_exec_control;
@@ -6483,7 +6484,8 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps)SECONDARY_EXEC_RDRAND_EXITING |SECONDARY_EXEC_ENABLE_INVPCID |SECONDARY_EXEC_RDSEED_EXITING | -SECONDARY_EXEC_XSAVES ; +SECONDARY_EXEC_XSAVES | +SECONDARY_EXEC_TSC_SCALING ;/* * We can emulate "VMCS shadowing," even if the hardware
Signature · 1 line
2.17.1
Paolo Bonzini
maintainerRe: [PATCH v3 03/12] KVM: X86: Rename kvm_compute_tsc_offset() to kvm_compute_tsc_offset_l1()
In reply to Ilias Stamatisarch/x86/kvm/x86.c int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) · line 3238
Quoted 1 line · Ilias Stamatis
arch/x86/kvm/x86.c +1
-0+u64 adj =kvm_compute_tsc_offset_l1 (vcpu, data) - vcpu->arch.l1_tsc_offset;
Better: kvm_compute_l1_tsc_offset. So far anyway I can adjust this myself.
Paolo
Paolo Bonzini
maintainerRe: [PATCH v3 04/12] KVM: X86: Add a ratio parameter to kvm_scale_tsc()
In reply to Ilias Stamatisarch/x86/kvm/x86.c int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) · line 3545
Quoted 12 lines · Ilias Stamatis
arch/x86/kvm/x86.c +4
-4@@ -3537,10 +3539,14 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)* return L1's TSC value to ensure backwards-compatible * behavior for migration. */ -u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset : - vcpu->arch.tsc_offset; - - msr_info->data =kvm_scale_tsc (vcpu,rdtsc ()) + tsc_offset; +if (msr_info->host_initiated) + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.l1_tsc_scaling_ratio + ) + vcpu->arch.l1_tsc_offset;
Better indentation:
msr_info->data = vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.tsc_scaling_ratio);
Same below.
Paolo
Quoted 8 lines
arch/x86/kvm/x86.c +4
-0+else + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.tsc_scaling_ratio + ) + vcpu->arch.tsc_offset;break ; }case MSR_MTRRcap:
Quoted 56 lines · Ilias Stamatis
KVM currently supports hardware-assisted TSC scaling but only for L1; the feature is not exposed to nested guests. This patch series adds support for nested TSC scaling and allows both L1 and L2 to be scaled with different scaling factors. That is achieved by "merging" the 01 and 02 values together.
Most of the logic in this series is implemented in common code (by doing the necessary restructurings), however the patches add support for VMX only. Adding support for SVM should be easy at this point and Maxim Levitsky has volunteered to do this (thanks!).
Changelog:
v3:
- Applied Sean's feedback
- Refactored patches 7 to 10v2:
- Applied all of Maxim's feedback
- Added amul_s64_u64_shrfunction inmath64.h
- Added a separatekvm_scale_tsc_l1function instead of passing an argument tokvm_scale_tsc
- Implemented the 02 fields calculations in common code
- Moved all of write_l1_tsc_offset's logic to common code
- Added a check for whether the TSC is stable in patch 10
- Used a random L1 factor and a negative offset in patch 10Ilias Stamatis (12):
math64.h: Addmul_s64_u64_shr()
KVM: X86: Store L1's TSC scaling ratio in 'structkvm_vcpu_arch'
KVM: X86: Renamekvm_compute_tsc_offset()tokvm_compute_tsc_offset_l1()KVM: X86: Add a ratio parameter to
kvm_scale_tsc()
KVM: VMX: Add a TSC multiplier field in VMCS12
KVM: X86: Add functions for retrieving L2 TSC fields from common code
KVM: X86: Add functions that calculate L2's TSC offset and multiplier
KVM: X86: Movewrite_l1_tsc_offset()logic to common code and rename
it
KVM: VMX: Removevmx->current_tsc_ratioanddecache_tsc_multiplier()
KVM: VMX: Set the TSC offset and multiplier on nested entry and exit
KVM: VMX: Expose TSC scaling to L2
KVM: selftests: x86: Addvmx_nested_tsc_scaling_testarch/x86/include/asm/kvm-x86-ops.h | 4 +- arch/x86/include/asm/kvm_host.h | 14 +- arch/x86/kvm/svm/svm.c | 29 ++- arch/x86/kvm/vmx/nested.c | 33 ++- arch/x86/kvm/vmx/vmcs12.c | 1 + arch/x86/kvm/vmx/vmcs12.h | 4 +- arch/x86/kvm/vmx/vmx.c | 49 ++-- arch/x86/kvm/vmx/vmx.h | 11 +- arch/x86/kvm/x86.c | 91 +++++-- include/linux/math64.h | 19 ++ tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/vmx_nested_tsc_scaling_test.c | 242 ++++++++++++++++++ 13 files changed, 417 insertions(+), 82 deletions(-)
Quoted 5 lines
create mode 100644
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c--
2.17.1
Queued, thanks.
The new kvm_x86_ops should go in kvm_x86_ops.nested, but those are not yet static_calls so we can leave that for later.
Paolo
Paolo Bonzini
maintainerRe: [PATCH v3 04/12] KVM: X86: Add a ratio parameter to kvm_scale_tsc()
In reply to Sean ChristophersonQuoted 9 lines · Sean Christopherson
if (msr_info->host_initiated) {
offset = vcpu->arch.l1_tsc_offset;
ratio = vcpu->arch.l1_tsc_scaling_ratio;
} else {
offset = vcpu->arch.tsc_offset;
ratio = vcpu->arch.tsc_scaling_ratio;
}
msr_info->data = kvm_scale_tsc(vcpu, rdtsc(), ratio) + offset;Looks good, indeed I didn't do this just out of laziness really (and instead got a typo).
Paolo
Sean Christopherson
maintainerRe: [PATCH v3 04/12] KVM: X86: Add a ratio parameter to kvm_scale_tsc()
In reply to Paolo BonziniQuoted 16 lines · Paolo Bonzini
arch/x86/kvm/x86.c int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) · line 3545
Quoted 12 lines · Ilias Stamatis
arch/x86/kvm/x86.c +4
-4@@ -3537,10 +3539,14 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)* return L1's TSC value to ensure backwards-compatible * behavior for migration. */ -u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset : - vcpu->arch.tsc_offset; - - msr_info->data =kvm_scale_tsc (vcpu,rdtsc ()) + tsc_offset; +if (msr_info->host_initiated) + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.l1_tsc_scaling_ratio + ) + vcpu->arch.l1_tsc_offset;Better indentation:
Quoted 2 lines
msr_info->data = vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, rdtsc(),
I like this more, but it's still a bit ugly.
Quoted 1 line
vcpu->arch.tsc_scaling_ratio);
Wrong pairing of ratio and offset (not necessarily what you queued, obviously).
Quoted 2 lines
Same below.
Alternatively, what about something like this?
u64 offset, ratio;
if (msr_info->host_initiated) {
offset = vcpu->arch.l1_tsc_offset;
ratio = vcpu->arch.l1_tsc_scaling_ratio;
} else {
offset = vcpu->arch.tsc_offset;
ratio = vcpu->arch.tsc_scaling_ratio;
}
msr_info->data = kvm_scale_tsc(vcpu, rdtsc(), ratio) + offset;or an option that would require additional refactoring:
struct kvm_guest_tsc *tsc; tsc = msr_info->host_initiated ? &vcpu->arch.l1_tsc : &vcpu->arch.current_tsc; msr_info->data = tsc->offset + kvm_scale_tsc(vcpu, rdtsc(), tsc->scaling_ratio);
Quoted 12 lines
Paolo
Quoted 8 lines
arch/x86/kvm/x86.c +4
-0+else + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.tsc_scaling_ratio + ) + vcpu->arch.tsc_offset;break ; }case MSR_MTRRcap:
Re: [PATCH v3 01/12] math64.h: Add mul_s64_u64_shr()
In reply to Ilias StamatisQuoted 36 lines · Ilias Stamatis
This function is needed for KVM's nested virtualization. The nested TSC scaling implementation requires multiplying the signed TSC offset with the unsigned TSC multiplier.
Signed-off-by Ilias Stamatis <[email protected]>--- include/linux/math64.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+)include/linux/math64.h +12
-0@@ -3,6 +3,7 @@#define _LINUX_MATH64_H#include <linux/types.h> +#include <linux/math.h> #include <vdso/math64.h> #include <asm/div64.h> @@ -234,6 +235,24 @@ static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)#endif +#ifndef mul_s64_u64_shr +static inline u64 mul_s64_u64_shr (s64 a,u64 b,unsigned int shift) +{ +u64 ret; + +/* +* Extract the sign before the multiplication and put it back +* afterwards if needed. +*/ + ret =mul_u64_u64_shr (abs (a), b, shift); +
include/linux/math64.h static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift) · line 258
Quoted 10 lines
include/linux/math64.h +7
-0+if (a <0 ) + ret = -((s64 ) ret); + +return ret; +} +#endif /* mul_s64_u64_shr */ +#ifndef mul_u64_u32_divstatic inline u64 mul_u64_u32_div (u64 a,u32 mul,u32 divisor) {
Best regards,
Maxim Levitsky
Re: [PATCH v3 02/12] KVM: X86: Store L1's TSC scaling ratio in 'struct kvm_vcpu_arch'
In reply to Ilias StamatisQuoted 81 lines · Ilias Stamatis
Store L1's scaling ratio in the
kvm_vcpu_archstruct like we already do for L1's TSC offset. This allows for easy save/restore when we enter and then exit the nested guest.Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/include/asm/kvm_host.h | 5 +++-- arch/x86/kvm/vmx/vmx.c | 4 ++-- arch/x86/kvm/x86.c | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-)arch/x86/include/asm/kvm_host.h +3
-2@@ -707,7 +707,7 @@ struct kvm_vcpu_arch { } st;u64 l1_tsc_offset; -u64 tsc_offset; +u64 tsc_offset;/* current tsc offset */ u64 last_guest_tsc;u64 last_host_tsc;u64 tsc_offset_adjustment;@@ -721,7 +721,8 @@ struct kvm_vcpu_arch {u32 virtual_tsc_khz;s64 ia32_tsc_adjust_msr;u64 msr_ia32_power_ctl; -u64 tsc_scaling_ratio; +u64 l1_tsc_scaling_ratio; +u64 tsc_scaling_ratio;/* current scaling ratio */ atomic_t nmi_queued;/* unprocessed asynchronous NMIs */ unsigned nmi_pending;/* NMI queued after currently running handler */ arch/x86/kvm/vmx/vmx.c +2
-2@@ -7453,10 +7453,10 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, delta_tsc =0 ;/* Convert to host delta tsc if tsc scaling is enabled */ -if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && +if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && delta_tsc &&u64_shl_div_u64 (delta_tsc, kvm_tsc_scaling_ratio_frac_bits, - vcpu->arch.tsc_scaling_ratio, &delta_tsc)) + vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc))return -ERANGE ;/* arch/x86/kvm/x86.c +3
-1@@ -2185,6 +2185,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)/* Guest TSC same frequency as host TSC? */ if (!scale) { + vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;return 0 ; }@@ -2211,7 +2212,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)return -1 ; } - vcpu->arch.tsc_scaling_ratio = ratio; + vcpu->arch.l1_tsc_scaling_ratio = vcpu->arch.tsc_scaling_ratio = ratio;return 0 ; }@@ -2223,6 +2224,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)/* tsc_khz can be zero if TSC calibration fails */ if (user_tsc_khz ==0 ) {/* set tsc_scaling_ratio to a safe value */ + vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;return -1 ; }
arch/x86/kvm/x86.c static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, · line 2467
Quoted 8 lines
arch/x86/kvm/x86.c +1
-1@@ -2459,7 +2461,7 @@ static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,static inline void adjust_tsc_offset_host (struct kvm_vcpu *vcpu,s64 adjustment) { -if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) +if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)WARN_ON (adjustment <0 ); adjustment =kvm_scale_tsc (vcpu, (u64 ) adjustment);adjust_tsc_offset_guest (vcpu, adjustment);
Best regards,
Maxim Levitsky
Maxim Levitsky
Reviewed-byRe: [PATCH v3 03/12] KVM: X86: Rename kvm_compute_tsc_offset() to kvm_compute_tsc_offset_l1()
In reply to Paolo BonziniQuoted 4 lines · Paolo Bonzini
arch/x86/kvm/x86.c int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) · line 3238
Quoted 1 line · Ilias Stamatis
arch/x86/kvm/x86.c +1
-0+u64 adj =kvm_compute_tsc_offset_l1 (vcpu, data) - vcpu->arch.l1_tsc_offset;Better:
kvm_compute_l1_tsc_offset. So far anyway I can adjust this myself.
Nothing against this either!
Quoted 3 lines
Paolo
Re: [PATCH v3 06/12] KVM: X86: Add functions for retrieving L2 TSC fields from common code
In reply to Ilias StamatisQuoted 47 lines · Ilias Stamatis
In order to implement as much of the nested TSC scaling logic as possible in common code, we need these vendor callbacks for retrieving the TSC offset and the TSC multiplier that L1 has set for L2.
Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/include/asm/kvm-x86-ops.h | 2 ++ arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/svm/svm.c | 14 ++++++++++++++ arch/x86/kvm/vmx/vmx.c | 23 +++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 3 +++ 5 files changed, 44 insertions(+)arch/x86/include/asm/kvm-x86-ops.h +2
-0@@ -87,6 +87,8 @@ KVM_X86_OP(set_identity_map_addr) KVM_X86_OP(get_mt_mask) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_NULL(has_wbinvd_exit) +KVM_X86_OP(get_l2_tsc_offset) +KVM_X86_OP(get_l2_tsc_multiplier) KVM_X86_OP(write_l1_tsc_offset) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept)arch/x86/include/asm/kvm_host.h +2
-0@@ -1305,6 +1305,8 @@ struct kvm_x86_ops {bool (*has_wbinvd_exit)(void ); +u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); +u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);/* Returns actual tsc_offset set in active VMCS */ u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset);arch/x86/kvm/svm/svm.c +0
-0@@ -1082,6 +1082,18 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) seg->base =0 ; }
arch/x86/kvm/svm/svm.c static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) · line 1094
Quoted 10 lines
arch/x86/kvm/svm/svm.c +10
-0+static u64 svm_get_l2_tsc_offset (struct kvm_vcpu *vcpu) +{ +struct vcpu_svm *svm =to_svm (vcpu); + +return svm->nested.ctl.tsc_offset; +} + +static u64 svm_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu) +{ +return kvm_default_tsc_scaling_ratio;
I guess this makes sense as long as we don't support nested TSC scaling on SVM. Or put a WARN_ON here maybe instead. Doesn't matter, as I'll implement this right after this series is done, which should be very easy.
Quoted 61 lines
arch/x86/kvm/svm/svm.c +4
-0+} +static u64 svm_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {struct vcpu_svm *svm =to_svm (vcpu);@@ -4526,6 +4538,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .has_wbinvd_exit = svm_has_wbinvd_exit, + .get_l2_tsc_offset = svm_get_l2_tsc_offset, + .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, .write_l1_tsc_offset = svm_write_l1_tsc_offset, .load_mmu_pgd = svm_load_mmu_pgd,arch/x86/kvm/vmx/vmx.c +21
-0@@ -1787,6 +1787,27 @@ static void setup_msrs(struct vcpu_vmx *vmx) vmx->guest_uret_msrs_loaded = false; } +u64 vmx_get_l2_tsc_offset (struct kvm_vcpu *vcpu) +{ +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); + +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING )) +return vmcs12->tsc_offset; + +return 0 ; +} + +u64 vmx_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu) +{ +struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); + +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING ) && +nested_cpu_has2 (vmcs12,SECONDARY_EXEC_TSC_SCALING )) +return vmcs12->tsc_multiplier; + +return kvm_default_tsc_scaling_ratio; +} +static u64 vmx_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {arch/x86/kvm/vmx/vmx.c +2
-0struct vmcs12 *vmcs12 =get_vmcs12 (vcpu);@@ -7700,6 +7721,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, + .get_l2_tsc_offset = vmx_get_l2_tsc_offset, + .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, .write_l1_tsc_offset = vmx_write_l1_tsc_offset, .load_mmu_pgd = vmx_load_mmu_pgd,arch/x86/kvm/vmx/vmx.h +0
-0
arch/x86/kvm/vmx/vmx.h void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu); · line 412
Quoted 9 lines
arch/x86/kvm/vmx/vmx.h +3
-0@@ -404,6 +404,9 @@ void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);void vmx_disable_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type);void vmx_enable_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type); +u64 vmx_get_l2_tsc_offset (struct kvm_vcpu *vcpu); +u64 vmx_get_l2_tsc_multiplier (struct kvm_vcpu *vcpu); +static inline void vmx_set_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type,bool value) {
Looks great.
Best regards,
Maxim Levitsky
Maxim Levitsky
Reviewed-byRe: [PATCH v3 04/12] KVM: X86: Add a ratio parameter to kvm_scale_tsc()
In reply to Paolo BonziniQuoted 16 lines · Paolo Bonzini
arch/x86/kvm/x86.c int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) · line 3545
Quoted 12 lines · Ilias Stamatis
arch/x86/kvm/x86.c +4
-4@@ -3537,10 +3539,14 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)* return L1's TSC value to ensure backwards-compatible * behavior for migration. */ -u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset : - vcpu->arch.tsc_offset; - - msr_info->data =kvm_scale_tsc (vcpu,rdtsc ()) + tsc_offset; +if (msr_info->host_initiated) + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.l1_tsc_scaling_ratio + ) + vcpu->arch.l1_tsc_offset;Better indentation:
Quoted 3 lines
msr_info->data = vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.tsc_scaling_ratio);
Good idea IMHO.
Other than that:
Quoted 13 lines
Same below.
Paolo
Quoted 8 lines
arch/x86/kvm/x86.c +4
-0+else + msr_info->data =kvm_scale_tsc ( + vcpu,rdtsc (), vcpu->arch.tsc_scaling_ratio + ) + vcpu->arch.tsc_offset;break ; }case MSR_MTRRcap:
Re: [PATCH v3 07/12] KVM: X86: Add functions that calculate L2's TSC offset and multiplier
In reply to Ilias StamatisQuoted 40 lines · Ilias Stamatis
When L2 is entered we need to "merge" the TSC multiplier and TSC offset values of 01 and 12 together.
The merging is done using the following equations:
offset_02= ((offset_01*mult_12) >>shift_bits) +offset_12mult_02= (mult_01*mult_12) >>shift_bitsWhere
shift_bitsiskvm_tsc_scaling_ratio_frac_bits.Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/x86.c | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+)arch/x86/include/asm/kvm_host.h +2
-0@@ -1792,6 +1792,8 @@ static inline bool kvm_is_supported_user_return_msr(u32 msr)u64 kvm_scale_tsc (struct kvm_vcpu *vcpu,u64 tsc,u64 ratio);u64 kvm_read_l1_tsc (struct kvm_vcpu *vcpu,u64 host_tsc); +u64 kvm_calc_nested_tsc_offset (u64 l1_offset,u64 l2_offset,u64 l2_multiplier); +u64 kvm_calc_nested_tsc_multiplier (u64 l1_multiplier,u64 l2_multiplier);unsigned long kvm_get_linear_rip (struct kvm_vcpu *vcpu);bool kvm_is_linear_rip (struct kvm_vcpu *vcpu,unsigned long linear_rip);arch/x86/kvm/x86.c +4
-0@@ -2334,6 +2334,31 @@ u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) }EXPORT_SYMBOL_GPL (kvm_read_l1_tsc); +u64 kvm_calc_nested_tsc_offset (u64 l1_offset,u64 l2_offset,u64 l2_multiplier) +{ +u64 nested_offset; +
arch/x86/kvm/x86.c u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) · line 2350
Quoted 10 lines
arch/x86/kvm/x86.c +10
-0+if (l2_multiplier == kvm_default_tsc_scaling_ratio) + nested_offset = l1_offset; +else + nested_offset =mul_s64_u64_shr ((s64 ) l1_offset, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + + nested_offset += l2_offset; +return nested_offset; +} +EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_offset);
Looks OK.
arch/x86/kvm/x86.c u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) · line 2360
Quoted 10 lines
arch/x86/kvm/x86.c +10
-0+ +u64 kvm_calc_nested_tsc_multiplier (u64 l1_multiplier,u64 l2_multiplier) +{ +if (l2_multiplier != kvm_default_tsc_scaling_ratio) +return mul_u64_u64_shr (l1_multiplier, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + +return l1_multiplier; +} +EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_multiplier);
Looks OK as well.
arch/x86/kvm/x86.c u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) · line 2364
Quoted 4 lines
arch/x86/kvm/x86.c +1
-0+static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) { vcpu->arch.l1_tsc_offset = offset;
Best regards,
Maxim Levitsky
Re: [PATCH v3 08/12] KVM: X86: Move write_l1_tsc_offset() logic to common code and rename it
In reply to Ilias StamatisQuoted 157 lines · Ilias Stamatis
The
write_l1_tsc_offset()callback has a misleading name. It does not set L1's TSC offset, it rather updates the current TSC offset which might be different if a nested guest is executing. Additionally, both the vmx and svm implementations use the same logic for calculating the current TSC before writing it to hardware.Rename the function and move the common logic to the caller. The vmx/svm specific code now merely sets the given offset to the corresponding hardware structure.
Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/include/asm/kvm-x86-ops.h | 2 +- arch/x86/include/asm/kvm_host.h | 3 +-- arch/x86/kvm/svm/svm.c | 21 ++++----------------- arch/x86/kvm/vmx/vmx.c | 23 +++-------------------- arch/x86/kvm/x86.c | 24 +++++++++++++++++++++--- 5 files changed, 30 insertions(+), 43 deletions(-)arch/x86/include/asm/kvm-x86-ops.h +1
-1@@ -89,7 +89,7 @@ KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_NULL(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) -KVM_X86_OP(write_l1_tsc_offset) +KVM_X86_OP(write_tsc_offset) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff)arch/x86/include/asm/kvm_host.h +1
-2@@ -1307,8 +1307,7 @@ struct kvm_x86_ops {u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); -/* Returns actual tsc_offset set in active VMCS */ -u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset); +void (*write_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset);/* * Retrieve somewhat arbitrary exit information. Intended to be used arch/x86/kvm/svm/svm.c +4
-17@@ -1094,26 +1094,13 @@ static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)return kvm_default_tsc_scaling_ratio; } -static u64 svm_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void svm_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) {struct vcpu_svm *svm =to_svm (vcpu); -u64 g_tsc_offset =0 ; - -if (is_guest_mode (vcpu)) { -/* Write L1's TSC offset. */ - g_tsc_offset = svm->vmcb->control.tsc_offset - - svm->vmcb01.ptr->control.tsc_offset; - svm->vmcb01.ptr->control.tsc_offset = offset; - } - -trace_kvm_write_tsc_offset (vcpu->vcpu_id, - svm->vmcb->control.tsc_offset - g_tsc_offset, - offset); - - svm->vmcb->control.tsc_offset = offset + g_tsc_offset; + svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; + svm->vmcb->control.tsc_offset = offset;vmcb_mark_dirty (svm->vmcb,VMCB_INTERCEPTS ); -return svm->vmcb->control.tsc_offset; }/* Evaluate instruction intercepts that depend on guest CPUID features. */ @@ -4540,7 +4527,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_l2_tsc_offset = svm_get_l2_tsc_offset, .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, - .write_l1_tsc_offset = svm_write_l1_tsc_offset, + .write_tsc_offset = svm_write_tsc_offset, .load_mmu_pgd = svm_load_mmu_pgd,arch/x86/kvm/vmx/vmx.c +3
-20@@ -1808,26 +1808,9 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)return kvm_default_tsc_scaling_ratio; } -static u64 vmx_write_l1_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void vmx_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) { -struct vmcs12 *vmcs12 =get_vmcs12 (vcpu); -u64 g_tsc_offset =0 ; - -/* -* We're here if L1 chose not to trap WRMSR to TSC. According -* to the spec, this should set L1's TSC; The offset that L1 -* set for L2 remains unchanged, and still needs to be added -* to the newly set TSC to get L2's TSC. -*/ -if (is_guest_mode (vcpu) && - (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING )) - g_tsc_offset = vmcs12->tsc_offset; - -trace_kvm_write_tsc_offset (vcpu->vcpu_id, - vcpu->arch.tsc_offset - g_tsc_offset, - offset); -vmcs_write64 (TSC_OFFSET , offset + g_tsc_offset); -return offset + g_tsc_offset; +vmcs_write64 (TSC_OFFSET , offset); }/* @@ -7723,7 +7706,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .get_l2_tsc_offset = vmx_get_l2_tsc_offset, .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, - .write_l1_tsc_offset = vmx_write_l1_tsc_offset, + .write_tsc_offset = vmx_write_tsc_offset, .load_mmu_pgd = vmx_load_mmu_pgd,arch/x86/kvm/x86.c +14
-3@@ -2359,10 +2359,28 @@ u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) }EXPORT_SYMBOL_GPL (kvm_calc_nested_tsc_multiplier); -static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 offset) +static void kvm_vcpu_write_tsc_offset (struct kvm_vcpu *vcpu,u64 l1_offset) { - vcpu->arch.l1_tsc_offset = offset; - vcpu->arch.tsc_offset =static_call (kvm_x86_write_l1_tsc_offset)(vcpu, offset); +trace_kvm_write_tsc_offset (vcpu->vcpu_id, + vcpu->arch.l1_tsc_offset, + l1_offset); + + vcpu->arch.l1_tsc_offset = l1_offset; + +/* +* If we are here because L1 chose not to trap WRMSR to TSC then +* according to the spec this should set L1's TSC (as opposed to +* setting L1's offset for L2). +*/ +if (is_guest_mode (vcpu)) + vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
arch/x86/kvm/x86.c u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) · line 2386
Quoted 10 lines
arch/x86/kvm/x86.c +7
-0+ l1_offset, +static_call (kvm_x86_get_l2_tsc_offset)(vcpu), + static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); + else + vcpu->arch.tsc_offset = l1_offset; + +static_call (kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); }static inline bool kvm_check_tsc_unstable (void )
Best regards,
Maxim Levitsky
Re: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Ilias StamatisQuoted 5 lines · Ilias Stamatis
The
vmx->current_tsc_ratiofield is redundant asvcpu->arch.tsc_scaling_ratioalready tracks the current TSC scaling ratio. Removing this field makesdecache_tsc_multiplier()an one-liner so remove that too and do avmcs_write64()directly in order to be more consistent with surrounding code.
Not to mention that 'decache_tsc_multiplier' isn't a good name IMHO for this....
Quoted 44 lines
Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/kvm/vmx/nested.c | 9 ++++----- arch/x86/kvm/vmx/vmx.c | 5 ++--- arch/x86/kvm/vmx/vmx.h | 8 -------- 3 files changed, 6 insertions(+), 16 deletions(-)arch/x86/kvm/vmx/nested.c +4
-5@@ -2533,9 +2533,8 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, }vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset); -if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);nested_vmx_transition_tlb_flush (vcpu, vmcs12, true);@@ -4501,12 +4500,12 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,vmcs_write32 (VM_EXIT_MSR_LOAD_COUNT , vmx->msr_autoload.host.nr);vmcs_write32 (VM_ENTRY_MSR_LOAD_COUNT , vmx->msr_autoload.guest.nr);vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio); +if (vmx->nested.l1_tpr_threshold !=-1 )vmcs_write32 (TPR_THRESHOLD , vmx->nested.l1_tpr_threshold); -if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); -if (vmx->nested.change_vmcs01_virtual_apic_mode) { vmx->nested.change_vmcs01_virtual_apic_mode = false;vmx_set_virtual_apic_mode (vcpu);arch/x86/kvm/vmx/vmx.c +0
-0
arch/x86/kvm/vmx/vmx.c void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, · line 1396
Quoted 8 lines
arch/x86/kvm/vmx/vmx.c +2
-3@@ -1392,9 +1392,8 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, }/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);
This might have an overhead of writing the TSC scaling ratio even if it is unchanged. I haven't measured how expensive vmread/vmwrites are but at least when nested, the vmreads/vmwrites can be very expensive (if they cause a vmexit).
This is why I think the 'vmx->current_tsc_ratio' exists - to have a cached value of TSC scale ratio to avoid either 'vmread'ing or 'vmwrite'ing it without a need.
Best regards,
Maxim Levitsky
Quoted 29 lines
arch/x86/kvm/vmx/vmx.c +0
-0}/* arch/x86/kvm/vmx/vmx.h +0
-8@@ -322,8 +322,6 @@ struct vcpu_vmx {/* apic deadline value in host tsc */ u64 hv_deadline_tsc; -u64 current_tsc_ratio; -unsigned long host_debugctlmsr;/* @@ -532,12 +530,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow) GFP_KERNEL_ACCOUNT); } -static inline void decache_tsc_multiplier (struct vcpu_vmx *vmx) -{ - vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio; -vmcs_write64 (TSC_MULTIPLIER , vmx->current_tsc_ratio); -} -static inline bool vmx_has_waitpkg (struct vcpu_vmx *vmx) { return vmx->secondary_exec_control &
Re: [PATCH v3 10/12] KVM: VMX: Set the TSC offset and multiplier on nested entry and exit
In reply to Ilias StamatisQuoted 15 lines · Ilias Stamatis
Calculate the nested TSC offset and multiplier on entering L2 using the corresponding functions. Restore the L1 values on L2's exit.
Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/kvm/vmx/nested.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)arch/x86/kvm/vmx/nested.c +0
-0@@ -2532,6 +2532,15 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,vmcs_write64 (GUEST_IA32_PAT , vmx->vcpu.arch.pat); }
arch/x86/kvm/vmx/nested.c static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, · line 2543
Quoted 10 lines
arch/x86/kvm/vmx/nested.c +9
-0+ vcpu->arch.tsc_offset =kvm_calc_nested_tsc_offset ( + vcpu->arch.l1_tsc_offset, +vmx_get_l2_tsc_offset (vcpu), +vmx_get_l2_tsc_multiplier (vcpu)); + + vcpu->arch.tsc_scaling_ratio =kvm_calc_nested_tsc_multiplier ( + vcpu->arch.l1_tsc_scaling_ratio, +vmx_get_l2_tsc_multiplier (vcpu)); +
This code can be in theory put to the common x86 code, since it uses the vendor callbacks anyway, but this is probably not worth it.
Quoted 13 lines
arch/x86/kvm/vmx/nested.c +0
-2vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset);if (kvm_has_tsc_control)vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);@@ -3353,8 +3362,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, }enter_guest_mode (vcpu); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset += vmcs12->tsc_offset;if (prepare_vmcs02 (vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic =EXIT_REASON_INVALID_STATE ;
arch/x86/kvm/vmx/nested.c void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, · line 4476
Quoted 10 lines
arch/x86/kvm/vmx/nested.c +5
-2@@ -4462,8 +4469,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,if (nested_cpu_has_preemption_timer (vmcs12))hrtimer_cancel (&to_vmx (vcpu)->nested.preemption_timer); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset -= vmcs12->tsc_offset; +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING )) { + vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; +if (nested_cpu_has2 (vmcs12,SECONDARY_EXEC_TSC_SCALING )) + vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; + }
Same here.
arch/x86/kvm/vmx/nested.c void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, · line 4479
Quoted 3 lines
arch/x86/kvm/vmx/nested.c +0
-0if (likely (!vmx->fail)) {sync_vmcs02_to_vmcs12 (vcpu, vmcs12);
Best regards,
Maxim Levitsky
Re: [PATCH v3 12/12] KVM: selftests: x86: Add vmx_nested_tsc_scaling_test
In reply to Ilias StamatisQuoted 178 lines · Ilias Stamatis
Test that nested TSC scaling works as expected with both L1 and L2 scaled.
Signed-off-by Ilias Stamatis <[email protected]>--- tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../kvm/x86_64/vmx_nested_tsc_scaling_test.c | 242 ++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.ctools/testing/selftests/kvm/.gitignore +1
-0@@ -29,6 +29,7 @@ /x86_64/vmx_preemption_timer_test /x86_64/vmx_set_nested_state_test /x86_64/vmx_tsc_adjust_test +/x86_64/vmx_nested_tsc_scaling_test /x86_64/xapic_ipi_test /x86_64/xen_shinfo_test /x86_64/xen_vmcall_testtools/testing/selftests/kvm/Makefile +1
-0@@ -60,6 +60,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test +TEST_GEN_PROGS_x86_64 += x86_64/vmx_nested_tsc_scaling_test TEST_GEN_PROGS_x86_64 += x86_64/xapic_ipi_test TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test TEST_GEN_PROGS_x86_64 += x86_64/debug_regstools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file +137
-0@@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* vmx_nested_tsc_scaling_test +* +* Copyright (C) 2021 Amazon.com, Inc. or its affiliates. +* +* This test case verifies that nested TSC scaling behaves as expected when +* both L1 and L2 are scaled using different ratios. For this test we scale +* L1 down and scale L2 up. +*/ + +#include <time.h> + +#include "kvm_util.h" +#include "vmx.h" +#include "kselftest.h" + + +#define VCPU_ID 0 + +/* L2 is scaled up (from L1's perspective) by this factor */ +#define L2_SCALE_FACTOR 4ULL + +#define TSC_OFFSET_L2 ((uint64_t) -33125236320908) +#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) + +#define L2_GUEST_STACK_SIZE 64 + +enum {USLEEP ,UCHECK_L1 ,UCHECK_L2 }; +#define GUEST_SLEEP (sec) ucall(UCALL_SYNC, 2, USLEEP, sec) +#define GUEST_CHECK (level, freq) ucall(UCALL_SYNC, 2, level, freq) + + +/* +* This function checks whether the "actual" TSC frequency of a guest matches +* its expected frequency. In order to account for delays in taking the TSC +* measurements, a difference of 1% between the actual and the expected value +* is tolerated. +*/ +static void compare_tsc_freq (uint64_t actual,uint64_t expected) +{ +uint64_t tolerance, thresh_low, thresh_high; + + tolerance = expected /100 ; + thresh_low = expected - tolerance; + thresh_high = expected + tolerance; + +TEST_ASSERT (thresh_low < actual, +"TSC freq is expected to be between %" PRIu64" and %" PRIu64 +" but it actually is %" PRIu64, + thresh_low, thresh_high, actual); +TEST_ASSERT (thresh_high > actual, +"TSC freq is expected to be between %" PRIu64" and %" PRIu64 +" but it actually is %" PRIu64, + thresh_low, thresh_high, actual); +} + +static void check_tsc_freq (int level) +{ +uint64_t tsc_start, tsc_end, tsc_freq; + +/* +* Reading the TSC twice with about a second's difference should give +* us an approximation of the TSC frequency from the guest's +* perspective. Now, this won't be completely accurate, but it should +* be good enough for the purposes of this test. +*/ + tsc_start =rdmsr (MSR_IA32_TSC ); +GUEST_SLEEP (1 ); + tsc_end =rdmsr (MSR_IA32_TSC ); + + tsc_freq = tsc_end - tsc_start; + +GUEST_CHECK (level, tsc_freq); +} + +static void l2_guest_code (void ) +{ +check_tsc_freq (UCHECK_L2 ); + +/* exit to L1 */ + __asm__ __volatile__("vmcall" ); +} + +static void l1_guest_code (struct vmx_pages *vmx_pages) +{ +unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE ]; +uint32_t control; + +/* check that L1's frequency looks alright before launching L2 */ +check_tsc_freq (UCHECK_L1 ); + +GUEST_ASSERT (prepare_for_vmx_operation (vmx_pages)); +GUEST_ASSERT (load_vmcs (vmx_pages)); + +/* prepare the VMCS for L2 execution */ +prepare_vmcs (vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE ]); + +/* enable TSC offsetting and TSC scaling for L2 */ + control =vmreadz (CPU_BASED_VM_EXEC_CONTROL ); + control |=CPU_BASED_USE_MSR_BITMAPS |CPU_BASED_USE_TSC_OFFSETTING ; +vmwrite (CPU_BASED_VM_EXEC_CONTROL , control); + + control =vmreadz (SECONDARY_VM_EXEC_CONTROL ); + control |=SECONDARY_EXEC_TSC_SCALING ; +vmwrite (SECONDARY_VM_EXEC_CONTROL , control); + +vmwrite (TSC_OFFSET ,TSC_OFFSET_L2 ); +vmwrite (TSC_MULTIPLIER ,TSC_MULTIPLIER_L2 ); +vmwrite (TSC_MULTIPLIER_HIGH ,TSC_MULTIPLIER_L2 >>32 ); + +/* launch L2 */ +GUEST_ASSERT (!vmlaunch ()); +GUEST_ASSERT (vmreadz (VM_EXIT_REASON ) ==EXIT_REASON_VMCALL ); + +/* check that L1's frequency still looks good */ +check_tsc_freq (UCHECK_L1 ); + +GUEST_DONE (); +} + +static void tsc_scaling_check_supported (void ) +{ +if (!kvm_check_cap (KVM_CAP_TSC_CONTROL )) { +print_skip ("TSC scaling not supported by the HW" ); +exit (KSFT_SKIP ); + } +} + +static void stable_tsc_check_supported (void ) +{ +FILE *fp; +char buf[4 ]; + + fp =fopen ("/sys/devices/system/clocksource/clocksource0/current_clocksource" ,"r" ); +if (fp ==NULL ) + goto skip_test;
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c line 147
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file +10
-0+ +if (fgets (buf,sizeof (buf), fp) ==NULL ) + goto skip_test; + +if (strncmp (buf,"tsc" ,sizeof (buf))) + goto skip_test; + +return ; +skip_test: +print_skip ("TSC is not stable" );
Tiny nitpick: I would print a message that expains the hack a bit better, something like
"Kernel doesn't use TSC clocksource - assuming that host TSC is not stable - skipping test"
Quoted 85 lines
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file +85
-0+exit (KSFT_SKIP ); +} + +int main (int argc,char *argv[]) +{ +struct kvm_vm *vm; +vm_vaddr_t vmx_pages_gva; + +uint64_t tsc_start, tsc_end; +uint64_t tsc_khz; +uint64_t l1_scale_factor; +uint64_t l0_tsc_freq =0 ; +uint64_t l1_tsc_freq =0 ; +uint64_t l2_tsc_freq =0 ; + +nested_vmx_check_supported (); +tsc_scaling_check_supported (); +stable_tsc_check_supported (); + +/* +* We set L1's scale factor to be a random number from 2 to 10. +* Ideally we would do the same for L2's factor but that one is +* referenced by both main() and l1_guest_code() and using a global +* variable does not work. +*/ +srand (time (NULL )); + l1_scale_factor = (rand () %9 ) +2 ; +printf ("L1's scale down factor is: %" PRIu64"\n" , l1_scale_factor); +printf ("L2's scale up factor is: %llu\n" ,L2_SCALE_FACTOR ); + + tsc_start =rdtsc (); +sleep (1 ); + tsc_end =rdtsc (); + + l0_tsc_freq = tsc_end - tsc_start; +printf ("real TSC frequency is around: %" PRIu64"\n" , l0_tsc_freq); + + vm =vm_create_default (VCPU_ID ,0 , (void *) l1_guest_code); +vcpu_alloc_vmx (vm, &vmx_pages_gva); +vcpu_args_set (vm,VCPU_ID ,1 , vmx_pages_gva); + + tsc_khz =_vcpu_ioctl (vm,VCPU_ID ,KVM_GET_TSC_KHZ ,NULL ); +TEST_ASSERT (tsc_khz !=-1 ,"vcpu ioctl KVM_GET_TSC_KHZ failed" ); + +/* scale down L1's TSC frequency */ +vcpu_ioctl (vm,VCPU_ID ,KVM_SET_TSC_KHZ , + (void *) (tsc_khz / l1_scale_factor)); + +for (;;) { +volatile struct kvm_run *run =vcpu_state (vm,VCPU_ID ); +struct ucall uc; + +vcpu_run (vm,VCPU_ID ); +TEST_ASSERT (run->exit_reason ==KVM_EXIT_IO , +"Got exit_reason other than KVM_EXIT_IO: %u (%s)\n" , + run->exit_reason, +exit_reason_str (run->exit_reason)); + +switch (get_ucall (vm,VCPU_ID , &uc)) { +case UCALL_ABORT : +TEST_FAIL ("%s" , (const char *) uc.args[0 ]); +case UCALL_SYNC : +switch (uc.args[0 ]) { +case USLEEP : +sleep (uc.args[1 ]); +break ; +case UCHECK_L1 : + l1_tsc_freq = uc.args[1 ]; +printf ("L1's TSC frequency is around: %" PRIu64 +"\n" , l1_tsc_freq); + +compare_tsc_freq (l1_tsc_freq, + l0_tsc_freq / l1_scale_factor); +break ; +case UCHECK_L2 : + l2_tsc_freq = uc.args[1 ]; +printf ("L2's TSC frequency is around: %" PRIu64 +"\n" , l2_tsc_freq); + +compare_tsc_freq (l2_tsc_freq, + l1_tsc_freq *L2_SCALE_FACTOR ); +break ; + } +break ; +case UCALL_DONE :
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c line 242
Quoted 10 lines
tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file +10
-0+ goto done; +default : +TEST_FAIL ("Unknown ucall %lu" , uc.cmd); + } + } + +done: +kvm_vm_free (vm); +return 0 ; +}
Overall looks good to me. Thanks!
Best regards,
Maxim Levitsky
Sean Christopherson
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Maxim LevitskyQuoted 20 lines · Maxim Levitsky
arch/x86/kvm/vmx/vmx.c void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, · line 1396
Quoted 13 lines · Ilias Stamatis
arch/x86/kvm/vmx/vmx.c +2
-3@@ -1392,9 +1392,8 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, }/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);This might have an overhead of writing the TSC scaling ratio even if it is unchanged. I haven't measured how expensive vmread/vmwrites are but at least when nested, the vmreads/vmwrites can be very expensive (if they cause a vmexit).
Quoted 3 lines
This is why I think the '
vmx->current_tsc_ratio' exists - to have a cached value of TSC scale ratio to avoid either 'vmread'ing or 'vmwrite'ing it without a need.
Yes, but its existence is a complete hack. vmx->current_tsc_ratio has the same scope as vcpu->arch.tsc_scaling_ratio, i.e. vmx == vcpu == vcpu->arch. Unlike per-VMCS tracking, it should not be useful, keyword "should".
What I meant by my earlier comment:
Its use in vmx_vcpu_load_vmcs() is basically "write the VMCS if we forgot to earlier", which is all kinds of wrong.
is that vmx_vcpu_load_vmcs() should never write vmcs.TSC_MULTIPLIER. The correct behavior is to set the field at VMCS initialization, and then immediately set it whenever the ratio is changed, e.g. on nested transition, from userspace, etc... In other words, my unclear feedback was to make it obsolete (and drop it) by fixing the underlying mess, not to just drop the optimization hack.
Re: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 21 lines · Sean Christopherson
Quoted 20 lines · Maxim Levitsky
arch/x86/kvm/vmx/vmx.c void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, · line 1396
Quoted 13 lines · Ilias Stamatis
arch/x86/kvm/vmx/vmx.c +2
-3@@ -1392,9 +1392,8 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, }/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); +if (kvm_has_tsc_control) +vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);This might have an overhead of writing the TSC scaling ratio even if it is unchanged. I haven't measured how expensive vmread/vmwrites are but at least when nested, the vmreads/vmwrites can be very expensive (if they cause a vmexit).
Quoted 3 lines
Quoted 3 lines
This is why I think the '
vmx->current_tsc_ratio' exists - to have a cached value of TSC scale ratio to avoid either 'vmread'ing or 'vmwrite'ing it without a need.
Right. I thought the overhead might not be that significant since we're doing lots of vmwrites on vmentry/vmexit anyway, but yeah, why introduce any kind of extra overhead anyway.
I'm fine with this particular patch getting dropped. It's not directly related to the series anyway.
Quoted 7 lines
Yes, but its existence is a complete hack.
vmx->current_tsc_ratiohas the same scope asvcpu->arch.tsc_scaling_ratio, i.e. vmx == vcpu ==vcpu->arch. Unlike per-VMCS tracking, it should not be useful, keyword "should".What I meant by my earlier comment:
Quoted 8 lines
Its use in
vmx_vcpu_load_vmcs()is basically "write the VMCS if we forgot to earlier", which is all kinds of wrong.is that
vmx_vcpu_load_vmcs()should never write vmcs.TSC_MULTIPLIER. The correct behavior is to set the field at VMCS initialization, and then immediately set it whenever the ratio is changed, e.g. on nested transition, from userspace, etc... In other words, my unclear feedback was to make it obsolete (and drop it) by fixing the underlying mess, not to just drop the optimization hack.
I understood this and replied earlier. The right place for the hw multiplier field to be updated is inside set_tsc_khz() in common code when the ratio changes. However, this requires adding another vendor callback etc. As all this is further refactoring I believe it's better to leave this series as is - ie only touching code that is directly related to nested TSC scaling and not try to do everything as part of the same series. This makes testing easier too. We can still implement these changes later.
Thanks,
Ilias
Sean Christopherson
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 16 lines · Ilias
Quoted 6 lines · Sean Christopherson
Yes, but its existence is a complete hack.
vmx->current_tsc_ratiohas the same scope asvcpu->arch.tsc_scaling_ratio, i.e. vmx == vcpu ==vcpu->arch. Unlike per-VMCS tracking, it should not be useful, keyword "should".What I meant by my earlier comment:
Quoted 8 lines
Its use in
vmx_vcpu_load_vmcs()is basically "write the VMCS if we forgot to earlier", which is all kinds of wrong.is that
vmx_vcpu_load_vmcs()should never write vmcs.TSC_MULTIPLIER. The correct behavior is to set the field at VMCS initialization, and then immediately set it whenever the ratio is changed, e.g. on nested transition, from userspace, etc... In other words, my unclear feedback was to make it obsolete (and drop it) by fixing the underlying mess, not to just drop the optimization hack.
Quoted 6 lines
I understood this and replied earlier. The right place for the hw multiplier field to be updated is inside
set_tsc_khz()in common code when the ratio changes. However, this requires adding another vendor callback etc. As all this is further refactoring I believe it's better to leave this series as is - ie only touching code that is directly related to nested TSC scaling and not try to do everything as part of the same series.
But it directly impacts your code, e.g. the nested enter/exit flows would need to dance around the decache silliness. And I believe it even more directly impacts this series: kvm_set_tsc_khz() fails to handle the case where userspace invokes KVM_SET_TSC_KHZ while L2 is active.
Quoted 1 line
This makes testing easier too.
Hmm, sort of. Yes, the fewer patches/modifications in a series definitely makes the series itself easier to test. But stepping back and looking at the total cost of testing, I would argue that punting related changes to a later time increases the overall cost. E.g. if someone else picks up the clean up work, then they have to redo most, if not all, of the testing that you are already doing, including getting access to the proper hardware, understanding what tests to prioritize, etc... Whereas adding one more patch to your series is an incremental cost since you already have the hardware setup, know which tests to run, etc...
Quoted 1 line
We can still implement these changes later.
We can, but we shouldn't. Simply dropping vmx->current_tsc_ratio is not an option; it knowingly introduces a (minor) performance regression, for no reason other than wanting to avoid code churn. Piling more stuff on top of the flawed decache logic is impolite, as it adds more work for the person that ends up doing the cleanup. I would 100% agree if this were a significant cleanup and/or completely unrelated, but IMO that's not the case.
Compile tested only...
arch/x86/include/asm/kvm-x86-ops.h +1 -0
@@ -90,6 +90,7 @@ KVM_X86_OP_NULL(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) KVM_X86_OP(write_tsc_offset) +KVM_X86_OP(write_tsc_multiplier) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff)
arch/x86/include/asm/kvm_host.h +1 -0
@@ -1308,6 +1308,7 @@ struct kvm_x86_ops {u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);void (*write_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset); +void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu,u64 multiplier);/* * Retrieve somewhat arbitrary exit information. Intended to be used
arch/x86/kvm/svm/svm.c +9 -0
@@ -1103,6 +1103,14 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)vmcb_mark_dirty (svm->vmcb,VMCB_INTERCEPTS ); } +static void svm_write_tsc_multiplier (struct kvm_vcpu *vcpu,u64 l1_multiplier) +{ +/* +* Handled when loading guest state since the ratio is programmed via +* MSR_AMD64_TSC_RATIO, not a field in the VMCB. +*/ +} +/* Evaluate instruction intercepts that depend on guest CPUID features. */ static void svm_recalc_instruction_intercepts (struct kvm_vcpu *vcpu,struct vcpu_svm *svm)
@@ -4528,6 +4536,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_l2_tsc_offset = svm_get_l2_tsc_offset, .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, .write_tsc_offset = svm_write_tsc_offset, + .write_tsc_multiplier = svm_write_tsc_multiplier, .load_mmu_pgd = svm_load_mmu_pgd,
arch/x86/kvm/vmx/nested.c +2 -2
@@ -2535,7 +2535,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset);if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_scaling_ratio);nested_vmx_transition_tlb_flush (vcpu, vmcs12, true);
@@ -4505,7 +4505,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,vmcs_write32 (TPR_THRESHOLD , vmx->nested.l1_tpr_threshold);if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_scaling_ratio);if (vmx->nested.change_vmcs01_virtual_apic_mode) { vmx->nested.change_vmcs01_virtual_apic_mode = false;
arch/x86/kvm/vmx/vmx.c +0 -5
@@ -1390,11 +1390,6 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, vmx->loaded_vmcs->cpu = cpu; } - -/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); }/*
@@ -1813,6 +1808,11 @@ static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)vmcs_write64 (TSC_OFFSET , offset);
...skipping...
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -322,8 +322,6 @@ struct vcpu_vmx {
/* apic deadline value in host tsc */
u64 hv_deadline_tsc;- u64 current_tsc_ratio;
-
unsigned long host_debugctlmsr;
/*
@@ -532,12 +530,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow)
GFP_KERNEL_ACCOUNT);}
-static inline void decache_tsc_multiplier(struct vcpu_vmx *vmx) -{
- vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
- vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
-}
-
static inline bool vmx_has_waitpkg(struct vcpu_vmx *vmx)
{
return vmx->secondary_exec_control &arch/x86/kvm/x86.c +25 -5
@@ -2179,14 +2179,16 @@ static u32 adjust_tsc_khz(u32 khz, s32 ppm)return v; } +static void kvm_vcpu_write_tsc_multiplier (struct kvm_vcpu *vcpu, +u64 l1_multiplier); +static int set_tsc_khz (struct kvm_vcpu *vcpu,u32 user_tsc_khz,bool scale) {u64 ratio;/* Guest TSC same frequency as host TSC? */ if (!scale) { - vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, kvm_default_tsc_scaling_ratio);return 0 ; }
@@ -2212,7 +2214,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)return -1 ; } - vcpu->arch.l1_tsc_scaling_ratio = vcpu->arch.tsc_scaling_ratio = ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, ratio);return 0 ; }
@@ -2224,8 +2226,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)/* tsc_khz can be zero if TSC calibration fails */ if (user_tsc_khz ==0 ) {/* set tsc_scaling_ratio to a safe value */ - vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, kvm_default_tsc_scaling_ratio);return -1 ; }
@@ -2383,6 +2384,25 @@ static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)static_call (kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); } +static void kvm_vcpu_write_tsc_multiplier (struct kvm_vcpu *vcpu, +u64 l1_multiplier) +{ +if (!kvm_has_tsc_control) +return ; + + vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; + +/* Userspace is changing the multiplier while L2 is active... */ +if (is_guest_mode (vcpu)) + vcpu->arch.tsc_scaling_ratio =kvm_calc_nested_tsc_multiplier ( + l1_multiplier, +static_call (kvm_x86_get_l2_tsc_multiplier)(vcpu)); +else + vcpu->arch.tsc_scaling_ratio = l1_multiplier; + +static_call (kvm_x86_write_tsc_multiplier)(vcpu, vcpu->arch.tsc_scaling_ratio); +} +static inline bool kvm_check_tsc_unstable (void ) {#ifdef CONFIG_X86_64
Sean Christopherson
maintainerRe: [PATCH v3 10/12] KVM: VMX: Set the TSC offset and multiplier on nested entry and exit
In reply to Ilias Stamatis"KVM: nVMX:" for the scope.
The shortlog is also a bit confusing. I usually think of "set == write", i.e. I expected VMWRITEs in the diff. The nested_vmx_vmexit() case in particular is gnarly because the actual VMWRITEs aren't captured in the diff's context.
What about combining this with the next patch that exposes the feature to L1? E.g. "KVM: nVMX: Enable nested TSC scaling" or so.
That would avoid bikeshedding the meaning of "set", fix the goof in the next patch's shortlog (KVM exposes the feature to L1, not L2), and eliminate an unnecessary patch for bisection purposes. Bisecting to a patch that exposes the feature but doesn't introduce any actual functionality isn't all that helpful, e.g. if there is a bug in _this_ patch then bisection will arguably point at the wrong patch.
Quoted 54 lines · Ilias Stamatis
Calculate the nested TSC offset and multiplier on entering L2 using the corresponding functions. Restore the L1 values on L2's exit.
Signed-off-by Ilias Stamatis <[email protected]>--- arch/x86/kvm/vmx/nested.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)arch/x86/kvm/vmx/nested.c +14
-4@@ -2532,6 +2532,15 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,vmcs_write64 (GUEST_IA32_PAT , vmx->vcpu.arch.pat); } + vcpu->arch.tsc_offset =kvm_calc_nested_tsc_offset ( + vcpu->arch.l1_tsc_offset, +vmx_get_l2_tsc_offset (vcpu), +vmx_get_l2_tsc_multiplier (vcpu)); + + vcpu->arch.tsc_scaling_ratio =kvm_calc_nested_tsc_multiplier ( + vcpu->arch.l1_tsc_scaling_ratio, +vmx_get_l2_tsc_multiplier (vcpu)); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset);if (kvm_has_tsc_control)vmcs_write64 (TSC_MULTIPLIER , vcpu->arch.tsc_scaling_ratio);@@ -3353,8 +3362,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, }enter_guest_mode (vcpu); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset += vmcs12->tsc_offset;if (prepare_vmcs02 (vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic =EXIT_REASON_INVALID_STATE ;@@ -4462,8 +4469,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,if (nested_cpu_has_preemption_timer (vmcs12))hrtimer_cancel (&to_vmx (vcpu)->nested.preemption_timer); -if (vmcs12->cpu_based_vm_exec_control &CPU_BASED_USE_TSC_OFFSETTING ) - vcpu->arch.tsc_offset -= vmcs12->tsc_offset; +if (nested_cpu_has (vmcs12,CPU_BASED_USE_TSC_OFFSETTING )) { + vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; +if (nested_cpu_has2 (vmcs12,SECONDARY_EXEC_TSC_SCALING )) + vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; + }if (likely (!vmx->fail)) {sync_vmcs02_to_vmcs12 (vcpu, vmcs12);Signature · 1 line
2.17.1
Paolo Bonzini
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Sean ChristophersonQuoted 3 lines · Sean Christopherson
Quoted 3 lines
The right place for the hw multiplier
field to be updated is insideset_tsc_khz()in common code when the ratio changes.
Sort of, the problem is that you have two VMCS's to update. If properly fixed, the cache is useful to fix the issue with KVM_SET_TSC_KHZ needing to update both of them. For that to work, you'd have to move the cache to struct loaded_vmcs.
So you can:
1) move the cached tsc_ratio to struct loaded_vmcs
2) add a function in common code (update_tsc_parameters or something like that) to update both the offset and the ratio depending on is_guest_mode()
3) call that function from nested vmentry/vmexit
And at that point the cache will do its job and figure out whether a vmwrite is needed, on both vmentry and vmexit.
I actually like the idea of storing the expected value in kvm_vcpu and the current value in loaded_vmcs. We might use it for other things such as reload_vmcs01_apic_access_page perhaps.
Paolo
Quoted 9 lines
Quoted 4 lines
However, this requires adding another vendor callback etc. As all this is further refactoring I believe it's better to leave this series as is - ie only touching code that is directly related to nested TSC scaling and not try to do everything as part of the same series.
But it directly impacts your code, e.g. the nested enter/exit flows would need to dance around the decache silliness. And I believe it even more directly impacts this series:
kvm_set_tsc_khz()fails to handle the case where userspace invokesKVM_SET_TSC_KHZwhile L2 is active.
Sean Christopherson
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 9 lines · Paolo Bonzini
Quoted 3 lines · Sean Christopherson
Quoted 3 lines
The right place for the hw multiplier
field to be updated is insideset_tsc_khz()in common code when the ratio changes.Sort of, the problem is that you have two VMCS's to update. If properly fixed, the cache is useful to fix the issue with
KVM_SET_TSC_KHZneeding to update both of them. For that to work, you'd have to move the cache to structloaded_vmcs.
vmcs01 and vmcs02 will get updated at enter/exit, if there's no caching then it all Just Works.
Quoted 7 lines
So you can:
1) move the cached
tsc_ratioto structloaded_vmcs2) add a function in common code (
update_tsc_parametersor something like that) to update both the offset and the ratio depending onis_guest_mode()
Quoted 8 lines
3) call that function from nested vmentry/vmexit
And at that point the cache will do its job and figure out whether a vmwrite is needed, on both vmentry and vmexit.
I actually like the idea of storing the expected value in
kvm_vcpuand the current value inloaded_vmcs. We might use it for other things such asreload_vmcs01_apic_access_pageperhaps.
I'm not necessarily opposed to aggressively shadowing the VMCS, but if we go that route then it should be a standalone series that implements a framework that can be easily extended to arbitrary fields. Adding fields to loaded_vmcs one at a time will be tedious and error prone. E.g. what makes TSC_MULTIPLIER more special than TSC_OFFSET, GUEST_IA32_PAT, GUEST_IA32_DEBUGCTL, GUEST_BNDCFGS, and other number of fields that are likely to persist for a given vmcs02?
The current caching logic is just plain ugly and should not exist.
Paolo Bonzini
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 9 lines · Sean Christopherson
Quoted 3 lines
I actually like the idea of storing the expected value in
kvm_vcpuand the current value inloaded_vmcs. We might use it for other things such asreload_vmcs01_apic_access_pageperhaps.I'm not necessarily opposed to aggressively shadowing the VMCS, but if we go that route then it should be a standalone series that implements a framework that can be easily extended to arbitrary fields. Adding fields to
loaded_vmcsone at a time will be tedious and error prone. E.g. what makesTSC_MULTIPLIERmore special thanTSC_OFFSET,GUEST_IA32_PAT,GUEST_IA32_DEBUGCTL,GUEST_BNDCFGS, and other number of fields that are likely to persist for a given vmcs02?
That it can be changed via ioctls in a way that affects both vmcs01 and vmcs02. So TSC_MULTIPLIER is in the same boat as TSC_OFFSET, which I agree we should shadow more aggressively, but the others are different.
Paolo
Sean Christopherson
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 4 lines · Paolo Bonzini
Quoted 3 lines · Sean Christopherson
Quoted 3 lines
I actually like the idea of storing the expected value in
kvm_vcpuand the current value inloaded_vmcs. We might use it for other things such asreload_vmcs01_apic_access_pageperhaps.
Quoted 8 lines
Quoted 6 lines
I'm not necessarily opposed to aggressively shadowing the VMCS, but if we go that route then it should be a standalone series that implements a framework that can be easily extended to arbitrary fields. Adding fields to
loaded_vmcsone at a time will be tedious and error prone. E.g. what makesTSC_MULTIPLIERmore special thanTSC_OFFSET,GUEST_IA32_PAT,GUEST_IA32_DEBUGCTL,GUEST_BNDCFGS, and other number of fields that are likely to persist for a given vmcs02?That it can be changed via ioctls in a way that affects both vmcs01 and vmcs02.
That holds true for any MSR that is conditionally loaded/cleared on enter/exit, e.g. userspace can stuff MSR_IA32_CR_PAT while L2 is active, and that can affect L1 if L1 is running without VM_EXIT_LOAD_IA32_PAT.
I'm not saying that the above is likely, but neither is changing the TSC scaling ratio while L2 is active (I assume it occurs on migration, but in the grand scheme that's not a common operation).
Re: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Sean ChristophersonQuoted 24 lines · Sean Christopherson
Quoted 16 lines · Ilias
Quoted 6 lines · Sean Christopherson
Yes, but its existence is a complete hack.
vmx->current_tsc_ratiohas the same scope asvcpu->arch.tsc_scaling_ratio, i.e. vmx == vcpu ==vcpu->arch. Unlike per-VMCS tracking, it should not be useful, keyword "should".What I meant by my earlier comment:
Quoted 8 lines
Its use in
vmx_vcpu_load_vmcs()is basically "write the VMCS if we forgot to earlier", which is all kinds of wrong.is that
vmx_vcpu_load_vmcs()should never write vmcs.TSC_MULTIPLIER. The correct behavior is to set the field at VMCS initialization, and then immediately set it whenever the ratio is changed, e.g. on nested transition, from userspace, etc... In other words, my unclear feedback was to make it obsolete (and drop it) by fixing the underlying mess, not to just drop the optimization hack.Quoted 6 lines
I understood this and replied earlier. The right place for the hw multiplier field to be updated is inside
set_tsc_khz()in common code when the ratio changes. However, this requires adding another vendor callback etc. As all this is further refactoring I believe it's better to leave this series as is - ie only touching code that is directly related to nested TSC scaling and not try to do everything as part of the same series.
Quoted 4 lines
But it directly impacts your code, e.g. the nested enter/exit flows would need to dance around the decache silliness. And I believe it even more directly impacts this series:
kvm_set_tsc_khz()fails to handle the case where userspace invokesKVM_SET_TSC_KHZwhile L2 is active.
Good catch!
Quoted 15 lines
Quoted 2 lines
This makes testing easier too.
Hmm, sort of. Yes, the fewer patches/modifications in a series definitely makes the series itself easier to test. But stepping back and looking at the total cost of testing, I would argue that punting related changes to a later time increases the overall cost. E.g. if someone else picks up the clean up work, then they have to redo most, if not all, of the testing that you are already doing, including getting access to the proper hardware, understanding what tests to prioritize, etc... Whereas adding one more patch to your series is an incremental cost since you already have the hardware setup, know which tests to run, etc...
Quoted 1 line
We can still implement these changes later.
Quoted 8 lines
We can, but we shouldn't. Simply dropping
vmx->current_tsc_ratiois not an option; it knowingly introduces a (minor) performance regression, for no reason other than wanting to avoid code churn. Piling more stuff on top of the flawed decache logic is impolite, as it adds more work for the person that ends up doing the cleanup. I would 100% agree if this were a significant cleanup and/or completely unrelated, but IMO that's not the case.Compile tested only...
Thank you.
Quoted 32 lines
arch/x86/include/asm/kvm-x86-ops.h +1
-0@@ -90,6 +90,7 @@ KVM_X86_OP_NULL(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) KVM_X86_OP(write_tsc_offset) +KVM_X86_OP(write_tsc_multiplier) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff)arch/x86/include/asm/kvm_host.h +1
-0@@ -1308,6 +1308,7 @@ struct kvm_x86_ops {u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);void (*write_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset); +void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu,u64 multiplier);/* * Retrieve somewhat arbitrary exit information. Intended to be used arch/x86/kvm/svm/svm.c +0
-0@@ -1103,6 +1103,14 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)vmcb_mark_dirty (svm->vmcb,VMCB_INTERCEPTS );
arch/x86/kvm/svm/svm.c static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) · line 1113
Quoted 10 lines
arch/x86/kvm/svm/svm.c +8
-0} +static void svm_write_tsc_multiplier (struct kvm_vcpu *vcpu,u64 l1_multiplier) +{ +/* +* Handled when loading guest state since the ratio is programmed via +* MSR_AMD64_TSC_RATIO, not a field in the VMCB. +*/ +} +
Ok, what I wanted to avoid really is having to dig into SVM code and see where exactly it sets the TSC multiplier or having to implement svm_write_tsc_multiplier as I knew AMD uses an MSR instead of a VMCB field.
But if we are fine with introducing this as is above (for now) I will include this in the series, apply the other small changes suggested and re-post the patches.
Re: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
In reply to Sean ChristophersonQuoted 226 lines · Sean Christopherson
Quoted 16 lines · Ilias
Quoted 6 lines · Sean Christopherson
Yes, but its existence is a complete hack.
vmx->current_tsc_ratiohas the same scope asvcpu->arch.tsc_scaling_ratio, i.e. vmx == vcpu ==vcpu->arch. Unlike per-VMCS tracking, it should not be useful, keyword "should".What I meant by my earlier comment:
Quoted 8 lines
Its use in
vmx_vcpu_load_vmcs()is basically "write the VMCS if we forgot to earlier", which is all kinds of wrong.is that
vmx_vcpu_load_vmcs()should never write vmcs.TSC_MULTIPLIER. The correct behavior is to set the field at VMCS initialization, and then immediately set it whenever the ratio is changed, e.g. on nested transition, from userspace, etc... In other words, my unclear feedback was to make it obsolete (and drop it) by fixing the underlying mess, not to just drop the optimization hack.Quoted 6 lines
I understood this and replied earlier. The right place for the hw multiplier field to be updated is inside
set_tsc_khz()in common code when the ratio changes. However, this requires adding another vendor callback etc. As all this is further refactoring I believe it's better to leave this series as is - ie only touching code that is directly related to nested TSC scaling and not try to do everything as part of the same series.But it directly impacts your code, e.g. the nested enter/exit flows would need to dance around the decache silliness. And I believe it even more directly impacts this series:
kvm_set_tsc_khz()fails to handle the case where userspace invokesKVM_SET_TSC_KHZwhile L2 is active.Quoted 1 line
This makes testing easier too.
Hmm, sort of. Yes, the fewer patches/modifications in a series definitely makes the series itself easier to test. But stepping back and looking at the total cost of testing, I would argue that punting related changes to a later time increases the overall cost. E.g. if someone else picks up the clean up work, then they have to redo most, if not all, of the testing that you are already doing, including getting access to the proper hardware, understanding what tests to prioritize, etc... Whereas adding one more patch to your series is an incremental cost since you already have the hardware setup, know which tests to run, etc...
Quoted 1 line
We can still implement these changes later.
We can, but we shouldn't. Simply dropping
vmx->current_tsc_ratiois not an option; it knowingly introduces a (minor) performance regression, for no reason other than wanting to avoid code churn. Piling more stuff on top of the flawed decache logic is impolite, as it adds more work for the person that ends up doing the cleanup. I would 100% agree if this were a significant cleanup and/or completely unrelated, but IMO that's not the case.Compile tested only...
arch/x86/include/asm/kvm-x86-ops.h +1
-0@@ -90,6 +90,7 @@ KVM_X86_OP_NULL(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) KVM_X86_OP(write_tsc_offset) +KVM_X86_OP(write_tsc_multiplier) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff)arch/x86/include/asm/kvm_host.h +1
-0@@ -1308,6 +1308,7 @@ struct kvm_x86_ops {u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);void (*write_tsc_offset)(struct kvm_vcpu *vcpu,u64 offset); +void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu,u64 multiplier);/* * Retrieve somewhat arbitrary exit information. Intended to be used arch/x86/kvm/svm/svm.c +9
-0@@ -1103,6 +1103,14 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)vmcb_mark_dirty (svm->vmcb,VMCB_INTERCEPTS ); } +static void svm_write_tsc_multiplier (struct kvm_vcpu *vcpu,u64 l1_multiplier) +{ +/* +* Handled when loading guest state since the ratio is programmed via +* MSR_AMD64_TSC_RATIO, not a field in the VMCB. +*/ +} +/* Evaluate instruction intercepts that depend on guest CPUID features. */ static void svm_recalc_instruction_intercepts (struct kvm_vcpu *vcpu,struct vcpu_svm *svm)@@ -4528,6 +4536,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_l2_tsc_offset = svm_get_l2_tsc_offset, .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, .write_tsc_offset = svm_write_tsc_offset, + .write_tsc_multiplier = svm_write_tsc_multiplier, .load_mmu_pgd = svm_load_mmu_pgd,arch/x86/kvm/vmx/nested.c +2
-2@@ -2535,7 +2535,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_offset);if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_scaling_ratio);nested_vmx_transition_tlb_flush (vcpu, vmcs12, true);@@ -4505,7 +4505,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,vmcs_write32 (TPR_THRESHOLD , vmx->nested.l1_tpr_threshold);if (kvm_has_tsc_control) -decache_tsc_multiplier (vmx); +vmcs_write64 (TSC_OFFSET , vcpu->arch.tsc_scaling_ratio);if (vmx->nested.change_vmcs01_virtual_apic_mode) { vmx->nested.change_vmcs01_virtual_apic_mode = false;arch/x86/kvm/vmx/vmx.c +0
-5@@ -1390,11 +1390,6 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, vmx->loaded_vmcs->cpu = cpu; } - -/* Setup TSC multiplier */ -if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) -decache_tsc_multiplier (vmx); }/* @@ -1813,6 +1808,11 @@ static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)vmcs_write64 (TSC_OFFSET , offset);...skipping...
+++b/arch/x86/kvm/vmx/vmx.harch/x86/kvm/vmx/vmx.c +0
-8@@ -322,8 +322,6 @@ struct vcpu_vmx {/* apic deadline value in host tsc */ u64 hv_deadline_tsc; -u64 current_tsc_ratio; -unsigned long host_debugctlmsr;/* @@ -532,12 +530,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow) GFP_KERNEL_ACCOUNT); } -static inline void decache_tsc_multiplier (struct vcpu_vmx *vmx) -{ - vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio; -vmcs_write64 (TSC_MULTIPLIER , vmx->current_tsc_ratio); -} -static inline bool vmx_has_waitpkg (struct vcpu_vmx *vmx) { return vmx->secondary_exec_control &arch/x86/kvm/x86.c +18
-5@@ -2179,14 +2179,16 @@ static u32 adjust_tsc_khz(u32 khz, s32 ppm)return v; } +static void kvm_vcpu_write_tsc_multiplier (struct kvm_vcpu *vcpu, +u64 l1_multiplier); +static int set_tsc_khz (struct kvm_vcpu *vcpu,u32 user_tsc_khz,bool scale) {u64 ratio;/* Guest TSC same frequency as host TSC? */ if (!scale) { - vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, kvm_default_tsc_scaling_ratio);return 0 ; }@@ -2212,7 +2214,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)return -1 ; } - vcpu->arch.l1_tsc_scaling_ratio = vcpu->arch.tsc_scaling_ratio = ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, ratio);return 0 ; }@@ -2224,8 +2226,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)/* tsc_khz can be zero if TSC calibration fails */ if (user_tsc_khz ==0 ) {/* set tsc_scaling_ratio to a safe value */ - vcpu->arch.l1_tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; +kvm_vcpu_write_tsc_multiplier (vcpu, kvm_default_tsc_scaling_ratio);return -1 ; }@@ -2383,6 +2384,25 @@ static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)static_call (kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); } +static void kvm_vcpu_write_tsc_multiplier (struct kvm_vcpu *vcpu, +u64 l1_multiplier) +{ +if (!kvm_has_tsc_control) +return ; + + vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; + +/* Userspace is changing the multiplier while L2 is active... */ +if (is_guest_mode (vcpu)) + vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( + l1_multiplier,
arch/x86/kvm/x86.c static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) · line 2408
Quoted 10 lines
arch/x86/kvm/x86.c +7
-0+ static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); + else + vcpu->arch.tsc_scaling_ratio = l1_multiplier; + +static_call (kvm_x86_write_tsc_multiplier)(vcpu, vcpu->arch.tsc_scaling_ratio); +} +static inline bool kvm_check_tsc_unstable (void ) {#ifdef CONFIG_X86_64
Hmm, this patch actually still removes the caching and introduces a small performance overhead. For example if neither L1 nor L2 are scaled it will still do a vmwrite for every L2 entry/write.
So do we want to get rid of decache_tsc_multiplier() but keep vmx->current_tsc_ratio and do the check inside write_tsc_multiplier()? Or alternatively delete vmx->current_tsc_ratio too and have write_tsc_multiplier() receive 2 parameters, one of the old multiplier and one of the new?
Sean Christopherson
maintainerRe: [PATCH v3 09/12] KVM: VMX: Remove vmx->current_tsc_ratio and decache_tsc_multiplier()
Quoted 3 lines · Ilias
Hmm, this patch actually still removes the caching and introduces a small performance overhead. For example if neither L1 nor L2 are scaled it will still do a vmwrite for every L2 entry/write.
True, but there is an ocean of difference between the relative performance of vmx_vcpu_load_vmcs() and a nested transition. vmx_vcpu_load_vmcs() is also called much more frequently.
Quoted 5 lines
So do we want to get rid of
decache_tsc_multiplier()but keepvmx->current_tsc_ratioand do the check insidewrite_tsc_multiplier()? Or alternatively deletevmx->current_tsc_ratiotoo and havewrite_tsc_multiplier()receive 2 parameters, one of the old multiplier and one of the new?
My vote is to kill it, eat the barely-noticeable perf hit on nVMX, and tackle the aggressive VMCS shadowing in a separate series.