fix VMX TSC synchronicity

The boot TSC sync check is failing on recent Linux SMP guests on TSC stable hosts.

Following patch attempts to address the problems, which are:

1) APIC_DM_STARTUP, which is only called for vcpu's other than vcpu0, will trigger ->vcpu_reset(), setting the TSC to 0. Fix that by moving the guest_write_tsc(0) to vcpu_load.

2) vcpu's are initialized at different times by QEMU (vcpu0 init happens way earlier than the rest). Fix that by initializing the TSC of vcpu's > 0 with reference to vcpu0 init tsc value. This way TSC synchronization is kept (apparently Xen does something similar).

3) The code which adjusts the TSC of a VCPU on physical CPU switch is meant to guarantee that the guest sees a monotonically increasing value. However there is a large gap, in terms of clocks, between the time which the source CPU TSC is read and the time the destination CPU TSC is read. So move those two reads to happen at vcpu_clear.

I believe that 3) is the reason for the -no-kvm-irqchip problems reported by Avi on RHEL5, with kernels < 2.6.21 (where vcpu->cpu pinning would fix the problem). Unfortunately I could reproduce that problem.

4-way guest with constant tick at 250Hz now reliably sees the TSC's synchronized, and idle guest CPU consumption is reduced by 50% (3-4% instead of 8%, the latter with acpi_pm_good boot parameter).

arch/x86/kvm/vmx.c +6 -1

index 4741806..e1c8cf4 100644
@@ -47,6 +47,8 @@ struct vcpu_vmx {
 	struct kvm_vcpu       vcpu;
 	int                   launched;
 	u8                    fail;
+	u64		      first_tsc;
+	u64		      tsc_this;
 	u32                   idt_vectoring_info;
 	struct kvm_msr_entry *guest_msrs;
 	struct kvm_msr_entry *host_msrs;

@@ -254,6 +256,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx)
 	if (vmx->vcpu.cpu == -1)
 		return;
 	smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1);
+	rdtscll(vmx->tsc_this);
 	vmx->launched = 0;
 }
 

@@ -480,6 +483,8 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx)
 	reload_host_efer(vmx);
 }
 
+static void guest_write_tsc(u64 host_tsc, u64 guest_tsc);
+
 /*
  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
  * vcpu mutex is already taken.

@@ -488,7 +493,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u64 phys_addr = __pa(vmx->vmcs);
-	u64 tsc_this, delta;
+	u64 delta;
 
 	if (vcpu->cpu != cpu) {
 		vcpu_clear(vmx);

Quoted 4 lines · Marcelo Tosatti

The boot TSC sync check is failing on recent Linux SMP guests on TSC stable hosts.

What about tsc unstable hosts? If your patch convinces the guest its tsc is table, while the host tsc is not, then it may cause confusion later on.

Quoted 71 lines

Following patch attempts to address the problems, which are:

1) APIC_DM_STARTUP, which is only called for vcpu's other than vcpu0, will trigger ->vcpu_reset(), setting the TSC to 0. Fix that by moving the guest_write_tsc(0) to vcpu_load.

2) vcpu's are initialized at different times by QEMU (vcpu0 init happens way earlier than the rest). Fix that by initializing the TSC of vcpu's > 0 with reference to vcpu0 init tsc value. This way TSC synchronization is kept (apparently Xen does something similar).

3) The code which adjusts the TSC of a VCPU on physical CPU switch is meant to guarantee that the guest sees a monotonically increasing value. However there is a large gap, in terms of clocks, between the time which the source CPU TSC is read and the time the destination CPU TSC is read. So move those two reads to happen at vcpu_clear.

I believe that 3) is the reason for the -no-kvm-irqchip problems reported by Avi on RHEL5, with kernels < 2.6.21 (where vcpu->cpu pinning would fix the problem). Unfortunately I could reproduce that problem.

4-way guest with constant tick at 250Hz now reliably sees the TSC's synchronized, and idle guest CPU consumption is reduced by 50% (3-4% instead of 8%, the latter with acpi_pm_good boot parameter).

arch/x86/kvm/vmx.c +6 -1

index 4741806..e1c8cf4 100644
@@ -47,6 +47,8 @@ struct vcpu_vmx {
 	struct kvm_vcpu       vcpu;
 	int                   launched;
 	u8                    fail;
+	u64		      first_tsc;
+	u64		      tsc_this;
 	u32                   idt_vectoring_info;
 	struct kvm_msr_entry *guest_msrs;
 	struct kvm_msr_entry *host_msrs;

@@ -254,6 +256,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx)
 	if (vmx->vcpu.cpu == -1)
 		return;
 	smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1);
+	rdtscll(vmx->tsc_this);
 	vmx->launched = 0;
 }
 

@@ -480,6 +483,8 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx)
 	reload_host_efer(vmx);
 }
 
+static void guest_write_tsc(u64 host_tsc, u64 guest_tsc);
+
 /*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.

@@ -488,7 +493,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u64 phys_addr = __pa(vmx->vmcs);
-	u64 tsc_this, delta;
+	u64 delta;
 
 	if (vcpu->cpu != cpu) {

		vcpu_clear(vmx);

arch/x86/kvm/vmx.c +0 -0

@@ -511,6 +516,19 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		struct descriptor_table dt;
 		unsigned long sysenter_esp;
 

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 521

Quoted 2 lines

arch/x86/kvm/vmx.c +1 -0

+		if (unlikely(vcpu->cpu == -1)) {
 

This can happen after migration, I believe.

Quoted 8 lines

arch/x86/kvm/vmx.c +8 -0

+			rdtscll(vcpu->arch.host_tsc);
+			vmx->tsc_this = vcpu->arch.host_tsc;
+			if (vcpu->vcpu_id == 0) {
+				guest_write_tsc(vcpu->arch.host_tsc, 0);
+				vmx->first_tsc = vcpu->arch.host_tsc;
+			} else {
+				struct vcpu_vmx *cpu0;
+				cpu0 = to_vmx(vcpu->kvm->vcpus[0]);

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 534

Quoted 5 lines

arch/x86/kvm/vmx.c +4 -0

+				guest_write_tsc(cpu0->first_tsc, 0);
+			}
+		}
+
 

Depending on vcpu_id == 0 can cause problems (for example, if vcpu 0 is somehow not the first to run).

We might initialize the tsc base on vm creation, and if the host tsc is synced, then the guest tsc should also be.

Quoted 5 lines

		vcpu->cpu = cpu;
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when 
		 switching

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 550

Quoted 9 lines

arch/x86/kvm/vmx.c +1 -2

@@ -526,8 +544,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		/*
 		 * Make sure the time stamp counter is monotonous.
 		 */
-		rdtscll(tsc_this);
-		delta = vcpu->arch.host_tsc - tsc_this;
+		delta = vcpu->arch.host_tsc - vmx->tsc_this;
 		vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta);
 

This is a little roundabout, how about moving the delta calculation immediately after the call to vcpu_clear()?

I don't think this is the cause of the problem, it can't account for more than a few hundred cycles, compared to the much greater vmentry cost.

Anyway it should be in a separate patch.

Hi Avi,

Quoted 9 lines · Avi Kivity

Quoted 4 lines · Marcelo Tosatti

The boot TSC sync check is failing on recent Linux SMP guests on TSC stable hosts.

What about tsc unstable hosts? If your patch convinces the guest its tsc is table, while the host tsc is not, then it may cause confusion later on.

The adjustment to zero won't fool the guest because it assumes that the TSC's are synchronized. It will simply set the guest TSC to zero on all VCPUs based on the time VCPU0 was initialized.

That is, setting -(vcpu[0].first_tsc) on all VCPU's will not correct any synchronization problem.

Quoted 71 lines

Quoted 71 lines

Following patch attempts to address the problems, which are:

1) APIC_DM_STARTUP, which is only called for vcpu's other than vcpu0, will trigger ->vcpu_reset(), setting the TSC to 0. Fix that by moving the guest_write_tsc(0) to vcpu_load.

2) vcpu's are initialized at different times by QEMU (vcpu0 init happens way earlier than the rest). Fix that by initializing the TSC of vcpu's > 0 with reference to vcpu0 init tsc value. This way TSC synchronization is kept (apparently Xen does something similar).

3) The code which adjusts the TSC of a VCPU on physical CPU switch is meant to guarantee that the guest sees a monotonically increasing value. However there is a large gap, in terms of clocks, between the time which the source CPU TSC is read and the time the destination CPU TSC is read. So move those two reads to happen at vcpu_clear.

I believe that 3) is the reason for the -no-kvm-irqchip problems reported by Avi on RHEL5, with kernels < 2.6.21 (where vcpu->cpu pinning would fix the problem). Unfortunately I could reproduce that problem.

4-way guest with constant tick at 250Hz now reliably sees the TSC's synchronized, and idle guest CPU consumption is reduced by 50% (3-4% instead of 8%, the latter with acpi_pm_good boot parameter).

arch/x86/kvm/vmx.c +6 -1

index 4741806..e1c8cf4 100644
@@ -47,6 +47,8 @@ struct vcpu_vmx {
 	struct kvm_vcpu       vcpu;
 	int                   launched;
 	u8                    fail;
+	u64		      first_tsc;
+	u64		      tsc_this;
 	u32                   idt_vectoring_info;
 	struct kvm_msr_entry *guest_msrs;
 	struct kvm_msr_entry *host_msrs;

@@ -254,6 +256,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx)
 	if (vmx->vcpu.cpu == -1)
 		return;
 	smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1);
+	rdtscll(vmx->tsc_this);
 	vmx->launched = 0;
 }
 

@@ -480,6 +483,8 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx)
 	reload_host_efer(vmx);
 }
 
+static void guest_write_tsc(u64 host_tsc, u64 guest_tsc);
+
 /*
 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
 * vcpu mutex is already taken.

@@ -488,7 +493,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u64 phys_addr = __pa(vmx->vmcs);
-	u64 tsc_this, delta;
+	u64 delta;
 
 	if (vcpu->cpu != cpu) {

		vcpu_clear(vmx);

arch/x86/kvm/vmx.c +0 -0

@@ -511,6 +516,19 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		struct descriptor_table dt;
 		unsigned long sysenter_esp;
 

Quoted 4 lines

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 521

Quoted 2 lines

arch/x86/kvm/vmx.c +1 -0

+		if (unlikely(vcpu->cpu == -1)) {
 

This can happen after migration, I believe.

Right now the destination host will do ->vcpu_reset() on all VCPU's on startup... so its already broken. This patch is not introducing any issues, just changing where it happens :)

Perhaps fixing migration should be the subject of a separate patch?

Quoted 8 lines

Quoted 8 lines

arch/x86/kvm/vmx.c +8 -0

+			rdtscll(vcpu->arch.host_tsc);
+			vmx->tsc_this = vcpu->arch.host_tsc;
+			if (vcpu->vcpu_id == 0) {
+				guest_write_tsc(vcpu->arch.host_tsc, 0);
+				vmx->first_tsc = vcpu->arch.host_tsc;
+			} else {
+				struct vcpu_vmx *cpu0;
+				cpu0 = to_vmx(vcpu->kvm->vcpus[0]);

Quoted 8 lines

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcp

Quoted 5 lines

arch/x86/kvm/vmx.c +4 -0

+				guest_write_tsc(cpu0->first_tsc, 0);
+			}
+		}
+
 

Depending on vcpu_id == 0 can cause problems (for example, if vcpu 0 is somehow not the first to run).

But QEMU will always run kvm_create() first (which does kvm_create_vcpu(0)), then start the other threads later. So I don't see how that can happen.

Quoted 21 lines

We might initialize the tsc base on vm creation, and if the host tsc is synced, then the guest tsc should also be.

Quoted 5 lines

		vcpu->cpu = cpu;
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when 
		 switching

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 550

Quoted 9 lines

arch/x86/kvm/vmx.c +1 -2

@@ -526,8 +544,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		/*
 		 * Make sure the time stamp counter is monotonous.
 		 */
-		rdtscll(tsc_this);
-		delta = vcpu->arch.host_tsc - tsc_this;
+		delta = vcpu->arch.host_tsc - vmx->tsc_this;
 		vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta);
 

This is a little roundabout, how about moving the delta calculation immediately after the call to vcpu_clear()?

Quoted 2 lines

I don't think this is the cause of the problem, it can't account for more than a few hundred cycles, compared to the much greater vmentry cost.

Actually it accounts for several thousand cycles warp by the time the kernel checks the TSC synchronization between CPU's, which happens very early on boot.

You saw the hang on userspace init, by then there could be many VCPU->CPU switchs.

Quoted 2 lines

Anyway it should be in a separate patch.

How does the following look (minus proper changelog):

So the first patch seemed alright (taking into account the comments from my previous email, that migration should probably be fixed in a separate patch since its broken already and that its guaranteed that vcpu0 is the first to hit vcpu_load).

Hi Avi,

Quoted 11 lines · Avi Kivity

Quoted 5 lines · Marcelo Tosatti

The boot TSC sync check is failing on recent Linux SMP guests on TSC stable hosts.

What about tsc unstable hosts? If your patch convinces the guest its tsc is table, while the host tsc is not, then it may cause confusion later on.

The adjustment to zero won't fool the guest because it assumes that the TSC's are synchronized. It will simply set the guest TSC to zero on all VCPUs based on the time VCPU0 was initialized.

That is, setting -(vcpu[0].first_tsc) on all VCPU's will not correct any synchronization problem.

Quoted 20 lines

Quoted 20 lines

(quoted diff) +2 -0

+static void guest_write_tsc(u64 host_tsc, u64 guest_tsc);
+

/*
* Switches to specified vcpu, until a matching vcpu_put(), but assumes
* vcpu mutex is already taken.

arch/x86/kvm/vmx.c +1 -1

@@ -488,7 +493,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u64 phys_addr = __pa(vmx->vmcs);
-	u64 tsc_this, delta;
+	u64 delta;
 
 	if (vcpu->cpu != cpu) {

		vcpu_clear(vmx);

arch/x86/kvm/vmx.c +0 -0

@@ -511,6 +516,19 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		struct descriptor_table dt;
 		unsigned long sysenter_esp;
 

Quoted 5 lines

Review static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 522

Quoted 3 lines

(quoted diff) +1 -0

+		if (unlikely(vcpu->cpu == -1)) {
 
 

This can happen after migration, I believe.

Right now the destination host will do ->vcpu_reset() on all VCPU's on startup... so its already broken. This patch is not introducing any issues, just changing where it happens :)

Quoted 19 lines

Quoted 6 lines

arch/x86/kvm/vmx.c +2 -0

+			rdtscll(vcpu->arch.host_tsc);
+			vmx->tsc_this = vcpu->arch.host_tsc;

arch/x86/kvm/vmx.c static void vmx_vcpu_load(struct kvm_vcp · line 527

Quoted 10 lines

arch/x86/kvm/vmx.c +10 -0

+			if (vcpu->vcpu_id == 0) {
+				guest_write_tsc(vcpu->arch.host_tsc, 0);
+				vmx->first_tsc = vcpu->arch.host_tsc;
+			} else {
+				struct vcpu_vmx *cpu0;
+				cpu0 = to_vmx(vcpu->kvm->vcpus[0]);
+				guest_write_tsc(cpu0->first_tsc, 0);
+			}
+		}
+

Depending on vcpu_id == 0 can cause problems (for example, if vcpu 0 is somehow not the first to run).

But QEMU will always run kvm_create() first (which does kvm_create_vcpu(0)), then start the other threads later. So I don't see how that can happen.

Quoted 18 lines

Quoted 6 lines

		vcpu->cpu = cpu;
		/*
		 * Linux uses per-cpu TSS and GDT, so set these when 
		 switching

Review static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int · line 550

Quoted 9 lines

(quoted diff) +1 -2

@@ -526,8 +544,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int 
 cpu)
 		/*
 		 * Make sure the time stamp counter is monotonous.
 		 */
-		rdtscll(tsc_this);
-		delta = vcpu->arch.host_tsc - tsc_this;
+		delta = vcpu->arch.host_tsc - vmx->tsc_this;
 		vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta);
 

This is a little roundabout, how about moving the delta calculation immediately after the call to vcpu_clear()?

Quoted 3 lines

I don't think this is the cause of the problem, it can't account for more than a few hundred cycles, compared to the much greater vmentry cost.

Actually it accounts for several thousand cycles warp by the time the kernel checks the TSC synchronization between CPU's, which happens very early on boot.

You saw the hang on userspace init, by then there could be many VCPU->CPU switchs.

Quoted 2 lines

Anyway it should be in a separate patch.

How does the following look (minus proper changelog):

Index: kvm.quilt/arch/x86/kvm/vmx.c
===================================================================

arch/x86/kvm/vmx.c +5 -0

@@ -47,6 +47,7 @@ struct vcpu_vmx {
 	struct kvm_vcpu       vcpu;
 	int                   launched;
 	u8                    fail;
+	u64		      first_tsc;
 	u32                   idt_vectoring_info;
 	struct kvm_msr_entry *guest_msrs;
 	struct kvm_msr_entry *host_msrs;

@@ -480,6 +481,7 @@ static void vmx_load_host_state(struct v
 	reload_host_efer(vmx);
 }
 
+static void guest_write_tsc(u64 host_tsc, u64 guest_tsc);
 /*
  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
  * vcpu mutex is already taken.

@@ -511,6 +513,18 @@ static void vmx_vcpu_load(struct kvm_vcp
 		struct descriptor_table dt;
 		unsigned long sysenter_esp;
 
+		if (unlikely(vcpu->cpu == -1)) {
 
+			rdtscll(vcpu->arch.host_tsc);
+			if (vcpu->vcpu_id == 0) {

arch/x86/kvm/vmx.c +17 -0

+static void __vcpu_clear(void *arg)
+{
+	struct vcpu_vmx *vmx = arg;
+	int cpu = raw_smp_processor_id();
+
+	if (vmx->vcpu.cpu == cpu)
+		vmcs_clear(vmx->vmcs);
+	if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
+		per_cpu(current_vmcs, cpu) = NULL;
+	rdtscll(vmx->vcpu.arch.host_tsc);
+}
+
+static void vcpu_clear(struct vcpu_vmx *vmx)
+{
+	u64 tsc_this, delta;
+
+	if (vmx->vcpu.cpu == -1)

Quoted 19 lines · Marcelo Tosatti

Hi Avi,

Quoted 9 lines · Avi Kivity

Quoted 6 lines · Marcelo Tosatti

The boot TSC sync check is failing on recent Linux SMP guests on TSC stable hosts.

Quoted 5 lines

What about tsc unstable hosts? If your patch convinces the guest its tsc is table, while the host tsc is not, then it may cause confusion later on.

Quoted 9 lines

The adjustment to zero won't fool the guest because it assumes that the TSC's are synchronized. It will simply set the guest TSC to zero on all VCPUs based on the time VCPU0 was initialized.

That is, setting -(vcpu[0].first_tsc) on all VCPU's will not correct any synchronization problem.

What I mean is, right now we present really broken tscs to the guest. After your patch, we present less-broken tscs (at boot, they will closely resemble stable tscs). But after the machine idles a bit and cpufreq takes over, the tscs won't be stable any more.

Quoted 10 lines

Quoted 3 lines

What I mean is, right now we present really broken tscs to the guest. After your patch, we present less-broken tscs (at boot, they will closely resemble stable tscs). But after the machine idles a bit and cpufreq takes over, the tscs won't be stable any more.

Why would the TSC break due to cpufreq? This patchset was against VMX, which is only available on current Intel CPUs. All of those guarantee a constant TSC increase at the maximum frequency.

Also see the Intel Documentation:

Vol. 3 18-37

For Pentium 4 processors, Intel Xeon processors (family [0FH], models [03H and higher]); for Intel Core Solo and Intel Core Duo processors (family [06H], model [0EH]); for the Intel Xeon processor 5100 series and Intel Core 2 Duo processors (family [06H], model [0FH]): the time-stamp counter increments at a constant rate. That rate may be set by the maximum core-clock to bus-clock ratio of the processor or may be set by the maximum resolved frequency at which the processor is booted. The maximum resolved frequency may differ from the maximum qualified frequency of the processor, see Section 18.17.5 for more detail.
The specific processor configuration determines the behavior. Constant TSC behavior ensures that the duration of each clock tick is uniform and supports the use of the TSC as a wall clock timer even if the processor core changes frequency.
This is the architectural behavior moving forward.

Quoted 20 lines · Alexander Graf

Quoted 4 lines

Quoted 4 lines

Quoted 8 lines

What I mean is, right now we present really broken tscs to the guest. After your patch, we present less-broken tscs (at boot, they will closely resemble stable tscs). But after the machine idles a bit and cpufreq takes over, the tscs won't be stable any more.

Why would the TSC break due to cpufreq? This patchset was against VMX, which is only available on current Intel CPUs. All of those guarantee a constant TSC increase at the maximum frequency.

Also see the Intel Documentation:

Vol. 3 18-37

Quoted 16 lines

For Pentium 4 processors, Intel Xeon processors (family [0FH], models [03H and higher]); for Intel Core Solo and Intel Core Duo processors (family [06H], model [0EH]); for the Intel Xeon processor 5100 series and Intel Core 2 Duo processors (family [06H], model [0FH]): the time-stamp counter increments at a constant rate. That rate may be set by the maximum core-clock to bus-clock ratio of the processor or may be set by the maximum resolved frequency at which the processor is booted. The maximum resolved frequency may differ from the maximum qualified frequency of the processor, see Section 18.17.5 for more detail.
The specific processor configuration determines the behavior. Constant TSC behavior ensures that the duration of each clock tick is uniform and supports the use of the TSC as a wall clock timer even if the processor core changes frequency.
This is the architectural behavior moving forward.

Thanks; that's reassuring to know that it will work (at least on Intel).

If I remember correctly, there was a statement on LKML by AMD which said that the TSC is completely broken on AMD systems, so you should not use it there anyway.

Quoted 3 lines · Alexander Graf

If I remember correctly, there was a statement on LKML by AMD which said that the TSC is completely broken on AMD systems, so you should not use it there anyway.

Well, that's up to the guest to decide. Looks like older Linux guests make the wrong decision.

Thanks; that's reassuring to know that it will work (at least on Intel).

Quoted 2 lines · Avi Kivity

Thanks; that's reassuring to know that it will work (at least on Intel).

Actually there are modern Intel systems which still have instable TSCs; e.g. IBM Summit multi node systems and some others. So you should still handle that case.

[fixing gmane emails, urgfhsz]

Quoted 6 lines · Andi Kleen

Quoted 2 lines · Avi Kivity

Thanks; that's reassuring to know that it will work (at least on Intel).

Quoted 4 lines

Actually there are modern Intel systems which still have instable TSCs; e.g. IBM Summit multi node systems and some others. So you should still handle that case.

I really don't see any way we could. If the guest assumes tscs are synchronous, and they really are not, there's nothing we can do.

You might taskset guests into a single node on such systems, which is a good idea anyway.

Quoted 3 lines · Andi Kleen

Linux checks a couple of things: e.g. if there are no deep C states and if there are no clustered nodes in the APIC etc.

Quoted 5 lines

It might be reasonable to check the clock source of the kernel and if it's not TSC force one of these in the emulated firmware environment

The problems are with older guests which assume the tsc is okay. Newer guests check the tsc and conclude that it isn't usable.

Quoted 7 lines

Quoted 4 lines

You might taskset guests into a single node on such systems, which is a good idea anyway.

Ah pushing the problem to the user. An easy, but typically wrong, solution.

If you have other suggestions I'll be happy to hear them. I don't like this either.

Check if host is using TSC source and if not force a clustered APIC mode (only works for 64bit unfortunately) or fake a C3 state in ACPI and on AMD clear the synchronous TSC bit.

Quoted 4 lines · Avi Kivity

Quoted 3 lines · Andi Kleen

Linux checks a couple of things: e.g. if there are no deep C states and if there are no clustered nodes in the APIC etc.

Quoted 8 lines

Quoted 5 lines

It might be reasonable to check the clock source of the kernel and if it's not TSC force one of these in the emulated firmware environment

The problems are with older guests which assume the tsc is okay. Newer guests check the tsc and conclude that it isn't usable.

If the guest would get it wrong running natively on the host I guess it would be reasonable to require an option that forces TSC off. Disabling the TSC bit unfortunately won't work for 64bit guests, but for probably most 32bit guests.

But for non broken guests they can only do that if the guest has the same visibility into the firmware state as the host. For the easy cases Linux will check it anyways becaused on standard the TSC synchronicity check, but there are cases where the TSCs only drift apart slowly over a longer time

[I finally fixed the clocksource watchdog now to catch this case, but it will be only in .25]

I think it would be better to fake at least some of the usual firmware cues for bad TSC if the host does not use it.

Quoted 8 lines

Quoted 7 lines

Quoted 4 lines

You might taskset guests into a single node on such systems, which is a good idea anyway.

Ah pushing the problem to the user. An easy, but typically wrong, solution.

Quoted 2 lines

If you have other suggestions I'll be happy to hear them. I don't like this either.

Check if host is using TSC source and if not force a clustered APIC mode (only works for 64bit unfortunately) or fake a C3 state in ACPI and on AMD clear the synchronous TSC bit.

Quoted 4 lines

I ran the same image under qemu 0.9.0, uniprocessor, and got the same
results: high host cpu usage while the guest is idle. Most likely the BIOS ACPI tables are misconfigured and the Windows idle loop is

confused.

And turning of the TSC bit (for 32-bit guests).

Avi, do you understand why C3 was causing the Windows SMP problems ?

Forcing clustered APIC mode works only on SMP, and there were high CPU consumption on Windows SMP guests due to C3 state being reported (fixed in kvm-30 something).

So perhaps:
- Faking clustered APIC on SMP
- Faking C3 on UP

And turning of the TSC bit (for 32-bit guests).

Is the way to go?

Avi, do you understand why C3 was causing the Windows SMP problems ?

/* Common C-state entry for C2, C3, .. */
static void acpi_cstate_enter(struct acpi_processor_cx *cstate)

{
        if (cstate->space_id == ACPI_CSTATE_FFH) {
                /* Call into architectural FFH based C-state */
                acpi_processor_ffh_cstate_enter(cstate);
        } else {
                int unused;
                /* IO port based C-state */
                inb(cstate->address);
                /* Dummy wait op - must do something useless after P_LVL2 read
                   because chipsets cannot guarantee that STPCLK# signal
                   gets asserted in time to freeze execution properly. */
                unused = inl(acpi_gbl_FADT.xpm_timer_block.address);
        }
}

Clearly that inb() won't actually idle under QEMU. So the question is, if C3 stated is reported, that port read should be emulated... But how?

[Resurrecting post from the dead]

Quoted 9 lines · Marcelo Tosatti

Forcing clustered APIC mode works only on SMP, and there were high CPU consumption on Windows SMP guests due to C3 state being reported (fixed in kvm-30 something).

So perhaps:
- Faking clustered APIC on SMP - Faking C3 on UP

And turning of the TSC bit (for 32-bit guests).

Quoted 4 lines

Is the way to go?
Avi, do you understand why C3 was causing the Windows SMP problems ?

It's probably inb()ing on the port in a loop. It's not SMP causing the problems, but the ACPI HAL. I'll check this.