Skip to contentSkip to conversation

x86/hyperv: Allow guests to enable InvariantTSC

PATCH Applied Andrea Parri 4 participants 7 messages Oct 3, 2019 – Nov 12, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 x86/hyperv: Allow guests to enable InvariantTSC

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

---
 arch/x86/include/asm/hyperv-tlfs.h | 5 +++++
 arch/x86/kernel/cpu/mshyperv.c     | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)
arch/x86/include/asm/hyperv-tlfs.h +5 -0
index 7741e211f7f51..5f10f7f2098db 100644
@@ -86,6 +86,8 @@
 #define HV_X64_ACCESS_FREQUENCY_MSRS		BIT(11)
 /* AccessReenlightenmentControls privilege */
 #define HV_X64_ACCESS_REENLIGHTENMENT		BIT(13)
+/* AccessTscInvariantControls privilege */
+#define HV_X64_ACCESS_TSC_INVARIANT		BIT(15)
 
 /*
  * Feature identification: indicates which flags were specified at partition
@@ -278,6 +280,9 @@
 #define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
 #define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
 
+/* TSC invariant control */
+#define HV_X64_MSR_TSC_INVARIANT_CONTROL	0x40000118
+
 /*
  * Declare the MSR used to setup pages used to communicate with the hypervisor.
  */
arch/x86/kernel/cpu/mshyperv.c +6 -1
index 267daad8c0360..105844d542e5c 100644
@@ -286,7 +286,12 @@ static void __init ms_hyperv_init_platform(void)
 	machine_ops.shutdown = hv_machine_shutdown;
 	machine_ops.crash_shutdown = hv_machine_crash_shutdown;
 #endif
-	mark_tsc_unstable("running on Hyper-V");
+	if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) {
+		wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1);
+		setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
+	} else {
+		mark_tsc_unstable("running on Hyper-V");
+	}
 
 	/*
 	 * Generation 2 instances don't support reading the NMI status from
Signature · 1 line
2.23.0
#1 lore raw
Quoted 10 lines · Andrea Parri

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

Subject should have been "[PATCH] ...", i.e., there is no 2/2 planned (not for this patchset at least). Please let me know if I should re- submit with the subject fixed.

Thanks,
Andrea

Reviewed-by on v1 1/2
#2 lore raw
In reply to Andrea Parri

From: Andrea Parri <[email protected]> Sent: Thursday, October 3, 2019 8:52 AM

Quoted 10 lines

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

Quoted 6 lines
---
 arch/x86/include/asm/hyperv-tlfs.h | 5 +++++
 arch/x86/kernel/cpu/mshyperv.c     | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

As noted in a separate email, this patch is standalone, not 1 of 2 as indicated in the subject line. Modulo that,

#3 lore raw
In reply to Andrea Parri
Quoted 8 lines · Andrea Parri

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

I tried getting more information from TLFS but as of 5.0C this feature is not described there. I'm really interested in why this additional interface is needed, e.g. why can't Hyper-V just set InvariantTSC unconditionally when TSC scaling is supported?

Quoted 48 lines
---
 arch/x86/include/asm/hyperv-tlfs.h | 5 +++++
 arch/x86/kernel/cpu/mshyperv.c     | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)
arch/x86/include/asm/hyperv-tlfs.h +5 -0
index 7741e211f7f51..5f10f7f2098db 100644
@@ -86,6 +86,8 @@
 #define HV_X64_ACCESS_FREQUENCY_MSRS		BIT(11)
 /* AccessReenlightenmentControls privilege */
 #define HV_X64_ACCESS_REENLIGHTENMENT		BIT(13)
+/* AccessTscInvariantControls privilege */
+#define HV_X64_ACCESS_TSC_INVARIANT		BIT(15)
 
 /*
  * Feature identification: indicates which flags were specified at partition
@@ -278,6 +280,9 @@
 #define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
 #define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
 
+/* TSC invariant control */
+#define HV_X64_MSR_TSC_INVARIANT_CONTROL	0x40000118
+
 /*
  * Declare the MSR used to setup pages used to communicate with the hypervisor.
  */
arch/x86/kernel/cpu/mshyperv.c +6 -1
index 267daad8c0360..105844d542e5c 100644
@@ -286,7 +286,12 @@ static void __init ms_hyperv_init_platform(void)
 	machine_ops.shutdown = hv_machine_shutdown;
 	machine_ops.crash_shutdown = hv_machine_crash_shutdown;
 #endif
-	mark_tsc_unstable("running on Hyper-V");
+	if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) {
+		wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1);
+		setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
+	} else {
+		mark_tsc_unstable("running on Hyper-V");
+	}
 
 	/*
 	 * Generation 2 instances don't support reading the NMI status from
Signature · 1 line
Vitaly
#4 lore raw

From: Vitaly Kuznetsov <[email protected]> Sent: Friday, October 4, 2019 9:57 AM

Quoted 12 lines
Quoted 8 lines · Andrea Parri

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

Quoted 5 lines

I tried getting more information from TLFS but as of 5.0C this feature is not described there. I'm really interested in why this additional interface is needed, e.g. why can't Hyper-V just set InvariantTSC unconditionally when TSC scaling is supported?

Yes, this is very new functionality that is not yet available in a released version of Hyper-V. And as you know, the Hyper-V TLFS has gotten woefully out-of-date. :-(

Your question is the same question I asked. The reason given by
Hyper-V is to take the more cautious approach of not "automatically" giving VMs an InvariantTSC due to updating the underlying Hyper-V version. Instead, guest VMs must have been explicitly coded to take advantage of the new InvariantTSC feature. It's not clear to me how much of this caution is driven by Windows guests vs. Linux or FreeBSD guests, but it is what it is.

Having to explicitly enable the InvariantTSC does give the Linux code the opportunity to be a bit cleaner by doing things like not marking the TSC as unstable when the InvariantTSC feature is present, and to mark the TSC as reliable so we don't try to do TSC synchronization (which Hyper-V does not want guests to try to do).

Michael

Reviewed-by
#5 lore raw
Quoted 31 lines · Michael Kelley

From: Vitaly Kuznetsov <[email protected]> Sent: Friday, October 4, 2019 9:57 AM

Quoted 12 lines
Quoted 8 lines · Andrea Parri

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

Quoted 5 lines

I tried getting more information from TLFS but as of 5.0C this feature is not described there. I'm really interested in why this additional interface is needed, e.g. why can't Hyper-V just set InvariantTSC unconditionally when TSC scaling is supported?

Yes, this is very new functionality that is not yet available in a released version of Hyper-V. And as you know, the Hyper-V TLFS has gotten woefully out-of-date. :-(

Your question is the same question I asked. The reason given by
Hyper-V is to take the more cautious approach of not "automatically" giving VMs an InvariantTSC due to updating the underlying Hyper-V version. Instead, guest VMs must have been explicitly coded to take advantage of the new InvariantTSC feature. It's not clear to me how much of this caution is driven by Windows guests vs. Linux or FreeBSD guests, but it is what it is.

Quoted 5 lines

Having to explicitly enable the InvariantTSC does give the Linux code the opportunity to be a bit cleaner by doing things like not marking the TSC as unstable when the InvariantTSC feature is present, and to mark the TSC as reliable so we don't try to do TSC synchronization (which Hyper-V does not want guests to try to do).

Thank you for these additional details Michael,

we'll probably have to add support for this bit to KVM and I'd like to know the background. From Linux perspective, no matter what's the interface we'd like to get InvariantTSC.

Feel free to add

Signature · 1 line
Vitaly
merge bot Applied
#6 lore raw
In reply to Andrea Parri
The following commit has been merged into the x86/hyperv branch of tip:

The following commit has been merged into the x86/hyperv branch of tip:

Commit-ID:     dce7cd62754b5d4a6e401b8b0769ec94cf971041
Gitweb:        https://git.kernel.org/tip/dce7cd62754b5d4a6e401b8b0769ec94cf971041
Author:        Andrea Parri <[email protected]>
AuthorDate:    Thu, 03 Oct 2019 17:52:00 +02:00
Committer:     Thomas Gleixner <[email protected]>
CommitterDate: Tue, 12 Nov 2019 11:44:21 +01:00

x86/hyperv: Allow guests to enable InvariantTSC

If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC.

---
 arch/x86/include/asm/hyperv-tlfs.h | 5 +++++
 arch/x86/kernel/cpu/mshyperv.c     | 7 ++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)
arch/x86/include/asm/hyperv-tlfs.h +5 -0
index 7a27056..887b1d6 100644
@@ -86,6 +86,8 @@
 #define HV_X64_ACCESS_FREQUENCY_MSRS		BIT(11)
 /* AccessReenlightenmentControls privilege */
 #define HV_X64_ACCESS_REENLIGHTENMENT		BIT(13)
+/* AccessTscInvariantControls privilege */
+#define HV_X64_ACCESS_TSC_INVARIANT		BIT(15)
 
 /*
  * Feature identification: indicates which flags were specified at partition
@@ -270,6 +272,9 @@
 #define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
 #define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
 
+/* TSC invariant control */
+#define HV_X64_MSR_TSC_INVARIANT_CONTROL	0x40000118
+
 /*
  * Declare the MSR used to setup pages used to communicate with the hypervisor.
  */
arch/x86/kernel/cpu/mshyperv.c +6 -1
index 062f772..6f7c822 100644
@@ -285,7 +285,12 @@ static void __init ms_hyperv_init_platform(void)
 	machine_ops.shutdown = hv_machine_shutdown;
 	machine_ops.crash_shutdown = hv_machine_crash_shutdown;
 #endif
-	mark_tsc_unstable("running on Hyper-V");
+	if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) {
+		wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1);
+		setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
+	} else {
+		mark_tsc_unstable("running on Hyper-V");
+	}
 
 	/*
 	 * Generation 2 instances don't support reading the NMI status from