Add support running nested Microsoft Hypervisor

When Linux runs as a root partition for Microsoft Hypervisor. It is possible to detect if it is running as nested hypervisor using hints exposed by mshv. While at it expose a new variable called hv_nested which can be used later for making decisions specific to nested use case.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 3 +++
 arch/x86/include/asm/mshyperv.h    | 2 ++
 arch/x86/kernel/cpu/mshyperv.c     | 7 +++++++
 3 files changed, 12 insertions(+)

arch/x86/include/asm/hyperv-tlfs.h +3 -0

index 3089ec352743..d9a611565859 100644
@@ -114,6 +114,9 @@
 /* Recommend using the newer ExProcessorMasks interface */
 #define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED		BIT(11)
 
+/* Indicates that the hypervisor is nested within a Hyper-V partition. */
+#define HV_X64_HYPERV_NESTED				BIT(12)
+
 /* Recommend using enlightened VMCS */
 #define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED		BIT(14)
 

arch/x86/include/asm/mshyperv.h +2 -0

index 61f0c206bff0..29388567eafd 100644
@@ -190,6 +190,8 @@ static inline void hv_ghcb_terminate(unsigned int set, unsigned int reason) {}
 
 extern bool hv_isolation_type_snp(void);
 
+extern bool hv_nested;
+
 static inline bool hv_is_synic_reg(unsigned int reg)
 {
 	if ((reg >= HV_REGISTER_SCONTROL) &&

arch/x86/kernel/cpu/mshyperv.c +2 -0

index 831613959a92..2555535f5237 100644
@@ -37,6 +37,8 @@
 
 /* Is Linux running as the root partition? */
 bool hv_root_partition;
+/* Is Linux running on nested Microsoft Hypervisor */
+bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
 #if IS_ENABLED(CONFIG_HYPERV)

arch/x86/kernel/cpu/mshyperv.c +2 -1

+	}
+
 	/*
 	 * Extract host information.
 	 */
--

2.25.1

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 11 +++++++++++
 arch/x86/include/asm/mshyperv.h    | 24 ++++++++++++++++++++++++
 drivers/hv/hv.c                    | 18 +++++++++++++-----
 3 files changed, 48 insertions(+), 5 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index d9a611565859..0319091e2019 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
 #define HV_REGISTER_SINT14			0x4000009E
 #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
 /*
  * Synthetic Timer MSRs. Four timers per vcpu.
  */

arch/x86/include/asm/mshyperv.h +24 -0

index 29388567eafd..415289757428 100644
@@ -200,10 +200,31 @@ static inline bool hv_is_synic_reg(unsigned int reg)
 	return false;
 }
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:
+		return HV_REGISTER_SIEFP;
+	case HV_REGISTER_SCONTROL:
+		return HV_REGISTER_NESTED_SCONTROL;
+	case HV_REGISTER_SINT0:
+		return HV_REGISTER_NESTED_SINT0;
+	case HV_REGISTER_EOM:
+		return HV_REGISTER_NESTED_EOM;
+	default:
+		return reg;
+	}
+}
+
 static inline u64 hv_get_register(unsigned int reg)
 {
 	u64 value;
 
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
 	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
 		hv_ghcb_msr_read(reg, &value);
 	else

@@ -213,6 +234,9 @@ static inline u64 hv_get_register(unsigned int reg)
 
 static inline void hv_set_register(unsigned int reg, u64 value)
 {
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
 	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
 		hv_ghcb_msr_write(reg, value);
 

drivers/hv/hv.c +0 -0

index 4d6480d57546..9e1eb50cc76f 100644

drivers/hv/hv.c +12 -5

 			hv_cpu->synic_message_page =
 				(void *)get_zeroed_page(GFP_ATOMIC);
 			if (hv_cpu->synic_message_page == NULL) {

@@ -188,8 +188,16 @@ void hv_synic_free(void)
 		struct hv_per_cpu_context *hv_cpu
 			= per_cpu_ptr(hv_context.cpu_context, cpu);
 
-		free_page((unsigned long)hv_cpu->synic_event_page);
-		free_page((unsigned long)hv_cpu->synic_message_page);
+		if (hv_root_partition) {
+			if (hv_cpu->synic_event_page != NULL)
+				memunmap(hv_cpu->synic_event_page);
+
+			if (hv_cpu->synic_message_page != NULL)
+				memunmap(hv_cpu->synic_message_page);
+		} else {
+			free_page((unsigned long)hv_cpu->synic_event_page);
+			free_page((unsigned long)hv_cpu->synic_message_page);
+		}
 		free_page((unsigned long)hv_cpu->post_msg_page);
 	}
 

@@ -216,7 +224,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
 	simp.simp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_message_page
 			= memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
 				   HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -233,7 +241,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
 	siefp.siefp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_event_page =
 			memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
 				 HV_HYP_PAGE_SIZE, MEMREMAP_WB);
--

2.25.1

According to TLFS, in order to communicate to L0 hypervisor there needs to be an additional bit set in the control register. This communication is required to perform priviledged instructions which can only be performed by L0 hypervisor. An example of that could be setting up the VMBus infrastructure.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h |  3 ++-
 arch/x86/include/asm/mshyperv.h    | 42 +++++++++++++++++++++++++++---
 include/asm-generic/hyperv-tlfs.h  |  1 +
 include/asm-generic/mshyperv.h     |  1 +
 4 files changed, 42 insertions(+), 5 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +2 -1

index 0319091e2019..fd066226f12b 100644
@@ -380,7 +380,8 @@ struct hv_nested_enlightenments_control {
 		__u32 reserved:31;
 	} features;
 	struct {
-		__u32 reserved;
+		__u32 inter_partition_comm:1;
+		__u32 reserved:31;
 	} hypercallControls;
 } __packed;
 

arch/x86/include/asm/mshyperv.h +8 -2

index 415289757428..451d8c3ab63b 100644
@@ -74,10 +74,16 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void
 *output)
 	return hv_status;
 }
 
+/* Hypercall to the L0 hypervisor */
+static inline u64 hv_do_nested_hypercall(u64 control, void *input, void *output)
+{
+	return hv_do_hypercall(control | HV_HYPERCALL_NESTED, input, output);
+}
+
 /* Fast hypercall with 8 bytes of input and no output */
-static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
+static inline u64 _hv_do_fast_hypercall8(u64 control, u16 code, u64 input1)
 {
-	u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
+	u64 hv_status;
 
 #ifdef CONFIG_X86_64

arch/x86/include/asm/mshyperv.h +30 -2

 	{

@@ -105,10 +111,24 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 		return hv_status;
 }
 
+static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT;
+
+	return _hv_do_fast_hypercall8(control, code, input1);
+}
+
+static inline u64 hv_do_fast_nested_hypercall8(u16 code, u64 input1)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT | HV_HYPERCALL_NESTED;
+
+	return _hv_do_fast_hypercall8(control, code, input1);
+}
+
 /* Fast hypercall with 16 bytes of input */
-static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
+static inline u64 _hv_do_fast_hypercall16(u64 control, u16 code, u64 input1, u64 input2)
 {
-	u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
+	u64 hv_status;
 
 #ifdef CONFIG_X86_64
 	{

@@ -139,6 +159,20 @@ static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
 	return hv_status;
 }
 
+static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT;
+
+	return _hv_do_fast_hypercall16(control, code, input1, input2);
+}
+
+static inline u64 hv_do_fast_nested_hypercall16(u16 code, u64 input1, u64 input2)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT | HV_HYPERCALL_NESTED;
+
+	return _hv_do_fast_hypercall16(control, code, input1, input2);
+}
+
 extern struct hv_vp_assist_page **hv_vp_assist_page;
 
 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)

include/asm-generic/hyperv-tlfs.h +0 -0

index fdce7a4cfc6f..c67836dd8468 100644

include/asm-generic/mshyperv.h +0 -1

 extern u64 hv_do_fast_hypercall8(u16 control, u64 input8);
 extern bool hv_isolation_type_snp(void);
 
--

2.25.1

Currently VMBus driver is not initialized for root partition but we need to enable the VMBus driver for nested root partition. This is required to expose VMBus devices to the L2 guest in the nested setup.

Signed-off-by Jinank Jain <[email protected]>

---
 drivers/hv/vmbus_drv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

drivers/hv/vmbus_drv.c +0 -0

index 8b2e413bf19c..2f0cf75e811b 100644

drivers/hv/vmbus_drv.c +0 -1

 		return 0;
 
 	/*
--

2.25.1

From: Jinank Jain <[email protected]> Sent: Wednesday, November 2, 2022 9:36 AM

Quoted 18 lines

Currently VMBus driver is not initialized for root partition but we need to enable the VMBus driver for nested root partition. This is required to expose VMBus devices to the L2 guest in the nested setup.

Signed-off-by Jinank Jain <[email protected]>

---
 drivers/hv/vmbus_drv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

drivers/hv/vmbus_drv.c +0 -0

index 8b2e413bf19c..2f0cf75e811b 100644
@@ -2723,7 +2723,7 @@ static int __init hv_acpi_init(void)
 	if (!hv_is_hyperv_initialized())
 		return -ENODEV;
 

drivers/hv/vmbus_drv.c static int __init hv_acpi_init(void) · line 2726

Quoted 2 lines

drivers/hv/vmbus_drv.c +1 -1

-	if (hv_root_partition)
+	if (hv_root_partition && !hv_nested)

Note that this code must compile and run when Linux is built to run as a guest on Hyper-V for ARM64. There's currently no definition for hv_nested on the ARM64 side, so the compile will fail. But per my comments in Patch 1 in this series, using the same technique as for hv_root_partition in hv_common.c should solve the ARM64 problem as well.

Currently VMBus driver is not initialized for root partition but we need to enable the VMBus driver for nested root partition. This is required to expose VMBus devices to the L2 guest in the nested setup.

Signed-off-by Jinank Jain <[email protected]>

---
 drivers/hv/vmbus_drv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

drivers/hv/vmbus_drv.c +1 -1

index 8b2e413bf19c..2f0cf75e811b 100644
@@ -2723,7 +2723,7 @@ static int __init hv_acpi_init(void)
 	if (!hv_is_hyperv_initialized())
 		return -ENODEV;
 
-	if (hv_root_partition)
+	if (hv_root_partition && !hv_nested)
 		return 0;
 
 	/*

Signature · 1 line

2.25.1

When Linux runs as a root partition for Microsoft Hypervisor. It is possible to detect if it is running as nested hypervisor using hints exposed by mshv. While at it expose a new variable called hv_nested which can be used later for making decisions specific to nested use case.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/arm64/hyperv/mshyperv.c       | 6 ++++++
 arch/x86/include/asm/hyperv-tlfs.h | 3 +++
 arch/x86/kernel/cpu/mshyperv.c     | 7 +++++++
 drivers/hv/hv_common.c             | 7 +++++--
 include/asm-generic/mshyperv.h     | 1 +
 5 files changed, 22 insertions(+), 2 deletions(-)

arch/arm64/hyperv/mshyperv.c +3 -0

index a406454578f0..2024b19dc514 100644
@@ -19,6 +19,9 @@
 
 static bool hyperv_initialized;
 
+/* Is Linux running on nested Microsoft Hypervisor */
+bool hv_nested;
+
 static int __init hyperv_init(void)
 {
 	struct hv_get_vp_registers_output	result;

arch/x86/include/asm/hyperv-tlfs.h +3 -0

index 6d9368ea3701..58c03d18c235 100644
@@ -114,6 +114,9 @@
 /* Recommend using the newer ExProcessorMasks interface */
 #define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED		BIT(11)
 
+/* Indicates that the hypervisor is nested within a Hyper-V partition. */
+#define HV_X64_HYPERV_NESTED				BIT(12)
+
 /* Recommend using enlightened VMCS */
 #define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED		BIT(14)
 

arch/x86/kernel/cpu/mshyperv.c +7 -0

index 831613959a92..9a4204139490 100644
@@ -37,6 +37,8 @@
 
 /* Is Linux running as the root partition? */
 bool hv_root_partition;
+/* Is Linux running on nested Microsoft Hypervisor */
+bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
 #if IS_ENABLED(CONFIG_HYPERV)

@@ -301,6 +303,11 @@ static void __init ms_hyperv_init_platform(void)
 		pr_info("Hyper-V: running as root partition\n");
 	}
 
+	if (ms_hyperv.hints & HV_X64_HYPERV_NESTED) {
+		hv_nested = true;
+		pr_info("Hyper-V: running on a nested hypervisor\n");
+	}
+
 	/*
 	 * Extract host information.
 	 */

drivers/hv/hv_common.c +0 -0

index ae68298c0dca..dcb336ce374f 100644

drivers/hv/hv_common.c +3 -0

@@ -36,6 +36,9 @@
 bool __weak hv_root_partition;
 EXPORT_SYMBOL_GPL(hv_root_partition);
 
+bool __weak hv_nested;
+EXPORT_SYMBOL_GPL(hv_nested);
+
 struct ms_hyperv_info __weak ms_hyperv;
 EXPORT_SYMBOL_GPL(ms_hyperv);
 

include/asm-generic/mshyperv.h +1 -1

index bfb9eb9d7215..5df6e944e6a9 100644
@@ -164,6 +164,7 @@ extern int vmbus_interrupt;
 extern int vmbus_irq;
 
 extern bool hv_root_partition;
+extern bool hv_nested;
 
 #if IS_ENABLED(CONFIG_HYPERV)
 /*
--

2.25.1

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 11 +++++++
 arch/x86/include/asm/mshyperv.h    | 26 ++--------------
 arch/x86/kernel/cpu/mshyperv.c     | 49 ++++++++++++++++++++++++++++++
 drivers/hv/hv.c                    | 18 ++++++++---
 4 files changed, 75 insertions(+), 29 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index 58c03d18c235..b5019becb618 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
 #define HV_REGISTER_SINT14			0x4000009E
 #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
 /*
  * Synthetic Timer MSRs. Four timers per vcpu.
  */

arch/x86/include/asm/mshyperv.h +2 -24

index 61f0c206bff0..326d699b30d5 100644
@@ -198,30 +198,8 @@ static inline bool hv_is_synic_reg(unsigned int reg)
 	return false;
 }
 
-static inline u64 hv_get_register(unsigned int reg)
-{
-	u64 value;
-
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
-		hv_ghcb_msr_read(reg, &value);
-	else
-		rdmsrl(reg, value);
-	return value;
-}
-
-static inline void hv_set_register(unsigned int reg, u64 value)
-{
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
-		hv_ghcb_msr_write(reg, value);
-
-		/* Write proxy bit via wrmsl instruction */
-		if (reg >= HV_REGISTER_SINT0 &&
-		    reg <= HV_REGISTER_SINT15)
-			wrmsrl(reg, value | 1 << 20);
-	} else {
-		wrmsrl(reg, value);
-	}
-}
+u64 hv_get_register(unsigned int reg);
+void hv_set_register(unsigned int reg, u64 value);
 
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}

arch/x86/kernel/cpu/mshyperv.c +9 -0

index 9a4204139490..3e6711a6af6b 100644
@@ -41,6 +41,55 @@ bool hv_root_partition;
 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:
+		return HV_REGISTER_SIEFP;
+	case HV_REGISTER_SCONTROL:
+		return HV_REGISTER_NESTED_SCONTROL;

arch/x86/kernel/cpu/mshyperv.c +4 -0

+{
+	u64 value;
+
+	if (hv_nested)

arch/x86/kernel/cpu/mshyperv.c +16 -0

+{
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
+		hv_ghcb_msr_write(reg, value);
+
+		/* Write proxy bit via wrmsl instruction */
+		if (reg >= HV_REGISTER_SINT0 &&
+		    reg <= HV_REGISTER_SINT15)
+			wrmsrl(reg, value | 1 << 20);
+	} else {
+		wrmsrl(reg, value);
+	}
+}
+
 #if IS_ENABLED(CONFIG_HYPERV)
 static void (*vmbus_handler)(void);
 static void (*hv_stimer0_handler)(void);

drivers/hv/hv.c +13 -6

index 4d6480d57546..9e1eb50cc76f 100644
@@ -147,7 +147,7 @@ int hv_synic_alloc(void)
 		 * Synic message and event pages are allocated by paravisor.
 		 * Skip these pages allocation here.
 		 */
-		if (!hv_isolation_type_snp()) {
+		if (!hv_isolation_type_snp() && !hv_root_partition) {
 			hv_cpu->synic_message_page =
 				(void *)get_zeroed_page(GFP_ATOMIC);
 			if (hv_cpu->synic_message_page == NULL) {

@@ -188,8 +188,16 @@ void hv_synic_free(void)
 		struct hv_per_cpu_context *hv_cpu
 			= per_cpu_ptr(hv_context.cpu_context, cpu);
 
-		free_page((unsigned long)hv_cpu->synic_event_page);
-		free_page((unsigned long)hv_cpu->synic_message_page);
+		if (hv_root_partition) {
+			if (hv_cpu->synic_event_page != NULL)
+				memunmap(hv_cpu->synic_event_page);
+
+			if (hv_cpu->synic_message_page != NULL)
+				memunmap(hv_cpu->synic_message_page);
+		} else {
+			free_page((unsigned long)hv_cpu->synic_event_page);
+			free_page((unsigned long)hv_cpu->synic_message_page);
+		}
 		free_page((unsigned long)hv_cpu->post_msg_page);
 	}
 

@@ -216,7 +224,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
 	simp.simp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_message_page
 			= memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
 				   HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -233,7 +241,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
 	siefp.siefp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_event_page =
 			memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
 				 HV_HYP_PAGE_SIZE, MEMREMAP_WB);
--

2.25.1

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 11 +++++++
 arch/x86/include/asm/mshyperv.h    | 26 ++--------------
 arch/x86/kernel/cpu/mshyperv.c     | 49 ++++++++++++++++++++++++++++++
 drivers/hv/hv.c                    | 18 ++++++++---
 4 files changed, 75 insertions(+), 29 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index 58c03d18c235..b5019becb618 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
 #define HV_REGISTER_SINT14			0x4000009E
 #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
 /*
  * Synthetic Timer MSRs. Four timers per vcpu.
  */

arch/x86/include/asm/mshyperv.h +2 -24

index 61f0c206bff0..326d699b30d5 100644
@@ -198,30 +198,8 @@ static inline bool hv_is_synic_reg(unsigned int reg)
 	return false;
 }
 
-static inline u64 hv_get_register(unsigned int reg)
-{
-	u64 value;
-
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
-		hv_ghcb_msr_read(reg, &value);
-	else
-		rdmsrl(reg, value);
-	return value;
-}
-
-static inline void hv_set_register(unsigned int reg, u64 value)
-{
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
-		hv_ghcb_msr_write(reg, value);
-
-		/* Write proxy bit via wrmsl instruction */
-		if (reg >= HV_REGISTER_SINT0 &&
-		    reg <= HV_REGISTER_SINT15)
-			wrmsrl(reg, value | 1 << 20);
-	} else {
-		wrmsrl(reg, value);
-	}
-}
+u64 hv_get_register(unsigned int reg);
+void hv_set_register(unsigned int reg, u64 value);
 
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}

arch/x86/kernel/cpu/mshyperv.c +38 -0

index 9a4204139490..3e6711a6af6b 100644
@@ -41,6 +41,55 @@ bool hv_root_partition;
 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:
+		return HV_REGISTER_SIEFP;
+	case HV_REGISTER_SCONTROL:
+		return HV_REGISTER_NESTED_SCONTROL;
+	case HV_REGISTER_SINT0:
+		return HV_REGISTER_NESTED_SINT0;
+	case HV_REGISTER_EOM:
+		return HV_REGISTER_NESTED_EOM;
+	default:
+		return reg;
+	}
+}
+
+inline u64 hv_get_register(unsigned int reg)
+{
+	u64 value;
+
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
+		hv_ghcb_msr_read(reg, &value);
+	else
+		rdmsrl(reg, value);
+	return value;
+}
+
+inline void hv_set_register(unsigned int reg, u64 value)
+{
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {

According to TLFS, in order to communicate to L0 hypervisor there needs to be an additional bit set in the control register. This communication is required to perform priviledged instructions which can only be

performed by L0 hypervisor. An example of that could be setting up the VMBus infrastructure.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h |  3 ++-
 arch/x86/include/asm/mshyperv.h    | 42 +++++++++++++++++++++++++++---
 include/asm-generic/hyperv-tlfs.h  |  1 +
 3 files changed, 41 insertions(+), 5 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +2 -1

index b5019becb618..7758c495541d 100644
@@ -380,7 +380,8 @@ struct hv_nested_enlightenments_control {
 		__u32 reserved:31;
 	} features;
 	struct {
-		__u32 reserved;
+		__u32 inter_partition_comm:1;
+		__u32 reserved:31;
 	} hypercallControls;
 } __packed;
 

arch/x86/include/asm/mshyperv.h +8 -2

index 326d699b30d5..42e42cea0384 100644
@@ -74,10 +74,16 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void
 *output)
 	return hv_status;
 }
 
+/* Hypercall to the L0 hypervisor */
+static inline u64 hv_do_nested_hypercall(u64 control, void *input, void *output)
+{
+	return hv_do_hypercall(control | HV_HYPERCALL_NESTED, input, output);
+}
+
 /* Fast hypercall with 8 bytes of input and no output */
-static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
+static inline u64 _hv_do_fast_hypercall8(u64 control, u16 code, u64 input1)
 {
-	u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
+	u64 hv_status;
 
 #ifdef CONFIG_X86_64

arch/x86/include/asm/mshyperv.h +16 -2

 	{

@@ -105,10 +111,24 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 		return hv_status;
 }
 
+static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT;
+
+	return _hv_do_fast_hypercall8(control, code, input1);
+}
+
+static inline u64 hv_do_fast_nested_hypercall8(u16 code, u64 input1)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT |
 HV_HYPERCALL_NESTED;
+
+	return _hv_do_fast_hypercall8(control, code, input1);
+}
+
 /* Fast hypercall with 16 bytes of input */
-static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
+static inline u64 _hv_do_fast_hypercall16(u64 control, u16 code, u64 input1, u64
 input2)
 {
-	u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
+	u64 hv_status;
 

#ifdef CONFIG_X86_64

arch/x86/include/asm/mshyperv.h +14 -0

 	{

@@ -139,6 +159,20 @@ static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1,
 u64 input2)
 	return hv_status;
 }
 
+static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT;
+
+	return _hv_do_fast_hypercall16(control, code, input1, input2);
+}
+
+static inline u64 hv_do_fast_nested_hypercall16(u16 code, u64 input1, u64 input2)
+{
+	u64 control = (u64)code | HV_HYPERCALL_FAST_BIT |
 HV_HYPERCALL_NESTED;
+
+	return _hv_do_fast_hypercall16(control, code, input1, input2);
+}
+
 extern struct hv_vp_assist_page **hv_vp_assist_page;

static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)

include/asm-generic/hyperv-tlfs.h +1 -1

index b17c6eeb9afa..e61ee461c4fc 100644
@@ -194,6 +194,7 @@ enum HV_GENERIC_SET_FORMAT {
 #define HV_HYPERCALL_VARHEAD_OFFSET	17
 #define HV_HYPERCALL_VARHEAD_MASK	GENMASK_ULL(26, 17)
 #define HV_HYPERCALL_RSVD0_MASK		GENMASK_ULL(31, 27)
+#define HV_HYPERCALL_NESTED		BIT_ULL(31)
 #define HV_HYPERCALL_REP_COMP_OFFSET	32
 #define HV_HYPERCALL_REP_COMP_1		BIT_ULL(32)
 #define HV_HYPERCALL_REP_COMP_MASK	GENMASK_ULL(43, 32)
--

2.25.1

Currently VMBus driver is not initialized for root partition but we need to enable the VMBus driver for nested root partition. This is required, so that L2 root can use the VMBus devices.

Signed-off-by Jinank Jain <[email protected]>

---
 drivers/hv/vmbus_drv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

drivers/hv/vmbus_drv.c +1 -2

index db00d20c726d..0937877eade9 100644
@@ -2744,7 +2744,7 @@ static int __init hv_acpi_init(void)
 	if (!hv_is_hyperv_initialized())
 		return -ENODEV;
 
-	if (hv_root_partition)
+	if (hv_root_partition && !hv_nested)
 		return 0;
 
 	/*
--

2.25.1

Traditionally we have been using the HYPERVISOR_CALLBACK_VECTOR to relay the VMBus interrupt. But this does not work in case of nested hypervisor. Microsoft Hypervisor reserves 0x31 to 0x34 as the interrupt vector range for VMBus and thus we have to use one of the vectors from that range and setup the IDT accordingly.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/idtentry.h    |  2 ++
 arch/x86/include/asm/irq_vectors.h |  6 ++++++
 arch/x86/kernel/cpu/mshyperv.c     | 15 +++++++++++++++
 arch/x86/kernel/idt.c              |  9 +++++++++
 drivers/hv/vmbus_drv.c             |  3 ++-
 5 files changed, 34 insertions(+), 1 deletion(-)

arch/x86/include/asm/idtentry.h +0 -0

index 72184b0b2219..c0648e3e4d4a 100644
@@ -686,6 +686,8 @@ DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR,
 	sysvec_kvm_posted_intr_nested
 DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,
 	sysvec_hyperv_callback);
 DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR,
 	sysvec_hyperv_reenlightenment);
 DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR,

	sysvec_hyperv_stimer0);

arch/x86/include/asm/idtentry.h +2 -0

+DECLARE_IDTENTRY_SYSVEC(HYPERV_INTR_NESTED_VMBUS_VECTOR,
+			sysvec_hyperv_nested_vmbus_intr);
 #endif
 
 #if IS_ENABLED(CONFIG_ACRN_GUEST)

arch/x86/include/asm/irq_vectors.h +6 -0

index 43dcb9284208..729d19eab7f5 100644
@@ -102,6 +102,12 @@
 #if IS_ENABLED(CONFIG_HYPERV)
 #define HYPERV_REENLIGHTENMENT_VECTOR	0xee
 #define HYPERV_STIMER0_VECTOR		0xed
+/*
+ * FIXME: Change this, once Microsoft Hypervisor changes its assumption
+ * around VMBus interrupt vector allocation for nested root partition.
+ * Or provides a better interface to detect this instead of hardcoding.
+ */
+#define HYPERV_INTR_NESTED_VMBUS_VECTOR	0x31
 #endif
 
 #define LOCAL_TIMER_VECTOR		0xec

arch/x86/kernel/cpu/mshyperv.c +15 -0

index 3e6711a6af6b..ec7fef43e03b 100644
@@ -110,6 +110,21 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback)
 	set_irq_regs(old_regs);
 }
 
+DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_nested_vmbus_intr)
+{
+	struct pt_regs *old_regs = set_irq_regs(regs);
+
+	inc_irq_stat(irq_hv_callback_count);
+
+	if (vmbus_handler)
+		vmbus_handler();
+
+	if (ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED)
+		ack_APIC_irq();
+
+	set_irq_regs(old_regs);
+}
+
 void hv_setup_vmbus_handler(void (*handler)(void))
 {
 	vmbus_handler = handler;

arch/x86/kernel/idt.c +9 -0

index a58c6bc1cd68..ace648856a0b 100644
@@ -160,6 +160,15 @@ static const __initconst struct idt_data apic_idts[] = {
 # endif
 	INTG(SPURIOUS_APIC_VECTOR,
 	asm_sysvec_spurious_apic_interrupt),
 	INTG(ERROR_APIC_VECTOR,			asm_sysvec_error_interrupt),
+#ifdef CONFIG_HYPERV
+	/*
+	 * This is a hack because we cannot install this interrupt handler via alloc_intr_gate
+	 * as it does not allow interrupt vector less than FIRST_SYSTEM_VECTORS. And hyperv
+	 * does not want anything other than 0x31-0x34 as the interrupt vector for vmbus
+	 * interrupt in case of nested setup.
+	 */
+	INTG(HYPERV_INTR_NESTED_VMBUS_VECTOR,
 asm_sysvec_hyperv_nested_vmbus_intr),
+#endif
 #endif

};

drivers/hv/vmbus_drv.c +2 -2

index 0937877eade9..c1477f3a08dd 100644
@@ -2767,7 +2767,8 @@ static int __init hv_acpi_init(void)
 	 * normal Linux IRQ mechanism is not used in this case.
 	 */
 #ifdef HYPERVISOR_CALLBACK_VECTOR
-	vmbus_interrupt = HYPERVISOR_CALLBACK_VECTOR;
+	vmbus_interrupt = hv_nested ? HYPERV_INTR_NESTED_VMBUS_VECTOR :
+					    HYPERVISOR_CALLBACK_VECTOR;
 	vmbus_irq = -1;
 #endif
 
--

2.25.1

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 11 +++++++
 arch/x86/include/asm/mshyperv.h    | 26 ++--------------
 arch/x86/kernel/cpu/mshyperv.c     | 49 ++++++++++++++++++++++++++++++
 drivers/hv/hv.c                    | 18 ++++++++---
 4 files changed, 75 insertions(+), 29 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index 58c03d18c235..b5019becb618 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
 #define HV_REGISTER_SINT14			0x4000009E
 #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
 /*
  * Synthetic Timer MSRs. Four timers per vcpu.
  */

arch/x86/include/asm/mshyperv.h +2 -24

index 61f0c206bff0..326d699b30d5 100644
@@ -198,30 +198,8 @@ static inline bool hv_is_synic_reg(unsigned int reg)
 	return false;
 }
 
-static inline u64 hv_get_register(unsigned int reg)
-{
-	u64 value;
-
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
-		hv_ghcb_msr_read(reg, &value);
-	else
-		rdmsrl(reg, value);
-	return value;
-}
-
-static inline void hv_set_register(unsigned int reg, u64 value)
-{
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
-		hv_ghcb_msr_write(reg, value);
-
-		/* Write proxy bit via wrmsl instruction */
-		if (reg >= HV_REGISTER_SINT0 &&
-		    reg <= HV_REGISTER_SINT15)
-			wrmsrl(reg, value | 1 << 20);
-	} else {
-		wrmsrl(reg, value);
-	}
-}
+u64 hv_get_register(unsigned int reg);
+void hv_set_register(unsigned int reg, u64 value);
 
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}

arch/x86/kernel/cpu/mshyperv.c +39 -0

index 9a4204139490..97d8ce744e47 100644
@@ -41,6 +41,55 @@ bool hv_root_partition;
 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:
+		return HV_REGISTER_SIEFP;
+	case HV_REGISTER_SCONTROL:
+		return HV_REGISTER_NESTED_SCONTROL;
+	case HV_REGISTER_SINT0:
+		return HV_REGISTER_NESTED_SINT0;
+	case HV_REGISTER_EOM:
+		return HV_REGISTER_NESTED_EOM;
+	default:
+		return reg;
+	}
+}
+
+u64 hv_get_register(unsigned int reg)
+{
+	u64 value;
+
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
+		hv_ghcb_msr_read(reg, &value);
+	else
+		rdmsrl(reg, value);
+	return value;
+}
+
+void hv_set_register(unsigned int reg, u64 value)
+{
+	if (hv_nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
+		hv_ghcb_msr_write(reg, value);

arch/x86/kernel/cpu/mshyperv.c +0 -0

 #if IS_ENABLED(CONFIG_HYPERV)
 static void (*vmbus_handler)(void);
 static void (*hv_stimer0_handler)(void);

drivers/hv/hv.c +13 -6

index 4d6480d57546..9e1eb50cc76f 100644
@@ -147,7 +147,7 @@ int hv_synic_alloc(void)
 		 * Synic message and event pages are allocated by paravisor.
 		 * Skip these pages allocation here.
 		 */
-		if (!hv_isolation_type_snp()) {
+		if (!hv_isolation_type_snp() && !hv_root_partition) {
 			hv_cpu->synic_message_page =
 				(void *)get_zeroed_page(GFP_ATOMIC);
 			if (hv_cpu->synic_message_page == NULL) {

@@ -188,8 +188,16 @@ void hv_synic_free(void)
 		struct hv_per_cpu_context *hv_cpu
 			= per_cpu_ptr(hv_context.cpu_context, cpu);
 
-		free_page((unsigned long)hv_cpu->synic_event_page);
-		free_page((unsigned long)hv_cpu->synic_message_page);
+		if (hv_root_partition) {
+			if (hv_cpu->synic_event_page != NULL)
+				memunmap(hv_cpu->synic_event_page);
+
+			if (hv_cpu->synic_message_page != NULL)
+				memunmap(hv_cpu->synic_message_page);
+		} else {
+			free_page((unsigned long)hv_cpu->synic_event_page);
+			free_page((unsigned long)hv_cpu->synic_message_page);
+		}
 		free_page((unsigned long)hv_cpu->post_msg_page);
 	}
 

@@ -216,7 +224,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
 	simp.simp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_message_page
 			= memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
 				   HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -233,7 +241,7 @@ void hv_synic_enable_regs(unsigned int cpu)
 	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
 	siefp.siefp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_event_page =
 			memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
 				 HV_HYP_PAGE_SIZE, MEMREMAP_WB);
--

2.25.1

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
 arch/x86/include/asm/hyperv-tlfs.h | 11 ++++
 arch/x86/include/asm/mshyperv.h    | 30 ++-------
 arch/x86/kernel/cpu/mshyperv.c     | 69 +++++++++++++++++++++
 drivers/hv/hv.c                    | 99 ++++++++++++++++++++++--------
 include/asm-generic/mshyperv.h     |  5 +-
 5 files changed, 165 insertions(+), 49 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index 58c03d18c235..b5019becb618 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
 #define HV_REGISTER_SINT14			0x4000009E
 #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
 /*
  * Synthetic Timer MSRs. Four timers per vcpu.
  */

arch/x86/include/asm/mshyperv.h +6 -24

index 61f0c206bff0..3197d49c888c 100644
@@ -198,30 +198,10 @@ static inline bool hv_is_synic_reg(unsigned int reg)
 	return false;
 }
 
-static inline u64 hv_get_register(unsigned int reg)
-{
-	u64 value;
-
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
-		hv_ghcb_msr_read(reg, &value);
-	else
-		rdmsrl(reg, value);
-	return value;
-}
-
-static inline void hv_set_register(unsigned int reg, u64 value)
-{
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
-		hv_ghcb_msr_write(reg, value);
-
-		/* Write proxy bit via wrmsl instruction */
-		if (reg >= HV_REGISTER_SINT0 &&
-		    reg <= HV_REGISTER_SINT15)
-			wrmsrl(reg, value | 1 << 20);
-	} else {
-		wrmsrl(reg, value);
-	}
-}
+u64 hv_get_register(unsigned int reg);
+void hv_set_register(unsigned int reg, u64 value);
+u64 hv_get_nested_register(unsigned int reg);
+void hv_set_nested_register(unsigned int reg, u64 value);
 
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}

@@ -241,6 +221,8 @@ static inline int hyperv_flush_guest_mapping_range(u64 as,
 }
 static inline void hv_set_register(unsigned int reg, u64 value) { }
 static inline u64 hv_get_register(unsigned int reg) { return 0; }
+static inline void hv_set_nested_register(unsigned int reg, u64 value) { }
+static inline u64 hv_get_nested_register(unsigned int reg) { return 0; }
 static inline int hv_set_mem_host_visibility(unsigned long addr, int numpages,
 					     bool visible)
 {

arch/x86/kernel/cpu/mshyperv.c +6 -0

index f9b78d4829e3..f2f6e10301a8 100644
@@ -41,7 +41,76 @@ bool hv_root_partition;
 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:

arch/x86/kernel/cpu/mshyperv.c +53 -0

+}
+
 #if IS_ENABLED(CONFIG_HYPERV)
+static u64 _hv_get_register(unsigned int reg, bool nested)
+{
+	u64 value;
+
+	if (nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
+		hv_ghcb_msr_read(reg, &value);
+	else
+		rdmsrl(reg, value);
+	return value;
+}
+
+static void _hv_set_register(unsigned int reg, u64 value, bool nested)
+{
+	if (nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
+		hv_ghcb_msr_write(reg, value);
+
+		/* Write proxy bit via wrmsl instruction */
+		if (reg >= HV_REGISTER_SINT0 &&
+		    reg <= HV_REGISTER_SINT15)
+			wrmsrl(reg, value | 1 << 20);
+	} else {
+		wrmsrl(reg, value);
+	}
+}
+
+u64 hv_get_register(unsigned int reg)
+{
+	return _hv_get_register(reg, false);
+}
+
+void hv_set_register(unsigned int reg, u64 value)
+{
+	_hv_set_register(reg, value, false);
+}
+
+u64 hv_get_nested_register(unsigned int reg)
+{
+	return _hv_get_register(reg, true);
+}
+
+void hv_set_nested_register(unsigned int reg, u64 value)
+{
+	_hv_set_register(reg, value, true);
+}
+
 static void (*vmbus_handler)(void);
 static void (*hv_stimer0_handler)(void);
 static void (*hv_kexec_handler)(void);

drivers/hv/hv.c +11 -3

index 4d6480d57546..0ed052f2423e 100644
@@ -147,7 +147,7 @@ int hv_synic_alloc(void)
 		 * Synic message and event pages are allocated by paravisor.
 		 * Skip these pages allocation here.
 		 */
-		if (!hv_isolation_type_snp()) {
+		if (!hv_isolation_type_snp() && !hv_root_partition) {
 			hv_cpu->synic_message_page =
 				(void *)get_zeroed_page(GFP_ATOMIC);
 			if (hv_cpu->synic_message_page == NULL) {

@@ -188,8 +188,16 @@ void hv_synic_free(void)
 		struct hv_per_cpu_context *hv_cpu
 			= per_cpu_ptr(hv_context.cpu_context, cpu);
 
-		free_page((unsigned long)hv_cpu->synic_event_page);
-		free_page((unsigned long)hv_cpu->synic_message_page);
+		if (hv_root_partition) {
+			if (hv_cpu->synic_event_page != NULL)
+				memunmap(hv_cpu->synic_event_page);
+
+			if (hv_cpu->synic_message_page != NULL)
+				memunmap(hv_cpu->synic_message_page);
+		} else {
+			free_page((unsigned long)hv_cpu->synic_event_page);
+			free_page((unsigned long)hv_cpu->synic_message_page);
+		}
 		free_page((unsigned long)hv_cpu->post_msg_page);
 	}
 

drivers/hv/hv.c +19 -7

+
 	simp.simp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_message_page
 			= memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
 				   HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -227,13 +237,18 @@ void hv_synic_enable_regs(unsigned int cpu)
 			>> HV_HYP_PAGE_SHIFT;
 	}
 
-	hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIMP, simp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
 
 	/* Setup the Synic's event page */
-	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
+	siefp.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SIEFP) :
+				  hv_get_register(HV_REGISTER_SIEFP);
 	siefp.siefp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
 		hv_cpu->synic_event_page =
 			memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
 				 HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -245,13 +260,19 @@ void hv_synic_enable_regs(unsigned int cpu)
 			>> HV_HYP_PAGE_SHIFT;
 	}
 
-	hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
 
 	/* Setup the shared SINT. */
 	if (vmbus_irq != -1)
 		enable_percpu_irq(vmbus_irq, 0);
-	shared_sint.as_uint64 = hv_get_register(HV_REGISTER_SINT0 +
-					VMBUS_MESSAGE_SINT);
+	shared_sint.as_uint64 =
+		hv_nested ?
+			hv_get_nested_register(HV_REGISTER_SINT0 +
+					       VMBUS_MESSAGE_SINT) :
+			hv_get_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT);
 
 	shared_sint.vector = vmbus_interrupt;

drivers/hv/hv.c +12 -4

 	shared_sint.masked = false;

@@ -266,14 +287,22 @@ void hv_synic_enable_regs(unsigned int cpu)
 #else
 	shared_sint.auto_eoi = 0;
 #endif
-	hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT,
+				       shared_sint.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
 				shared_sint.as_uint64);
-
 	/* Enable the global synic bit */
-	sctrl.as_uint64 = hv_get_register(HV_REGISTER_SCONTROL);
+	sctrl.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SCONTROL) :
+				  hv_get_register(HV_REGISTER_SCONTROL);
 	sctrl.enable = 1;
 
-	hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
 }
 

int hv_synic_init(unsigned int cpu)

drivers/hv/hv.c +12 -4

@@ -297,17 +326,25 @@ void hv_synic_disable_regs(unsigned int cpu)
 	union hv_synic_siefp siefp;
 	union hv_synic_scontrol sctrl;
 
-	shared_sint.as_uint64 = hv_get_register(HV_REGISTER_SINT0 +
-					VMBUS_MESSAGE_SINT);
+	shared_sint.as_uint64 =
+		hv_nested ?
+			hv_get_nested_register(HV_REGISTER_SINT0 +
+					       VMBUS_MESSAGE_SINT) :
+			hv_get_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT);
 
 	shared_sint.masked = 1;
 
 	/* Need to correctly cleanup in the case of SMP!!! */
 	/* Disable the interrupt */
-	hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT,
+				       shared_sint.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
 				shared_sint.as_uint64);
 
-	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
+	simp.as_uint64 = hv_nested ? hv_get_nested_register(HV_REGISTER_SIMP) :
+				     hv_get_register(HV_REGISTER_SIMP);
 	/*

drivers/hv/hv.c +19 -5

 	 * In Isolation VM, sim and sief pages are allocated by
 	 * paravisor. These pages also will be used by kdump

@@ -320,9 +357,14 @@ void hv_synic_disable_regs(unsigned int cpu)
 	else
 		simp.base_simp_gpa = 0;
 
-	hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIMP, simp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
 
-	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
+	siefp.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SIEFP) :
+				  hv_get_register(HV_REGISTER_SIEFP);
 	siefp.siefp_enabled = 0;
 
 	if (hv_isolation_type_snp())

@@ -330,12 +372,21 @@ void hv_synic_disable_regs(unsigned int cpu)
 	else
 		siefp.base_siefp_gpa = 0;
 
-	hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
 
 	/* Disable the global synic bit */
-	sctrl.as_uint64 = hv_get_register(HV_REGISTER_SCONTROL);
+	sctrl.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SCONTROL) :
+				  hv_get_register(HV_REGISTER_SCONTROL);
 	sctrl.enable = 0;
-	hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
 
 	if (vmbus_irq != -1)
 		disable_percpu_irq(vmbus_irq);

include/asm-generic/mshyperv.h +4 -2

index f131027830c3..db0b5be1e087 100644
@@ -147,7 +147,10 @@ static inline void vmbus_signal_eom(struct hv_message *msg,
 u32 old_msg_type)
 		 * possibly deliver another msg from the
 		 * hypervisor
 		 */
-		hv_set_register(HV_REGISTER_EOM, 0);
+		if (hv_nested)
+			hv_set_nested_register(HV_REGISTER_EOM, 0);
+		else
+			hv_set_register(HV_REGISTER_EOM, 0);
 	}
 }
 
--

2.25.1

From: Jinank Jain <[email protected]> Sent: Thursday, December 1, 2022 3:04 AM

Quoted 104 lines

Child partitions are free to allocate SynIC message and event page but in case of root partition it must use the pages allocated by Microsoft Hypervisor (MSHV). Base address for these pages can be found using synthetic MSRs exposed by MSHV. There is a slight difference in those MSRs for nested vs non-nested root partition.

Signed-off-by Jinank Jain <[email protected]>

---
  arch/x86/include/asm/hyperv-tlfs.h | 11 ++++
  arch/x86/include/asm/mshyperv.h    | 30 ++-------
  arch/x86/kernel/cpu/mshyperv.c     | 69 +++++++++++++++++++++
  drivers/hv/hv.c                    | 99 ++++++++++++++++++++++--------
  include/asm-generic/mshyperv.h     |  5 +-
  5 files changed, 165 insertions(+), 49 deletions(-)

arch/x86/include/asm/hyperv-tlfs.h +11 -0

index 58c03d18c235..b5019becb618 100644
@@ -225,6 +225,17 @@ enum hv_isolation_type {
  #define HV_REGISTER_SINT14			0x4000009E
  #define HV_REGISTER_SINT15			0x4000009F
 
+/*
+ * Define synthetic interrupt controller model specific registers for
+ * nested hypervisor.
+ */
+#define HV_REGISTER_NESTED_SCONTROL            0x40001080
+#define HV_REGISTER_NESTED_SVERSION            0x40001081
+#define HV_REGISTER_NESTED_SIEFP               0x40001082
+#define HV_REGISTER_NESTED_SIMP                0x40001083
+#define HV_REGISTER_NESTED_EOM                 0x40001084
+#define HV_REGISTER_NESTED_SINT0               0x40001090
+
  /*
   * Synthetic Timer MSRs. Four timers per vcpu.
   */

arch/x86/include/asm/mshyperv.h +6 -24

index 61f0c206bff0..3197d49c888c 100644
@@ -198,30 +198,10 @@ static inline bool hv_is_synic_reg(unsigned int reg)
  	return false;
  }
 
-static inline u64 hv_get_register(unsigned int reg)
-{
-	u64 value;
-
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
-		hv_ghcb_msr_read(reg, &value);
-	else
-		rdmsrl(reg, value);
-	return value;
-}
-
-static inline void hv_set_register(unsigned int reg, u64 value)
-{
-	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
-		hv_ghcb_msr_write(reg, value);
-
-		/* Write proxy bit via wrmsl instruction */
-		if (reg >= HV_REGISTER_SINT0 &&
-		    reg <= HV_REGISTER_SINT15)
-			wrmsrl(reg, value | 1 << 20);
-	} else {
-		wrmsrl(reg, value);
-	}
-}
+u64 hv_get_register(unsigned int reg);
+void hv_set_register(unsigned int reg, u64 value);
+u64 hv_get_nested_register(unsigned int reg);
+void hv_set_nested_register(unsigned int reg, u64 value);
 
  #else /* CONFIG_HYPERV */
  static inline void hyperv_init(void) {}

@@ -241,6 +221,8 @@ static inline int hyperv_flush_guest_mapping_range(u64 as,
  }
  static inline void hv_set_register(unsigned int reg, u64 value) { }
  static inline u64 hv_get_register(unsigned int reg) { return 0; }
+static inline void hv_set_nested_register(unsigned int reg, u64 value) { }
+static inline u64 hv_get_nested_register(unsigned int reg) { return 0; }
  static inline int hv_set_mem_host_visibility(unsigned long addr, int numpages,
  					     bool visible)
  {

arch/x86/kernel/cpu/mshyperv.c +11 -0

index f9b78d4829e3..f2f6e10301a8 100644
@@ -41,7 +41,76 @@ bool hv_root_partition;
  bool hv_nested;
  struct ms_hyperv_info ms_hyperv;
 
+static inline unsigned int hv_get_nested_reg(unsigned int reg)
+{
+	switch (reg) {
+	case HV_REGISTER_SIMP:
+		return HV_REGISTER_NESTED_SIMP;
+	case HV_REGISTER_NESTED_SIEFP:
+		return HV_REGISTER_SIEFP;
+	case HV_REGISTER_SCONTROL:
+		return HV_REGISTER_NESTED_SCONTROL;
+	case HV_REGISTER_SINT0:
+		return HV_REGISTER_NESTED_SINT0;

arch/x86/kernel/cpu/mshyperv.c bool hv_root_partition; · line 59

Quoted 5 lines

arch/x86/kernel/cpu/mshyperv.c +5 -0

+	case HV_REGISTER_EOM:
+		return HV_REGISTER_NESTED_EOM;
+	default:
+		return reg;
+	}

Just a question: You added #defines for 6 nested registers. But the switch statement above maps only 5 registers. Is it intentional that there's not a mapping for HV_REGISTER_SVERSION?

Quoted 91 lines

arch/x86/kernel/cpu/mshyperv.c +53 -0

+}
+
  #if IS_ENABLED(CONFIG_HYPERV)
+static u64 _hv_get_register(unsigned int reg, bool nested)
+{
+	u64 value;
+
+	if (nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp())
+		hv_ghcb_msr_read(reg, &value);
+	else
+		rdmsrl(reg, value);
+	return value;
+}
+
+static void _hv_set_register(unsigned int reg, u64 value, bool nested)
+{
+	if (nested)
+		reg = hv_get_nested_reg(reg);
+
+	if (hv_is_synic_reg(reg) && hv_isolation_type_snp()) {
+		hv_ghcb_msr_write(reg, value);
+
+		/* Write proxy bit via wrmsl instruction */
+		if (reg >= HV_REGISTER_SINT0 &&
+		    reg <= HV_REGISTER_SINT15)
+			wrmsrl(reg, value | 1 << 20);
+	} else {
+		wrmsrl(reg, value);
+	}
+}
+
+u64 hv_get_register(unsigned int reg)
+{
+	return _hv_get_register(reg, false);
+}
+
+void hv_set_register(unsigned int reg, u64 value)
+{
+	_hv_set_register(reg, value, false);
+}
+
+u64 hv_get_nested_register(unsigned int reg)
+{
+	return _hv_get_register(reg, true);
+}
+
+void hv_set_nested_register(unsigned int reg, u64 value)
+{
+	_hv_set_register(reg, value, true);
+}
+
  static void (*vmbus_handler)(void);
  static void (*hv_stimer0_handler)(void);
  static void (*hv_kexec_handler)(void);

drivers/hv/hv.c +11 -3

index 4d6480d57546..0ed052f2423e 100644
@@ -147,7 +147,7 @@ int hv_synic_alloc(void)
  		 * Synic message and event pages are allocated by paravisor.
  		 * Skip these pages allocation here.
  		 */
-		if (!hv_isolation_type_snp()) {
+		if (!hv_isolation_type_snp() && !hv_root_partition) {
  			hv_cpu->synic_message_page =
  				(void *)get_zeroed_page(GFP_ATOMIC);
  			if (hv_cpu->synic_message_page == NULL) {

@@ -188,8 +188,16 @@ void hv_synic_free(void)
  		struct hv_per_cpu_context *hv_cpu
  			= per_cpu_ptr(hv_context.cpu_context, cpu);
 
-		free_page((unsigned long)hv_cpu->synic_event_page);
-		free_page((unsigned long)hv_cpu->synic_message_page);
+		if (hv_root_partition) {
+			if (hv_cpu->synic_event_page != NULL)
+				memunmap(hv_cpu->synic_event_page);
+
+			if (hv_cpu->synic_message_page != NULL)
+				memunmap(hv_cpu->synic_message_page);
+		} else {
+			free_page((unsigned long)hv_cpu->synic_event_page);
+			free_page((unsigned long)hv_cpu->synic_message_page);
+		}
  		free_page((unsigned long)hv_cpu->post_msg_page);
  	}
 

drivers/hv/hv.c void hv_synic_enable_regs(unsigned int cpu) · line 225

Quoted 6 lines

drivers/hv/hv.c +2 -1

@@ -213,10 +221,12 @@ void hv_synic_enable_regs(unsigned int cpu)
  	union hv_synic_scontrol sctrl;
 
  	/* Setup the Synic's message page */
-	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
+	simp.as_uint64 = hv_nested ? hv_get_nested_register(HV_REGISTER_SIMP) :
+				     hv_get_register(HV_REGISTER_SIMP);

Unfortunately, this code and the similar places below will run into problems on ARM64. Drivers/hv/hv.c is common code on all architectures so it needs to compile and run on ARM64 as well as x86/x64. But there's no hv_get_nested_register() defined or implemented on the ARM64 side, so the code will fail to compile.

I think there's a better way to do this. Based on Nuno's comments, it
seems like there are two hv_get_register() functions needed:

1) Get the value of the register or its nested cousin, based on the value of hv_nested. That's what you are explicitly coding here.
2) Get the value of the register. Don't access the nested cousin, regardless of the value of hv_nested.

Based on how you coded things earlier, I'm assuming #1 is what you want to use in most cases, and specifically here in drivers/hv/hv.c. That's good, because #1 can hide the testing of hv_nested in the x86-specific implementation of hv_get_register(), while the ARM64 version of hv_get_register() continues to do whatever it does now with no changes.

I'm also assuming that #2 may be used in particular cases in the code that is specifically related to nesting. Give the #2 version a different name --- maybe hv_get_nonnested_register(), or something like that -- and use it only in code under arch/x86 that is related to nesting. That way, ARM64 won't be affected.

Of course, the same approach applies to hv_set_register().

hv_get_register() and hv_get_nonnested_register() will obviously share some code. But rather than calling a common function starting with underscore like you've done above, let me suggest that hv_get_register() test hv_nested and potentially do the translation, then call hv_get_nonnested_register(). That way you'll end up with just two functions instead of three as above with hv_get_register(), hv_get_nested_register(), and _hv_get_register().

I haven't coded up any of this, so take it as a suggestion. There could be some problem with it that I haven't seen, or my assumptions might be wrong. But give it a try and see if it works out. I'm hoping it can all be handled on the x86 side without having to add complexity on the ARM64 side.

Michael

Quoted 173 lines

drivers/hv/hv.c +19 -7

+
  	simp.simp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
  		hv_cpu->synic_message_page
  			= memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
  				   HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -227,13 +237,18 @@ void hv_synic_enable_regs(unsigned int cpu)
  			>> HV_HYP_PAGE_SHIFT;
  	}
 
-	hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIMP, simp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
 
  	/* Setup the Synic's event page */
-	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
+	siefp.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SIEFP) :
+				  hv_get_register(HV_REGISTER_SIEFP);
  	siefp.siefp_enabled = 1;
 
-	if (hv_isolation_type_snp()) {
+	if (hv_isolation_type_snp() || hv_root_partition) {
  		hv_cpu->synic_event_page =
  			memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
  				 HV_HYP_PAGE_SIZE, MEMREMAP_WB);

@@ -245,13 +260,19 @@ void hv_synic_enable_regs(unsigned int cpu)
  			>> HV_HYP_PAGE_SHIFT;
  	}
 
-	hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
 
  	/* Setup the shared SINT. */
  	if (vmbus_irq != -1)
  		enable_percpu_irq(vmbus_irq, 0);
-	shared_sint.as_uint64 = hv_get_register(HV_REGISTER_SINT0 +
-					VMBUS_MESSAGE_SINT);
+	shared_sint.as_uint64 =
+		hv_nested ?
+			hv_get_nested_register(HV_REGISTER_SINT0 +
+					       VMBUS_MESSAGE_SINT) :
+			hv_get_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT);
 
  	shared_sint.vector = vmbus_interrupt;

  	shared_sint.masked = false;

drivers/hv/hv.c +12 -4

@@ -266,14 +287,22 @@ void hv_synic_enable_regs(unsigned int cpu)
  #else
  	shared_sint.auto_eoi = 0;
  #endif
-	hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT,
+				       shared_sint.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
  				shared_sint.as_uint64);
-
  	/* Enable the global synic bit */
-	sctrl.as_uint64 = hv_get_register(HV_REGISTER_SCONTROL);
+	sctrl.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SCONTROL) :
+				  hv_get_register(HV_REGISTER_SCONTROL);
  	sctrl.enable = 1;
 
-	hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
  }
 

int hv_synic_init(unsigned int cpu)

drivers/hv/hv.c +12 -4

@@ -297,17 +326,25 @@ void hv_synic_disable_regs(unsigned int cpu)
  	union hv_synic_siefp siefp;
  	union hv_synic_scontrol sctrl;
 
-	shared_sint.as_uint64 = hv_get_register(HV_REGISTER_SINT0 +
-					VMBUS_MESSAGE_SINT);
+	shared_sint.as_uint64 =
+		hv_nested ?
+			hv_get_nested_register(HV_REGISTER_SINT0 +
+					       VMBUS_MESSAGE_SINT) :
+			hv_get_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT);
 
  	shared_sint.masked = 1;
 
  	/* Need to correctly cleanup in the case of SMP!!! */
  	/* Disable the interrupt */
-	hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SINT0 +
 VMBUS_MESSAGE_SINT,
+				       shared_sint.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SINT0 + VMBUS_MESSAGE_SINT,
  				shared_sint.as_uint64);
 
-	simp.as_uint64 = hv_get_register(HV_REGISTER_SIMP);
+	simp.as_uint64 = hv_nested ? hv_get_nested_register(HV_REGISTER_SIMP) :
+				     hv_get_register(HV_REGISTER_SIMP);
  	/*

* In Isolation VM, sim and sief pages are allocated by
* paravisor. These pages also will be used by kdump

drivers/hv/hv.c +19 -5

@@ -320,9 +357,14 @@ void hv_synic_disable_regs(unsigned int cpu)
  	else
  		simp.base_simp_gpa = 0;
 
-	hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIMP, simp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIMP, simp.as_uint64);
 
-	siefp.as_uint64 = hv_get_register(HV_REGISTER_SIEFP);
+	siefp.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SIEFP) :
+				  hv_get_register(HV_REGISTER_SIEFP);
  	siefp.siefp_enabled = 0;
 
  	if (hv_isolation_type_snp())

@@ -330,12 +372,21 @@ void hv_synic_disable_regs(unsigned int cpu)
  	else
  		siefp.base_siefp_gpa = 0;
 
-	hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SIEFP, siefp.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SIEFP, siefp.as_uint64);
 
  	/* Disable the global synic bit */
-	sctrl.as_uint64 = hv_get_register(HV_REGISTER_SCONTROL);
+	sctrl.as_uint64 = hv_nested ?
+				  hv_get_nested_register(HV_REGISTER_SCONTROL) :
+				  hv_get_register(HV_REGISTER_SCONTROL);
  	sctrl.enable = 0;
-	hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+
+	if (hv_nested)
+		hv_set_nested_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
+	else
+		hv_set_register(HV_REGISTER_SCONTROL, sctrl.as_uint64);
 
  	if (vmbus_irq != -1)
  		disable_percpu_irq(vmbus_irq);

include/asm-generic/mshyperv.h +4 -1

index f131027830c3..db0b5be1e087 100644
@@ -147,7 +147,10 @@ static inline void vmbus_signal_eom(struct hv_message *msg,
 u32 old_msg_type)
  		 * possibly deliver another msg from the
  		 * hypervisor
  		 */
-		hv_set_register(HV_REGISTER_EOM, 0);
+		if (hv_nested)
+			hv_set_nested_register(HV_REGISTER_EOM, 0);
+		else
+			hv_set_register(HV_REGISTER_EOM, 0);
  	}
  }