Skip to contentSkip to conversation

KVM: nVMX: Fully support of nested VMX preemption timer

PATCH Superseded Arthur Chunqi Li 3 participants 8 messages Sep 4, 2013 – Sep 13, 2013

Subsystems (3)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 KVM: nVMX: Fully support of nested VMX preemption timer

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully supported.

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51 ++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +1 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29)
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c +47 -4

97 lines not on this page · raw message

Signature · 1 line
1.7.9.5
#1 lore raw

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 48 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully supported.

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +1 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29)
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index 1f1da43..870caa8 renamed from arch/x86/kvm/vmx.c +0 -0

100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64

arch/x86/kvm/vmx.c +3 -1
 		VM_EXIT_HOST_ADDR_SPACE_SIZE |
 #endif
-		VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+		VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
+		VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+	if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +3 -0
+		nested_vmx_exit_ctls_high &=
+			(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
+	if (!(nested_vmx_exit_ctls_high &

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c static __init void nested_vmx_setup_ctls_msrs(void) · line 2214

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+		nested_vmx_pinbased_ctls_high &=
+			(~PIN_BASED_VMX_PREEMPTION_TIMER);

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER)

	nested_vmx_exit_ctls_high |= VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's capability when expose those vmx features(not just preemption timer) to L1.

Quoted 39 lines

nested_vmx_exit_ctls_high |=
(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |

arch/x86/kvm/vmx.c +16 -0
 				      VM_EXIT_LOAD_IA32_EFER);
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct kvm_vcpu
 *vcpu, u64 *info1, u64 *info2)
 	*info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+	u64 delta_tsc_l1;
+	u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+	preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+			MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;
+	preempt_val_l2 = vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+	delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+			native_read_tsc()) - vcpu->arch.last_guest_tsc;
+	preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+	if (preempt_val_l2 - preempt_val_l1 < 0)
+		preempt_val_l2 = 0;
+	else
+		preempt_val_l2 -= preempt_val_l1;
+	vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, preempt_val_l2); }
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u32 exit_reason = vmx->exit_reason;
 	u32 vectoring_info = vmx->idt_vectoring_info;
+	int ret;
 
 	/* If guest state is invalid, start emulating */
 	if (vmx->emulation_required)
@@ -6795,12 +6820,15 @@ static int vmx_handle_exit(struct kvm_vcpu
 *vcpu)
 
 	if (exit_reason < kvm_vmx_max_exit_handlers
 	    && kvm_vmx_exit_handlers[exit_reason])

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu · line 6831

Quoted 10 lines
arch/x86/kvm/vmx.c +4 -2
-		return kvm_vmx_exit_handlers[exit_reason](vcpu);
+		ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
 	else {
 		vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
 		vcpu->run->hw.hardware_exit_reason = exit_reason;
+		ret = 0;
 	}
-	return 0;
+	if (is_guest_mode(vcpu))
+		nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

Quoted 21 lines
arch/x86/kvm/vmx.c +1 -0
+	return ret;
 }
 

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) {

arch/x86/kvm/vmx.c +1 -1
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u32 exec_control;
+	u32 exit_control;
 
 	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
 	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); @@
-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu,

struct vmcs12 *vmcs12)

arch/x86/kvm/vmx.c +2 -1
 	 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
 	 * bits are further modified by vmx_set_efer() below.
 	 */
-	vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+	exit_control = vmcs_config.vmexit_ctrl;
+	if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)

arch/x86/kvm/vmx.c static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 7726

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+		exit_control |= VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+	vmcs_write32(VM_EXIT_CONTROLS, exit_control);

And here should be problem if host doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.

Quoted 8 lines

/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
are

arch/x86/kvm/vmx.c +0 -0
 	 * emulated by vmx_set_efer(), below.
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu,
 struct vmcs12 *vmcs12)
 	vmcs12->guest_pending_dbg_exceptions =
 		vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, · line 8134

Quoted 10 lines
arch/x86/kvm/vmx.c +9 -0
 
+	if (vmcs12->pin_based_vm_exec_control &
+			PIN_BASED_VMX_PREEMPTION_TIMER) {
+		if (vmcs12->vm_exit_controls &
+				VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
+			vmcs12->vmx_preemption_timer_value =
+				vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+		else
+			vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,
+					vmcs12->vmx_preemption_timer_value);

Why write it to vmcs directly if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER not set?

Quoted 8 lines
arch/x86/kvm/vmx.c +2 -0
+	}
+
 	/*
 	 * In some cases (usually, nested EPT), L2 is allowed to change its

* own CR3 without exiting. If it has changed it, we must keep it.
--
1.7.9.5

Quoted 4 lines

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

#2 lore raw
Quoted 53 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 48 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully supported.

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +1 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL << 29)
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index 1f1da43..870caa8 renamed from arch/x86/kvm/vmx.c +0 -0

100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64
VM_EXIT_HOST_ADDR_SPACE_SIZE |
#endif

arch/x86/kvm/vmx.c +3 -1
-             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
+             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +3 -0
+             nested_vmx_exit_ctls_high &=
+                     (~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
+     if (!(nested_vmx_exit_ctls_high &

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c static __init void nested_vmx_setup_ctls_msrs(void) · line 2214

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_pinbased_ctls_high &=
+                     (~PIN_BASED_VMX_PREEMPTION_TIMER);

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER)

Quoted 1 line
        nested_vmx_exit_ctls_high |= VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this does occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

You remind that this piece of codes can write like this:
if (!(nested_vmx_pin_based_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER) ||
!(nested_vmx_exit_ctls_high &

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) {
        nested_vmx_exit_ctls_high &=(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
        nested_vmx_pinbased_ctls_high &= (~PIN_BASED_VMX_PREEMPTION_TIMER);
}

This may reflect the logic I describe above that these two flags should support simultaneously, and brings less confusion.

Quoted 2 lines

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's capability when expose those vmx features(not just preemption timer) to L1.

The codes just above here, when setting pinbased control for nested vmx, it firstly "rdmsr MSR_IA32_VMX_PINBASED_CTLS", then use this to mask the features hardware not support. So does other control fields.

Quoted 38 lines
Quoted 38 lines
      nested_vmx_exit_ctls_high |=
(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
                                    VM_EXIT_LOAD_IA32_EFER);
arch/x86/kvm/vmx.c +16 -0
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct kvm_vcpu
 *vcpu, u64 *info1, u64 *info2)
      *info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+     u64 delta_tsc_l1;
+     u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+     preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+                     MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;
+     preempt_val_l2 = vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+     delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+                     native_read_tsc()) - vcpu->arch.last_guest_tsc;
+     preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+     if (preempt_val_l2 - preempt_val_l1 < 0)
+             preempt_val_l2 = 0;
+     else
+             preempt_val_l2 -= preempt_val_l1;
+     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, preempt_val_l2); }
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exit_reason = vmx->exit_reason;
      u32 vectoring_info = vmx->idt_vectoring_info;
+     int ret;
 
      /* If guest state is invalid, start emulating */
      if (vmx->emulation_required)
@@ -6795,12 +6820,15 @@ static int vmx_handle_exit(struct kvm_vcpu
 *vcpu)
 
Quoted 13 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu *vcpu) · line 6830

Quoted 12 lines
arch/x86/kvm/vmx.c +4 -2
      if (exit_reason < kvm_vmx_max_exit_handlers
          && kvm_vmx_exit_handlers[exit_reason])
-             return kvm_vmx_exit_handlers[exit_reason](vcpu);
+             ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
      else {
              vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
              vcpu->run->hw.hardware_exit_reason = exit_reason;
+             ret = 0;
      }
-     return 0;
+     if (is_guest_mode(vcpu))
+             nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

The previous codes simply "return kvm_vmx_exit_handlers[exit_reason](vcpu);", which may also consumes CPU times. So put "nested_adjust_preemption_timer" ahead may cause the statistics inaccuracy.

Quoted 23 lines
Quoted 21 lines
arch/x86/kvm/vmx.c +1 -0
+     return ret;
 }
 

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu,

struct vmcs12 *vmcs12)  {
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exec_control;

+ u32 exit_control;

      vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
      vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); @@

-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
* we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
* bits are further modified by vmx_set_efer() below.
*/

arch/x86/kvm/vmx.c +2 -1
-     vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+     exit_control = vmcs_config.vmexit_ctrl;
+     if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)

arch/x86/kvm/vmx.c static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 7726

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             exit_control |= VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     vmcs_write32(VM_EXIT_CONTROLS, exit_control);
Quoted 1 line

And here should be problem if host doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.

Nested vmx does check the hardware support of these features in "nested_vmx_setup_ctls_msrs", see my comments above.

Quoted 10 lines
Quoted 10 lines

      /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE

are
* emulated by vmx_set_efer(), below.

arch/x86/kvm/vmx.c +0 -0
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu,
 struct vmcs12 *vmcs12)
      vmcs12->guest_pending_dbg_exceptions =
              vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
 
Quoted 10 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8133

Quoted 9 lines
arch/x86/kvm/vmx.c +9 -0
+     if (vmcs12->pin_based_vm_exec_control &
+                     PIN_BASED_VMX_PREEMPTION_TIMER) {
+             if (vmcs12->vm_exit_controls &
+                             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
+                     vmcs12->vmx_preemption_timer_value =
+                             vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+             else
+                     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,
+                                     vmcs12->vmx_preemption_timer_value);

Why write it to vmcs directly if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER not set?

Yes, writing is needless here since vmcs02 will be re-prepared via prepare_vmcs02() when L1->L2. This function just save information needed, vmcs_write is useless.

Arthur

Quoted 18 lines
Quoted 9 lines
arch/x86/kvm/vmx.c +2 -0
+     }
+
      /*
       * In some cases (usually, nested EPT), L2 is allowed to change its

* own CR3 without exiting. If it has changed it, we must keep it.
--
1.7.9.5

Quoted 4 lines

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

#3 lore raw

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 57 lines
Quoted 26 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 25 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully supported.

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +0 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
Quoted 2 lines

arch/x86/include/uapi/asm/msr-index.h line 538

Quoted 2 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL <<

29)

Quoted 17 lines
Quoted 17 lines
arch/x86/include/uapi/asm/msr-index.h +1 -0
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index renamed from arch/x86/kvm/vmx.c +0 -0

1f1da43..870caa8
100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64
VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif

arch/x86/kvm/vmx.c +3 -1
-             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
+             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

Quoted 8 lines

arch/x86/kvm/vmx.c static __init void nested_vmx_setup_ctls_msrs(void) · line 2214

Quoted 6 lines
arch/x86/kvm/vmx.c +3 -0
+             nested_vmx_exit_ctls_high &=
+                     (~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
+     if (!(nested_vmx_exit_ctls_high &

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_pinbased_ctls_high &=
+                     (~PIN_BASED_VMX_PREEMPTION_TIMER);

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER)

Quoted 9 lines
Quoted 1 line
        nested_vmx_exit_ctls_high |=

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER
Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this does occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

No. Only VM_EXIT_SAVE_VMX_PREEMPTION_TIMER depends on PIN_BASED_VMX_PREEMPTION_TIMER. PIN_BASED_VMX_PREEMPTION_TIMER is an independent feature

Quoted 11 lines

You remind that this piece of codes can write like this:
if (!(nested_vmx_pin_based_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER) ||
!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) {
nested_vmx_exit_ctls_high
&=(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
nested_vmx_pinbased_ctls_high &=
(~PIN_BASED_VMX_PREEMPTION_TIMER);
}

Quoted 9 lines

This may reflect the logic I describe above that these two flags should support simultaneously, and brings less confusion.

Quoted 2 lines

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's

capability when expose those vmx features(not just preemption timer) to L1. The codes just above here, when setting pinbased control for nested vmx, it firstly "rdmsr MSR_IA32_VMX_PINBASED_CTLS", then use this to mask the features hardware not support. So does other control fields.

Quoted 1 line

Yes, I saw it.

Quoted 53 lines
Quoted 8 lines
Quoted 8 lines
      nested_vmx_exit_ctls_high |=
(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
                                    VM_EXIT_LOAD_IA32_EFER);
arch/x86/kvm/vmx.c +0 -0
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct kvm_vcpu
 *vcpu, u64 *info1, u64 *info2)
      *info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+     u64 delta_tsc_l1;
+     u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+     preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+

MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;

Quoted 9 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 9 lines
arch/x86/kvm/vmx.c +9 -0
+     preempt_val_l2 = vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+     delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+                     native_read_tsc()) - vcpu->arch.last_guest_tsc;
+     preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+     if (preempt_val_l2 - preempt_val_l1 < 0)
+             preempt_val_l2 = 0;
+     else
+             preempt_val_l2 -= preempt_val_l1;
+     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

preempt_val_l2); }

Quoted 4 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu

Quoted 4 lines
arch/x86/kvm/vmx.c +0 -0
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu

*vcpu)

Quoted 10 lines
Quoted 10 lines
arch/x86/kvm/vmx.c +1 -0
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exit_reason = vmx->exit_reason;
      u32 vectoring_info = vmx->idt_vectoring_info;
+     int ret;
 
      /* If guest state is invalid, start emulating */
      if (vmx->emulation_required)
@@ -6795,12 +6820,15 @@ static int vmx_handle_exit(struct kvm_vcpu
 *vcpu)
 
Quoted 13 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu *vcpu) · line 6830

Quoted 12 lines
arch/x86/kvm/vmx.c +4 -2
      if (exit_reason < kvm_vmx_max_exit_handlers
          && kvm_vmx_exit_handlers[exit_reason])
-             return kvm_vmx_exit_handlers[exit_reason](vcpu);
+             ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
      else {
              vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
              vcpu->run->hw.hardware_exit_reason = exit_reason;
+             ret = 0;
      }
-     return 0;
+     if (is_guest_mode(vcpu))
+             nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

Quoted 4 lines

The previous codes simply "return kvm_vmx_exit_handlers[exit_reason](vcpu);", which may also consumes CPU times. So put "nested_adjust_preemption_timer" ahead may cause the statistics inaccuracy.

Then you should put it before vmentry. Here still far from the point of vmentry.

Quoted 25 lines
Quoted 11 lines
Quoted 11 lines
arch/x86/kvm/vmx.c +1 -0
+     return ret;
 }
 

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int
irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu,

struct vmcs12 *vmcs12)  {
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exec_control;

+ u32 exit_control;

Quoted 2 lines
Quoted 2 lines
      vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
      vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);

@@

Quoted 3 lines
Quoted 3 lines

-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
* we should use its exit controls. Note that

VM_EXIT_LOAD_IA32_EFER

Quoted 7 lines
Quoted 7 lines

* bits are further modified by vmx_set_efer() below. */

arch/x86/kvm/vmx.c +2 -1
-     vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+     exit_control = vmcs_config.vmexit_ctrl;
+     if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)
+ exit_control |=

Quoted 6 lines

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;

Quoted 2 lines
Quoted 1 line

+ vmcs_write32(VM_EXIT_CONTROLS, exit_control);

And here should be problem if host doesn't support

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.
Nested vmx does check the hardware support of these features in "nested_vmx_setup_ctls_msrs", see my comments above.

But here you don't check it. You just set it unconditionally.

Quoted 20 lines
Quoted 3 lines
Quoted 3 lines

      /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and

VM_ENTRY_IA32E_MODE are

Quoted 10 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8129

Quoted 10 lines

* emulated by vmx_set_efer(), below.

arch/x86/kvm/vmx.c +4 -0
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu
 *vcpu, struct vmcs12 *vmcs12)
      vmcs12->guest_pending_dbg_exceptions =
              vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
 
+     if (vmcs12->pin_based_vm_exec_control &
+                     PIN_BASED_VMX_PREEMPTION_TIMER) {
+             if (vmcs12->vm_exit_controls &
+

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+                     vmcs12->vmx_preemption_timer_value =
+

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             else
+
Quoted 9 lines

vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

Quoted 3 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+
+ vmcs12->vmx_preemption_timer_value);

Why write it to vmcs directly if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

not set?
Yes, writing is needless here since vmcs02 will be re-prepared via prepare_vmcs02() when L1->L2. This function just save information needed, vmcs_write is useless.

I don't sure I am seeing your point. Per my point, even the VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not setting, you still need to save the value to vmcs12->vmx_preemption_timer_value. Or else, prepare_vmcs02 cannot get the right value.

Quoted 15 lines

Arthur

Quoted 14 lines
Quoted 13 lines
arch/x86/kvm/vmx.c +2 -1
+     }
+
      /*
       * In some cases (usually, nested EPT), L2 is allowed to change its
       * own CR3 without exiting. If it has changed it, we must keep it.
--

1.7.9.5

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Quoted 8 lines
Quoted 4 lines

Best regards,
Yang

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

#4 lore raw
Quoted 144 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 57 lines
Quoted 26 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 25 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully supported.

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +0 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
Quoted 2 lines

arch/x86/include/uapi/asm/msr-index.h line 538

Quoted 2 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL <<

29)

Quoted 17 lines
Quoted 17 lines
arch/x86/include/uapi/asm/msr-index.h +1 -0
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index renamed from arch/x86/kvm/vmx.c +0 -0

1f1da43..870caa8
100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64
VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif

arch/x86/kvm/vmx.c +3 -1
-             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
+             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

Quoted 8 lines

arch/x86/kvm/vmx.c static __init void nested_vmx_setup_ctls_msrs(void) · line 2214

Quoted 6 lines
arch/x86/kvm/vmx.c +3 -0
+             nested_vmx_exit_ctls_high &=
+                     (~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
+     if (!(nested_vmx_exit_ctls_high &

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_pinbased_ctls_high &=
+                     (~PIN_BASED_VMX_PREEMPTION_TIMER);

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER)

Quoted 9 lines
Quoted 1 line
        nested_vmx_exit_ctls_high |=

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER
Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this does occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

No. Only VM_EXIT_SAVE_VMX_PREEMPTION_TIMER depends on PIN_BASED_VMX_PREEMPTION_TIMER. PIN_BASED_VMX_PREEMPTION_TIMER is an independent feature

Quoted 11 lines

You remind that this piece of codes can write like this:
if (!(nested_vmx_pin_based_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER) ||
!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) {
nested_vmx_exit_ctls_high
&=(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
nested_vmx_pinbased_ctls_high &=
(~PIN_BASED_VMX_PREEMPTION_TIMER);
}

Quoted 9 lines

This may reflect the logic I describe above that these two flags should support simultaneously, and brings less confusion.

Quoted 2 lines

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's

capability when expose those vmx features(not just preemption timer) to L1. The codes just above here, when setting pinbased control for nested vmx, it firstly "rdmsr MSR_IA32_VMX_PINBASED_CTLS", then use this to mask the features hardware not support. So does other control fields.

Quoted 1 line

Yes, I saw it.

Quoted 53 lines
Quoted 8 lines
Quoted 8 lines
      nested_vmx_exit_ctls_high |=
(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
                                    VM_EXIT_LOAD_IA32_EFER);
arch/x86/kvm/vmx.c +0 -0
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct kvm_vcpu
 *vcpu, u64 *info1, u64 *info2)
      *info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+     u64 delta_tsc_l1;
+     u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+     preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+

MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;

Quoted 9 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 9 lines
arch/x86/kvm/vmx.c +9 -0
+     preempt_val_l2 = vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);
+     delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+                     native_read_tsc()) - vcpu->arch.last_guest_tsc;
+     preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+     if (preempt_val_l2 - preempt_val_l1 < 0)
+             preempt_val_l2 = 0;
+     else
+             preempt_val_l2 -= preempt_val_l1;
+     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

preempt_val_l2); }

Quoted 4 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu

Quoted 4 lines
arch/x86/kvm/vmx.c +0 -0
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu

*vcpu)

Quoted 23 lines
Quoted 12 lines
arch/x86/kvm/vmx.c +1 -0
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exit_reason = vmx->exit_reason;
      u32 vectoring_info = vmx->idt_vectoring_info;
+     int ret;
 
      /* If guest state is invalid, start emulating */
      if (vmx->emulation_required)
@@ -6795,12 +6820,15 @@ static int vmx_handle_exit(struct kvm_vcpu
 *vcpu)
 
      if (exit_reason < kvm_vmx_max_exit_handlers
          && kvm_vmx_exit_handlers[exit_reason])

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu · line 6831

Quoted 10 lines
arch/x86/kvm/vmx.c +4 -2
-             return kvm_vmx_exit_handlers[exit_reason](vcpu);
+             ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
      else {
              vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
              vcpu->run->hw.hardware_exit_reason = exit_reason;
+             ret = 0;
      }
-     return 0;
+     if (is_guest_mode(vcpu))
+             nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

Quoted 5 lines
Quoted 4 lines

The previous codes simply "return kvm_vmx_exit_handlers[exit_reason](vcpu);", which may also consumes CPU times. So put "nested_adjust_preemption_timer" ahead may cause the statistics inaccuracy.

Then you should put it before vmentry. Here still far from the point of vmentry.

So where is the actual point of vmentry? I'm not quite familiar with that piece of codes.

Quoted 26 lines
Quoted 26 lines
Quoted 12 lines
Quoted 12 lines
arch/x86/kvm/vmx.c +1 -0
+     return ret;
 }
 

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int
irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu,

struct vmcs12 *vmcs12)  {
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exec_control;

+ u32 exit_control;

Quoted 2 lines
Quoted 2 lines
      vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
      vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);

@@

Quoted 3 lines
Quoted 3 lines

-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
* we should use its exit controls. Note that

VM_EXIT_LOAD_IA32_EFER

Quoted 7 lines
Quoted 7 lines

* bits are further modified by vmx_set_efer() below. */

arch/x86/kvm/vmx.c +2 -1
-     vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+     exit_control = vmcs_config.vmexit_ctrl;
+     if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)
+ exit_control |=

Quoted 7 lines
Quoted 6 lines

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;

Quoted 2 lines
Quoted 1 line

+ vmcs_write32(VM_EXIT_CONTROLS, exit_control);

And here should be problem if host doesn't support

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.
Nested vmx does check the hardware support of these features in "nested_vmx_setup_ctls_msrs", see my comments above.

But here you don't check it. You just set it unconditionally.

What we have checked in nested_vmx_setup_ctls_msrs() confirms that only if these two features supported by hardware, nested vmx can enable preemption timer. That is to say, if L2 can set PIN_BASED_VMX_PREEMPTION_TIMER without failure, hardware must support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, which should guarantee in nested_vmx_setup_ctls_msrs (). (This is also why I write code like that in nested_vmx_setup_ctls_msrs ()).

Quoted 24 lines
Quoted 24 lines
Quoted 4 lines
Quoted 4 lines


      /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and

VM_ENTRY_IA32E_MODE are

Quoted 10 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8129

Quoted 10 lines

* emulated by vmx_set_efer(), below.

arch/x86/kvm/vmx.c +4 -0
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu
 *vcpu, struct vmcs12 *vmcs12)
      vmcs12->guest_pending_dbg_exceptions =
              vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
 
+     if (vmcs12->pin_based_vm_exec_control &
+                     PIN_BASED_VMX_PREEMPTION_TIMER) {
+             if (vmcs12->vm_exit_controls &
+

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+                     vmcs12->vmx_preemption_timer_value =
+

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             else
+

vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

Quoted 2 lines
Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+
+ vmcs12->vmx_preemption_timer_value);
Quoted 7 lines
Quoted 6 lines
Quoted 1 line

Why write it to vmcs directly if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

not set?
Yes, writing is needless here since vmcs02 will be re-prepared via prepare_vmcs02() when L1->L2. This function just save information needed, vmcs_write is useless.

I don't sure I am seeing your point. Per my point, even the VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not setting, you still need to save the value to vmcs12->vmx_preemption_timer_value. Or else, prepare_vmcs02 cannot get the right value.

If L2 doesn't enable VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, its value will be reset to what we set initially. This function is called due to L2->L1, so we should emulate L2's real exit process here. What we have done in other parts is to handle vmexit of L2->L0->L2, which is not the "real" L2 vmexit.

Arthur

Quoted 29 lines
Quoted 16 lines

Arthur

Quoted 14 lines
Quoted 13 lines
arch/x86/kvm/vmx.c +2 -1
+     }
+
      /*
       * In some cases (usually, nested EPT), L2 is allowed to change its
       * own CR3 without exiting. If it has changed it, we must keep it.
--

1.7.9.5

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Quoted 8 lines
Quoted 4 lines

Best regards,
Yang

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

#5 lore raw

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 157 lines
Quoted 13 lines

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 12 lines
Quoted 9 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 8 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully

supported.

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +0 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

arch/x86/include/uapi/asm/msr-index.h line 538

Quoted 2 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL

<<

Quoted 4 lines
Quoted 4 lines

29)

Quoted 3 lines
Quoted 3 lines
arch/x86/include/uapi/asm/msr-index.h +1 -0
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

arch/x86/kvm/vmx.c

Quoted 11 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index renamed from arch/x86/kvm/vmx.c +0 -0

1f1da43..870caa8
100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64
VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif

arch/x86/kvm/vmx.c +1 -1
-             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT

|

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

arch/x86/kvm/vmx.c

Quoted 5 lines
arch/x86/kvm/vmx.c +2 -0
+             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_exit_ctls_high &=
+

(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

+ if (!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_pinbased_ctls_high &=
+                     (~PIN_BASED_VMX_PREEMPTION_TIMER);
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER)

Quoted 1 line
        nested_vmx_exit_ctls_high |=

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER
Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this

does

Quoted 6 lines
Quoted 4 lines

occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

No. Only VM_EXIT_SAVE_VMX_PREEMPTION_TIMER depends on PIN_BASED_VMX_PREEMPTION_TIMER.

PIN_BASED_VMX_PREEMPTION_TIMER is an

Quoted 16 lines

independent feature

Quoted 14 lines

You remind that this piece of codes can write like this:
if (!(nested_vmx_pin_based_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER) ||
!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) {
nested_vmx_exit_ctls_high
&=(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
nested_vmx_pinbased_ctls_high &=
(~PIN_BASED_VMX_PREEMPTION_TIMER);
}

This may reflect the logic I describe above that these two flags should support simultaneously, and brings less confusion.

Quoted 1 line
Quoted 12 lines
Quoted 7 lines
Quoted 2 lines

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's

capability when expose those vmx features(not just preemption timer) to L1. The codes just above here, when setting pinbased control for nested vmx, it firstly "rdmsr MSR_IA32_VMX_PINBASED_CTLS", then use this to mask the features hardware not support. So does other control fields.

Quoted 1 line

Yes, I saw it.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
      nested_vmx_exit_ctls_high |=

(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |

VM_EXIT_LOAD_IA32_EFER);

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
arch/x86/kvm/vmx.c +0 -0
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct
 kvm_vcpu *vcpu, u64 *info1, u64 *info2)
      *info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+     u64 delta_tsc_l1;
+     u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+     preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+

MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;

Quoted 1 line
Quoted 1 line
arch/x86/kvm/vmx.c +1 -0
+     preempt_val_l2 =

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct · line 6725

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+     delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+                     native_read_tsc()) -

vcpu->arch.last_guest_tsc;

Quoted 35 lines
Quoted 35 lines
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+     preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+     if (preempt_val_l2 - preempt_val_l1 < 0)
+             preempt_val_l2 = 0;
+     else
+             preempt_val_l2 -= preempt_val_l1;
+     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

preempt_val_l2); }

Quoted 4 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu

Quoted 4 lines
arch/x86/kvm/vmx.c +0 -0
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu

*vcpu)

Quoted 23 lines
Quoted 12 lines
arch/x86/kvm/vmx.c +1 -0
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exit_reason = vmx->exit_reason;
      u32 vectoring_info = vmx->idt_vectoring_info;
+     int ret;
 
      /* If guest state is invalid, start emulating */
      if (vmx->emulation_required) @@ -6795,12 +6820,15 @@ static

int vmx_handle_exit(struct kvm_vcpu
*vcpu)

if (exit_reason < kvm_vmx_max_exit_handlers
&& kvm_vmx_exit_handlers[exit_reason])

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu *vcpu) · line 6830

Quoted 10 lines
arch/x86/kvm/vmx.c +4 -2
-             return kvm_vmx_exit_handlers[exit_reason](vcpu);
+             ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
      else {
              vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
              vcpu->run->hw.hardware_exit_reason = exit_reason;
+             ret = 0;
      }
-     return 0;
+     if (is_guest_mode(vcpu))
+             nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

Quoted 8 lines
Quoted 5 lines
Quoted 4 lines

The previous codes simply "return kvm_vmx_exit_handlers[exit_reason](vcpu);", which may also consumes CPU times. So put "nested_adjust_preemption_timer" ahead may cause the statistics inaccuracy.

Then you should put it before vmentry. Here still far from the point of

vmentry.
So where is the actual point of vmentry? I'm not quite familiar with that piece of codes.

vmx_vcpu_run should be the right place.

Quoted 35 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

+ return ret;
}

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr,
int
irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu

*vcpu, struct vmcs12 *vmcs12)  {
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exec_control;

+ u32 exit_control;

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

vmcs_write16(GUEST_ES_SELECTOR,

vmcs12->guest_es_selector);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

vmcs_write16(GUEST_CS_SELECTOR,

vmcs12->guest_cs_selector);

Quoted 18 lines
Quoted 18 lines

@@

Quoted 3 lines
Quoted 3 lines

-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
* we should use its exit controls. Note that

VM_EXIT_LOAD_IA32_EFER

Quoted 7 lines
Quoted 7 lines

* bits are further modified by vmx_set_efer() below. */

arch/x86/kvm/vmx.c +2 -1
-     vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+     exit_control = vmcs_config.vmexit_ctrl;
+     if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)
+ exit_control |=

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;

Quoted 2 lines
Quoted 1 line

+ vmcs_write32(VM_EXIT_CONTROLS, exit_control);

And here should be problem if host doesn't support

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.
Nested vmx does check the hardware support of these features in "nested_vmx_setup_ctls_msrs", see my comments above.

Quoted 8 lines
Quoted 1 line

But here you don't check it. You just set it unconditionally.

What we have checked in nested_vmx_setup_ctls_msrs() confirms that only if these two features supported by hardware, nested vmx can enable preemption timer. That is to say, if L2 can set PIN_BASED_VMX_PREEMPTION_TIMER without failure, hardware must support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, which should guarantee in nested_vmx_setup_ctls_msrs (). (This is also why I write code like that in nested_vmx_setup_ctls_msrs ()).

I see your point. But the fact is PIN_BASED_VMX_PREEMPTION_TIMER is an independent feature. So your previous assumption is wrong. The hardware may support PIN_BASED_VMX_PREEMPTION_TIMER without VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

Quoted 32 lines
Quoted 18 lines
Quoted 18 lines
Quoted 4 lines
Quoted 4 lines


      /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and

VM_ENTRY_IA32E_MODE are

Quoted 10 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8129

Quoted 10 lines

* emulated by vmx_set_efer(), below.

arch/x86/kvm/vmx.c +4 -0
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu
 *vcpu, struct vmcs12 *vmcs12)
      vmcs12->guest_pending_dbg_exceptions =
              vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
 
+     if (vmcs12->pin_based_vm_exec_control &
+                     PIN_BASED_VMX_PREEMPTION_TIMER) {
+             if (vmcs12->vm_exit_controls &
+

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)

Quoted 2 lines
Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+                     vmcs12->vmx_preemption_timer_value =
+
Quoted 7 lines
Quoted 7 lines

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             else
+

vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

Quoted 3 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+
+ vmcs12->vmx_preemption_timer_value);

Why write it to vmcs directly if

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

Quoted 6 lines
Quoted 5 lines

not set?
Yes, writing is needless here since vmcs02 will be re-prepared via prepare_vmcs02() when L1->L2. This function just save information needed, vmcs_write is useless.

I don't sure I am seeing your point. Per my point, even the

Quoted 8 lines

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not setting, you still need to save the value to vmcs12->vmx_preemption_timer_value. Or else, prepare_vmcs02 cannot get the right value.
If L2 doesn't enable VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, its value will be reset to what we set initially. This function is called due to L2->L1, so we should emulate L2's real exit process here. What we have done in other parts is to handle vmexit of L2->L0->L2, which is not the "real" L2 vmexit.

I am not describing it clearly. I mean if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not set, then prepare_vmcs02() always write the initial value which means vmx_preemption_timer_value to vmcs02 before L2 running. So the vmcs wirte here is redundant. Am I wrong?

Quoted 32 lines

Arthur

Quoted 7 lines
Quoted 7 lines

Arthur

Quoted 5 lines
Quoted 5 lines
arch/x86/kvm/vmx.c +2 -0
+     }
+
      /*
       * In some cases (usually, nested EPT), L2 is allowed to change

its

Quoted 22 lines
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines

* own CR3 without exiting. If it has changed it, we must keep it.
--
1.7.9.5

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 8 lines
Quoted 4 lines

Best regards,
Yang

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

Best regards,
Yang

#6 lore raw
Quoted 204 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 157 lines
Quoted 13 lines

Arthur Chunqi Li wrote on 2013-09-05:

Quoted 12 lines
Quoted 9 lines · Yang Z

Arthur Chunqi Li wrote on 2013-09-04:

Quoted 8 lines

This patch contains the following two changes:
1. Fix the bug in nested preemption timer support. If vmexit L2->L0 with some reasons not emulated by L1, preemption timer value should be save in such exits.
2. Add support of "Save VMX-preemption timer value" VM-Exit controls to nVMX.

With this patch, nested VMX preemption timer features are fully

supported.

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines

---
This series depends on queue.

 arch/x86/include/uapi/asm/msr-index.h |    1 +
 arch/x86/kvm/vmx.c                    |   51
++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 4 deletions(-)
arch/x86/include/uapi/asm/msr-index.h +0 -0
index bb04650..b93e09a 100644
@@ -536,6 +536,7 @@
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

arch/x86/include/uapi/asm/msr-index.h line 538

Quoted 2 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 /* MSR_IA32_VMX_MISC bits */
 #define MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS (1ULL

<<

Quoted 4 lines
Quoted 4 lines

29)

Quoted 3 lines
Quoted 3 lines
arch/x86/include/uapi/asm/msr-index.h +1 -0
+#define MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE   0x1F
 /* AMD-V MSRs */
 
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

arch/x86/kvm/vmx.c

Quoted 11 lines
arch/x86/include/uapi/asm/msr-index.h +0 -0
 #define MSR_VM_CR                       0xc0010114
arch/x86/kvm/vmx.c index renamed from arch/x86/kvm/vmx.c +0 -0

1f1da43..870caa8
100644

arch/x86/kvm/vmx.c +0 -0
@@ -2204,7 +2204,14 @@ static __init void

nested_vmx_setup_ctls_msrs(void) #ifdef CONFIG_X86_64
VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif

arch/x86/kvm/vmx.c +1 -1
-             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
+             VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT

|

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

arch/x86/kvm/vmx.c

Quoted 5 lines
arch/x86/kvm/vmx.c +2 -0
+             VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;
+     if (!(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_exit_ctls_high &=
+

(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

+ if (!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER))

arch/x86/kvm/vmx.c +2 -0
+             nested_vmx_pinbased_ctls_high &=
+                     (~PIN_BASED_VMX_PREEMPTION_TIMER);
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The following logic is more clearly:
if(nested_vmx_pinbased_ctls_high &

PIN_BASED_VMX_PREEMPTION_TIMER)

Quoted 1 line
        nested_vmx_exit_ctls_high |=

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER
Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this

does

Quoted 6 lines
Quoted 4 lines

occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

No. Only VM_EXIT_SAVE_VMX_PREEMPTION_TIMER depends on PIN_BASED_VMX_PREEMPTION_TIMER.

PIN_BASED_VMX_PREEMPTION_TIMER is an

Quoted 16 lines

independent feature

Quoted 14 lines

You remind that this piece of codes can write like this:
if (!(nested_vmx_pin_based_ctls_high & PIN_BASED_VMX_PREEMPTION_TIMER) ||
!(nested_vmx_exit_ctls_high &
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) {
nested_vmx_exit_ctls_high
&=(~VM_EXIT_SAVE_VMX_PREEMPTION_TIMER);
nested_vmx_pinbased_ctls_high &=
(~PIN_BASED_VMX_PREEMPTION_TIMER);
}

This may reflect the logic I describe above that these two flags should support simultaneously, and brings less confusion.

Quoted 1 line
Quoted 12 lines
Quoted 7 lines
Quoted 2 lines

BTW: I don't see nested_vmx_setup_ctls_msrs() considers the hardware's

capability when expose those vmx features(not just preemption timer) to L1. The codes just above here, when setting pinbased control for nested vmx, it firstly "rdmsr MSR_IA32_VMX_PINBASED_CTLS", then use this to mask the features hardware not support. So does other control fields.

Quoted 1 line

Yes, I saw it.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
      nested_vmx_exit_ctls_high |=

(VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |

VM_EXIT_LOAD_IA32_EFER);

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
arch/x86/kvm/vmx.c +0 -0
@@ -6707,6 +6714,23 @@ static void vmx_get_exit_info(struct
 kvm_vcpu *vcpu, u64 *info1, u64 *info2)
      *info2 = vmcs_read32(VM_EXIT_INTR_INFO);  }
 
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+static void nested_adjust_preemption_timer(struct kvm_vcpu *vcpu) {
+     u64 delta_tsc_l1;
+     u32 preempt_val_l1, preempt_val_l2, preempt_scale;
+
+     preempt_scale = native_read_msr(MSR_IA32_VMX_MISC) &
+

MSR_IA32_VMX_MISC_PREEMPTION_TIMER_SCALE;

Quoted 1 line
Quoted 1 line
arch/x86/kvm/vmx.c +1 -0
+     preempt_val_l2 =

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct · line 6725

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+     delta_tsc_l1 = kvm_x86_ops->read_l1_tsc(vcpu,
+                     native_read_tsc()) -

vcpu->arch.last_guest_tsc;

Quoted 35 lines
Quoted 35 lines
Quoted 6 lines

arch/x86/kvm/vmx.c static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)

Quoted 6 lines
arch/x86/kvm/vmx.c +6 -0
+     preempt_val_l1 = delta_tsc_l1 >> preempt_scale;
+     if (preempt_val_l2 - preempt_val_l1 < 0)
+             preempt_val_l2 = 0;
+     else
+             preempt_val_l2 -= preempt_val_l1;
+     vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

preempt_val_l2); }

Quoted 4 lines

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu

Quoted 4 lines
arch/x86/kvm/vmx.c +0 -0
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
@@ -6716,6 +6740,7 @@ static int vmx_handle_exit(struct kvm_vcpu

*vcpu)

Quoted 23 lines
Quoted 12 lines
arch/x86/kvm/vmx.c +1 -0
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exit_reason = vmx->exit_reason;
      u32 vectoring_info = vmx->idt_vectoring_info;
+     int ret;
 
      /* If guest state is invalid, start emulating */
      if (vmx->emulation_required) @@ -6795,12 +6820,15 @@ static

int vmx_handle_exit(struct kvm_vcpu
*vcpu)

if (exit_reason < kvm_vmx_max_exit_handlers
&& kvm_vmx_exit_handlers[exit_reason])

arch/x86/kvm/vmx.c static int vmx_handle_exit(struct kvm_vcpu *vcpu) · line 6830

Quoted 10 lines
arch/x86/kvm/vmx.c +4 -2
-             return kvm_vmx_exit_handlers[exit_reason](vcpu);
+             ret = kvm_vmx_exit_handlers[exit_reason](vcpu);
      else {
              vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
              vcpu->run->hw.hardware_exit_reason = exit_reason;
+             ret = 0;
      }
-     return 0;
+     if (is_guest_mode(vcpu))
+             nested_adjust_preemption_timer(vcpu);

Move this forward to avoid the changes for ret.

Quoted 8 lines
Quoted 5 lines
Quoted 4 lines

The previous codes simply "return kvm_vmx_exit_handlers[exit_reason](vcpu);", which may also consumes CPU times. So put "nested_adjust_preemption_timer" ahead may cause the statistics inaccuracy.

Then you should put it before vmentry. Here still far from the point of

vmentry.
So where is the actual point of vmentry? I'm not quite familiar with that piece of codes.

vmx_vcpu_run should be the right place.

Quoted 36 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

+ return ret;
}

static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr,
int
irr) @@
-7518,6 +7546,7 @@ static void prepare_vmcs02(struct kvm_vcpu

*vcpu, struct vmcs12 *vmcs12)  {
      struct vcpu_vmx *vmx = to_vmx(vcpu);
      u32 exec_control;

+ u32 exit_control;

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

vmcs_write16(GUEST_ES_SELECTOR,

vmcs12->guest_es_selector);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

vmcs_write16(GUEST_CS_SELECTOR,

vmcs12->guest_cs_selector);

Quoted 19 lines
Quoted 12 lines

@@

Quoted 3 lines
Quoted 3 lines

-7691,7 +7720,10 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
* we should use its exit controls. Note that

VM_EXIT_LOAD_IA32_EFER

Quoted 7 lines
Quoted 7 lines

* bits are further modified by vmx_set_efer() below. */

arch/x86/kvm/vmx.c +2 -1
-     vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
+     exit_control = vmcs_config.vmexit_ctrl;
+     if (vmcs12->pin_based_vm_exec_control &

PIN_BASED_VMX_PREEMPTION_TIMER)
+ exit_control |=

Quoted 6 lines

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER;

Quoted 2 lines
Quoted 1 line

+ vmcs_write32(VM_EXIT_CONTROLS, exit_control);

And here should be problem if host doesn't support

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.
Nested vmx does check the hardware support of these features in "nested_vmx_setup_ctls_msrs", see my comments above.

But here you don't check it. You just set it unconditionally.

Quoted 8 lines
Quoted 7 lines

What we have checked in nested_vmx_setup_ctls_msrs() confirms that only if these two features supported by hardware, nested vmx can enable preemption timer. That is to say, if L2 can set PIN_BASED_VMX_PREEMPTION_TIMER without failure, hardware must support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, which should guarantee in nested_vmx_setup_ctls_msrs (). (This is also why I write code like that in nested_vmx_setup_ctls_msrs ()).

I see your point. But the fact is nested_vmx_setup_ctls_msrs s an independent feature. So your previous assumption is wrong. The hardware may support PIN_BASED_VMX_PREEMPTION_TIMER without VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

I know what you mean, but see the changes of nested_vmx_setup_ctls_msrs() in this patch. "nested_vmx_setup_ctls_msrs()" is used to advertise features to nested VM. The changes to this function confirms that nested guest can set PIN_BASED_VMX_PREEMPTION_TIMER only if hardware support both PIN_BASED_VMX_PREEMPTION_TIMER and VM_EXIT_SAVE_VMX_PREEMPTION_TIMER. So I don't need to check here because if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not supported by hardware, both features will not exploit to nested VM. Nested VM cannot vmenter if they set any, so they of course can't reach here.

Quoted 41 lines
Quoted 33 lines
Quoted 19 lines
Quoted 19 lines
Quoted 5 lines
Quoted 5 lines



      /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and

VM_ENTRY_IA32E_MODE are

Quoted 10 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8129

Quoted 10 lines

* emulated by vmx_set_efer(), below.

arch/x86/kvm/vmx.c +4 -0
@@ -8090,6 +8122,17 @@ static void prepare_vmcs12(struct kvm_vcpu
 *vcpu, struct vmcs12 *vmcs12)
      vmcs12->guest_pending_dbg_exceptions =
              vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
 
+     if (vmcs12->pin_based_vm_exec_control &
+                     PIN_BASED_VMX_PREEMPTION_TIMER) {
+             if (vmcs12->vm_exit_controls &
+

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)

Quoted 2 lines
Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+                     vmcs12->vmx_preemption_timer_value =
+
Quoted 7 lines
Quoted 7 lines

vmcs_read32(VMX_PREEMPTION_TIMER_VALUE);

Quoted 2 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+             else
+

vmcs_write32(VMX_PREEMPTION_TIMER_VALUE,

Quoted 3 lines

arch/x86/kvm/vmx.c static void prepare_vmcs12(struct kvm_vcpu · line 8135

Quoted 2 lines
arch/x86/kvm/vmx.c +2 -0
+
+ vmcs12->vmx_preemption_timer_value);

Why write it to vmcs directly if

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER

Quoted 6 lines
Quoted 5 lines

not set?
Yes, writing is needless here since vmcs02 will be re-prepared via prepare_vmcs02() when L1->L2. This function just save information needed, vmcs_write is useless.

I don't sure I am seeing your point. Per my point, even the

Quoted 8 lines

VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not setting, you still need to save the value to vmcs12->vmx_preemption_timer_value. Or else, prepare_vmcs02 cannot get the right value.
If L2 doesn't enable VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, its value will be reset to what we set initially. This function is called due to L2->L1, so we should emulate L2's real exit process here. What we have done in other parts is to handle vmexit of L2->L0->L2, which is not the "real" L2 vmexit.

Quoted 1 line

I am not describing it clearly. I mean if VM_EXIT_SAVE_VMX_PREEMPTION_TIMER is not set, then prepare_vmcs02() always write the initial value which means vmx_preemption_timer_value to vmcs02 before L2 running. So the vmcs wirte here is redundant. Am I wrong?

Yes, I get your point and I meant the same, vmcs_write is redundant. I may confuse you by my poor description. ;)

Arthur

Quoted 38 lines
Quoted 4 lines

Arthur

Quoted 1 line
Quoted 29 lines
Quoted 6 lines
Quoted 6 lines

Arthur

Quoted 5 lines
Quoted 5 lines
arch/x86/kvm/vmx.c +2 -0
+     }
+
      /*
       * In some cases (usually, nested EPT), L2 is allowed to change

its

Quoted 22 lines
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines

* own CR3 without exiting. If it has changed it, we must keep it.
--
1.7.9.5

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 8 lines
Quoted 4 lines

Best regards,
Yang

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

Best regards,
Yang

Best regards,
Yang

maintainer
#7 lore raw
In reply to Zhang, Yang Z

Il 05/09/2013 11:24, Zhang, Yang Z ha scritto:

Quoted 10 lines
Quoted 6 lines

Here I have such consideration: this logic is wrong if CPU support PIN_BASED_VMX_PREEMPTION_TIMER but doesn't support VM_EXIT_SAVE_VMX_PREEMPTION_TIMER, though I don't know if this does occurs. So the codes above reads the MSR and reserves the features it supports, and here I just check if these two features are supported simultaneously.

No. Only VM_EXIT_SAVE_VMX_PREEMPTION_TIMER depends on PIN_BASED_VMX_PREEMPTION_TIMER. PIN_BASED_VMX_PREEMPTION_TIMER is an independent feature

Not for the current implementation of the preemption timer in nested VMX.

In order to emulate PIN_BASED_VMX_PREEMPTION_TIMER, you need to decrement the preemption timer in vmcs02 whenever you have an L2->L0->L2 exit. The current implementation chooses to use VM_EXIT_SAVE_VMX_PREEMPTION_TIMER to do this decrement.

It is possible to not use it. As the Intel SDM says (31.13), "the VMM may use another timer (e.g. the TSC) to track the amount of time the VM has run while still using the VMX-preemption timer for VM preemption. In this scenario the VMM would not save the VMX-preemption timer on each VM-exit but instead would reload the VMX-preemption timer with initial VM quantum less the time the VM has already run. This scenario includes all the VM-entry and VM-exit latencies in the VM run time". In fact, perhaps it's even a better choice, but for Arthur's code it is correct to depend on VM_EXIT_SAVE_VMX_PREEMPTION_TIMER.

Paolo