kvm: x86: Improve virtual machine startup performance
PATCH 1/1 kvm: x86: Improve virtual machine startup performance
From: Peng Hao <[email protected]>
vcpu 0 will repeatedly enter/exit the smm state during the startup phase, and kvm_init_mmu will be called repeatedly during this process. There are parts of the mmu initialization code that do not need to be modified after the first initialization.
Statistics on my server, vcpu0 when starting the virtual machine Calling kvm_init_mmu more than 600 times (due to smm state switching). The patch can save about 36 microseconds in total.
--- arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu/mmu.c | 39 ++++++++++++++++++++++----------------- arch/x86/kvm/svm/nested.c | 2 +- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 2 +- 5 files changed, 26 insertions(+), 21 deletions(-)
arch/x86/kvm/mmu.h +1 -1
@@ -67,7 +67,7 @@ static __always_inline u64 rsvd_bits(int s, int e)void kvm_mmu_set_mmio_spte_mask (u64 mmio_value,u64 mmio_mask,u64 access_mask);void kvm_mmu_set_ept_masks (bool has_ad_bits,bool has_exec_only); -void kvm_init_mmu (struct kvm_vcpu *vcpu); +void kvm_init_mmu (struct kvm_vcpu *vcpu,bool init);void kvm_init_shadow_npt_mmu (struct kvm_vcpu *vcpu,unsigned long cr0,unsigned long cr4,u64 efer,gpa_t nested_cr3); void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
arch/x86/kvm/mmu/mmu.c +22 -17
@@ -4738,7 +4738,7 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,return role; } -static void init_kvm_tdp_mmu (struct kvm_vcpu *vcpu) +static void init_kvm_tdp_mmu (struct kvm_vcpu *vcpu,bool init) {struct kvm_mmu *context = &vcpu->arch.root_mmu;struct kvm_mmu_role_regs regs =vcpu_to_role_regs (vcpu);
@@ -4749,14 +4749,17 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)return ; context->mmu_role.as_u64 = new_role.as_u64; - context->page_fault = kvm_tdp_page_fault; - context->sync_page = nonpaging_sync_page; - context->invlpg =NULL ; - context->shadow_root_level =kvm_mmu_get_tdp_level (vcpu); - context->direct_map = true; - context->get_guest_pgd = get_cr3; - context->get_pdptr = kvm_pdptr_read; - context->inject_page_fault = kvm_inject_page_fault; + +if (init) { + context->page_fault = kvm_tdp_page_fault; + context->sync_page = nonpaging_sync_page; + context->invlpg =NULL ; + context->shadow_root_level =kvm_mmu_get_tdp_level (vcpu); + context->direct_map = true; + context->get_guest_pgd = get_cr3; + context->get_pdptr = kvm_pdptr_read; + context->inject_page_fault = kvm_inject_page_fault; + } context->root_level =role_regs_to_root_level (®s);if (!is_cr0_pg (context))
@@ -4924,16 +4927,18 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, }EXPORT_SYMBOL_GPL (kvm_init_shadow_ept_mmu); -static void init_kvm_softmmu (struct kvm_vcpu *vcpu) +static void init_kvm_softmmu (struct kvm_vcpu *vcpu,bool init) {struct kvm_mmu *context = &vcpu->arch.root_mmu;struct kvm_mmu_role_regs regs =vcpu_to_role_regs (vcpu);kvm_init_shadow_mmu (vcpu, ®s); - context->get_guest_pgd = get_cr3; - context->get_pdptr = kvm_pdptr_read; - context->inject_page_fault = kvm_inject_page_fault; +if (init) { + context->get_guest_pgd = get_cr3; + context->get_pdptr = kvm_pdptr_read; + context->inject_page_fault = kvm_inject_page_fault; + } } static union kvm_mmu_role
@@ -4994,14 +4999,14 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)reset_guest_paging_metadata (vcpu, g_context); } -void kvm_init_mmu (struct kvm_vcpu *vcpu) +void kvm_init_mmu (struct kvm_vcpu *vcpu,bool init) {if (mmu_is_nested (vcpu))init_kvm_nested_mmu (vcpu);else if (tdp_enabled) -init_kvm_tdp_mmu (vcpu); +init_kvm_tdp_mmu (vcpu, init);else -init_kvm_softmmu (vcpu); +init_kvm_softmmu (vcpu, init); }EXPORT_SYMBOL_GPL (kvm_init_mmu);
@@ -5054,7 +5059,7 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) {kvm_mmu_unload (vcpu); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, false); }EXPORT_SYMBOL_GPL (kvm_mmu_reset_context);
arch/x86/kvm/svm/nested.c +1 -1
@@ -447,7 +447,7 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,kvm_register_mark_available (vcpu,VCPU_EXREG_CR3 );/* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, true);return 0 ; }
arch/x86/kvm/vmx/nested.c +1 -1
@@ -1101,7 +1101,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,kvm_register_mark_available (vcpu,VCPU_EXREG_CR3 );/* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, true);return 0 ; }
arch/x86/kvm/x86.c +1 -1
@@ -10895,7 +10895,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)vcpu_load (vcpu);kvm_set_tsc_khz (vcpu, max_tsc_khz);kvm_vcpu_reset (vcpu, false); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, true);vcpu_put (vcpu);return 0 ;
Signature · 1 line
2.27.0
Sean Christopherson
maintainerQuoted 14 lines · Peng Hao
From: Peng Hao <[email protected]>
vcpu 0 will repeatedly enter/exit the smm state during the startup phase, and
kvm_init_mmuwill be called repeatedly during this process. There are parts of the mmu initialization code that do not need to be modified after the first initialization.Statistics on my server, vcpu0 when starting the virtual machine Calling
kvm_init_mmumore than 600 times (due to smm state switching). The patch can save about 36 microseconds in total.Signed-off-by Peng Hao <[email protected]>---
arch/x86/kvm/mmu/mmu.c void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) · line 5062
Quoted 5 lines
arch/x86/kvm/mmu/mmu.c +1
-1@@ -5054,7 +5059,7 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) {kvm_mmu_unload (vcpu); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, false);
This is wrong, kvm_mmu_reset_context() is the "big hammer" and is expected to unconditionally get the MMU to a known good state. E.g. failure to initialize means this code:
context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);
will not update the shadow_root_level as expected in response to userspace changing guest.MAXPHYADDR in such a way that KVM enables/disables 5-level paging.
The SMM transitions definitely need to be fixed, and we're slowly getting there, but sadly there's no quick fix.
Quoted 26 lines · Sean Christopherson
Quoted 14 lines · Peng Hao
From: Peng Hao <[email protected]>
vcpu 0 will repeatedly enter/exit the smm state during the startup phase, and
kvm_init_mmuwill be called repeatedly during this process. There are parts of the mmu initialization code that do not need to be modified after the first initialization.Statistics on my server, vcpu0 when starting the virtual machine Calling
kvm_init_mmumore than 600 times (due to smm state switching). The patch can save about 36 microseconds in total.Signed-off-by Peng Hao <[email protected]>---
arch/x86/kvm/mmu/mmu.c void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) · line 5062
Quoted 5 lines
arch/x86/kvm/mmu/mmu.c +1
-1@@ -5054,7 +5059,7 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) {kvm_mmu_unload (vcpu); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, false);This is wrong,
kvm_mmu_reset_context()is the "big hammer" and is expected to unconditionally get the MMU to a known good state. E.g. failure to initialize means this code:
Quoted 5 lines
context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);will not update the
shadow_root_levelas expected in response to userspace changing guest.MAXPHYADDR in such a way that KVM enables/disables 5-level paging.
Thanks for pointing this out. However, other than shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition of tdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?
thanks.
Quoted 2 lines
The SMM transitions definitely need to be fixed, and we're slowly getting there, but sadly there's no quick fix.
Sean Christopherson
maintainerQuoted 32 lines · Hao Peng
Quoted 26 lines · Sean Christopherson
Quoted 14 lines · Peng Hao
From: Peng Hao <[email protected]>
vcpu 0 will repeatedly enter/exit the smm state during the startup phase, and
kvm_init_mmuwill be called repeatedly during this process. There are parts of the mmu initialization code that do not need to be modified after the first initialization.Statistics on my server, vcpu0 when starting the virtual machine Calling
kvm_init_mmumore than 600 times (due to smm state switching). The patch can save about 36 microseconds in total.Signed-off-by Peng Hao <[email protected]>---
arch/x86/kvm/mmu/mmu.c void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) · line 5062
Quoted 5 lines
arch/x86/kvm/mmu/mmu.c +1
-1@@ -5054,7 +5059,7 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) {kvm_mmu_unload (vcpu); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, false);This is wrong,
kvm_mmu_reset_context()is the "big hammer" and is expected to unconditionally get the MMU to a known good state. E.g. failure to initialize means this code:Quoted 5 lines
context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);will not update the
shadow_root_levelas expected in response to userspace changing guest.MAXPHYADDR in such a way that KVM enables/disables 5-level paging.
Quoted 7 lines
Thanks for pointing this out. However, other than
shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition oftdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?
No, passing around the "init" flag is a hack.
But, we can achieve what you want simply by initializing the constant data once per vCPU. There's a _lot_ of state that is constant for a given MMU now that KVM uses separate MMUs for L1 vs. L2 when TDP is enabled. I should get patches posted tomorrow, just need to test (famous last words).
Also, based on the number of SMM transitions, I'm guessing you're using SeaBIOS. Have you tried disabling CONFIG_CALL32_SMM, or CONFIG_USE_SMM altogether? That might be an even better way to improve performance in your environment.
Last question, do you happen to know why eliminating this code shaves 36us? The raw writes don't seem like they'd take that long. Maybe the writes to function pointers trigger stalls or mispredicts or something? If you don't have an easy answer, don't bother investigating, I'm just curious.
Quoted 44 lines · Sean Christopherson
Quoted 32 lines · Hao Peng
Quoted 26 lines · Sean Christopherson
Quoted 14 lines · Peng Hao
From: Peng Hao <[email protected]>
vcpu 0 will repeatedly enter/exit the smm state during the startup phase, and
kvm_init_mmuwill be called repeatedly during this process. There are parts of the mmu initialization code that do not need to be modified after the first initialization.Statistics on my server, vcpu0 when starting the virtual machine Calling
kvm_init_mmumore than 600 times (due to smm state switching). The patch can save about 36 microseconds in total.Signed-off-by Peng Hao <[email protected]>---
arch/x86/kvm/mmu/mmu.c void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) · line 5062
Quoted 5 lines
arch/x86/kvm/mmu/mmu.c +1
-1@@ -5054,7 +5059,7 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) {kvm_mmu_unload (vcpu); -kvm_init_mmu (vcpu); +kvm_init_mmu (vcpu, false);This is wrong,
kvm_mmu_reset_context()is the "big hammer" and is expected to unconditionally get the MMU to a known good state. E.g. failure to initialize means this code:Quoted 5 lines
context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);will not update the
shadow_root_levelas expected in response to userspace changing guest.MAXPHYADDR in such a way that KVM enables/disables 5-level paging.Quoted 7 lines
Thanks for pointing this out. However, other than
shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition oftdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?No, passing around the "init" flag is a hack.
Quoted 9 lines
But, we can achieve what you want simply by initializing the constant data once per vCPU. There's a _lot_ of state that is constant for a given MMU now that KVM uses separate MMUs for L1 vs. L2 when TDP is enabled. I should get patches posted tomorrow, just need to test (famous last words).
Also, based on the number of SMM transitions, I'm guessing you're using SeaBIOS. Have you tried disabling
CONFIG_CALL32_SMM, orCONFIG_USE_SMMaltogether? That might be an even better way to improve performance in your environment.
Both options are disabled in guest.
Quoted 4 lines
Last question, do you happen to know why eliminating this code shaves 36us? The raw writes don't seem like they'd take that long. Maybe the writes to function pointers trigger stalls or mispredicts or something? If you don't have an easy answer, don't bother investigating, I'm just curious.
I'm guessing it's because of the cache. At first, I wanted to replace it with memcpy, if the modified fields are continuous enough, I can use instructions such as erms/fsrm.
Sean Christopherson
maintainerQuoted 11 lines · Hao Peng
Quoted 10 lines · Sean Christopherson
Quoted 7 lines · Hao Peng
Thanks for pointing this out. However, other than
shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition oftdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?
Quoted 6 lines
Quoted 6 lines
No, passing around the "init" flag is a hack.
But, we can achieve what you want simply by initializing the constant data once per vCPU. There's a _lot_ of state that is constant for a given MMU now that KVM uses separate MMUs for L1 vs. L2 when TDP is enabled. I should get patches posted tomorrow, just need to test (famous last words).
Famous last words indeed. Long story short, the patches were mostly easy, but I wandered deep into a rabbit hole when trying to make ->inject_page_fault() constant per MMU. I'll get something posted this week, though exactly what that something is remains to be seen :-)
Paolo Bonzini
maintainerQuoted 19 lines · Sean Christopherson
Quoted 11 lines · Hao Peng
Quoted 10 lines · Sean Christopherson
Quoted 7 lines · Hao Peng
Thanks for pointing this out. However, other than
shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition oftdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?Quoted 6 lines
Quoted 6 lines
No, passing around the "init" flag is a hack.
But, we can achieve what you want simply by initializing the constant data once per vCPU. There's a _lot_ of state that is constant for a given MMU now that KVM uses separate MMUs for L1 vs. L2 when TDP is enabled. I should get patches posted tomorrow, just need to test (famous last words).
Quoted 4 lines
Famous last words indeed. Long story short, the patches were mostly easy, but I wandered deep into a rabbit hole when trying to make ->inject_page_fault() constant per MMU. I'll get something posted this week, though exactly what that something is remains to be seen :-)
This is exactly what I have posted a few weeks ago:
https://patchew.org/linux/[email protected]/
See in particular
KVM: nVMX/nSVM: do not monkey-patch inject_page_fault callback
KVM: x86/mmu: initialize constant-value fields just once
Paolo
Sean Christopherson
maintainerQuoted 20 lines · Paolo Bonzini
Quoted 19 lines · Sean Christopherson
Quoted 17 lines · Hao Peng
Quoted 16 lines · Sean Christopherson
Quoted 7 lines · Hao Peng
Thanks for pointing this out. However, other than
shadow_root_level, other fields of context will not
change during the entire operation, such as page_fault/sync_page/direct_map and so on under
the condition oftdp_mmu.
Is this patch still viable after careful confirmation of the fields that won't be modified?No, passing around the "init" flag is a hack.
But, we can achieve what you want simply by initializing the constant data once per vCPU. There's a _lot_ of state that is constant for a given MMU now that KVM uses separate MMUs for L1 vs. L2 when TDP is enabled. I should get patches posted tomorrow, just need to test (famous last words).
Quoted 8 lines
Quoted 4 lines
Famous last words indeed. Long story short, the patches were mostly easy, but I wandered deep into a rabbit hole when trying to make ->inject_page_fault() constant per MMU. I'll get something posted this week, though exactly what that something is remains to be seen :-)
This is exactly what I have posted a few weeks ago:
Heh, guess who's woefully behind on reviews... I'll respond to those threads. Thanks for the heads up!
Quoted 4 lines
See in particular
KVM: nVMX/nSVM: do not monkey-patch
inject_page_faultcallback
KVM: x86/mmu: initialize constant-value fields just once