x86/kvm/nVMX: optimize MMU switch between L1 and L2
cover x86/kvm/nVMX: optimize MMU switch between L1 and L2
Currently, when we switch from L1 to L2 we do the following:
- Re-initialize L1 MMU as shadow EPT MMU (nested_ept_init_mmu_context())
- Re-initialize 'nested' MMU (nested_vmx_load_cr3() -> init_kvm_nested_mmu())
- Reload MMU root upon guest entry.
When we switch back we do:
- Re-initialize L1 MMU (nested_vmx_load_cr3() -> init_kvm_tdp_mmu())
- Reload MMU root upon guest entry.
This seems to be sub-optimal. Initializing MMU is expensive (thanks to update_permission_bitmask(), update_pkru_bitmask(),..) and reloading MMU root doesn't come for free.
Try to approach the issue by splitting L1-normal and L1-nested MMUs and checking if MMU reset is really needed. This spares us about 1000 cpu cycles on nested vmexit.
RFC part:
- Does this look like a plausible solution?
- SVM nested can probably be optimized in the same way.
- Doesn mmu_update_needed() cover everything?
Vitaly Kuznetsov (7):
x86/kvm/mmu: make vcpu->mmu a pointer to the current MMU x86/kvm/mmu.c: set get_pdptr hook in kvm_init_shadow_ept_mmu() x86/kvm/mmu.c: add kvm_mmu parameter to kvm_mmu_free_roots() x86/kvm/mmu: introduce guest_mmu
x86/kvm/mmu: get rid of redundant kvm_mmu_setup() x86/kvm/nVMX: introduce scache for kvm_init_shadow_ept_mmu x86/kvm/nVMX: optimize MMU switch from nested_vmx_load_cr3()
arch/x86/include/asm/kvm_host.h | 36 ++++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/mmu.c | 282 ++++++++++++++++++++++++++-------------- arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu_audit.c | 12 +- arch/x86/kvm/paging_tmpl.h | 17 +-- arch/x86/kvm/svm.c | 20 +-- arch/x86/kvm/vmx.c | 52 +++++--- arch/x86/kvm/x86.c | 34 ++--- 9 files changed, 292 insertions(+), 165 deletions(-)
Signature · 1 line
2.14.4
RFC PATCH 1/7 x86/kvm/mmu: make vcpu->mmu a pointer to the current MMU
As a preparation to full MMU split between L1 and L2 make vcpu->arch.mmu a pointer to the currently used mmu. For now, this is always vcpu->arch.root_mmu. No functional change.
--- arch/x86/include/asm/kvm_host.h | 5 +- arch/x86/kvm/mmu.c | 150 ++++++++++++++++++++-------------------- arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu_audit.c | 12 ++-- arch/x86/kvm/paging_tmpl.h | 17 ++--- arch/x86/kvm/svm.c | 14 ++-- arch/x86/kvm/vmx.c | 13 ++-- arch/x86/kvm/x86.c | 20 +++--- 8 files changed, 120 insertions(+), 113 deletions(-)
arch/x86/include/asm/kvm_host.h +4 -1
@@ -521,7 +521,10 @@ struct kvm_vcpu_arch {* the paging mode of the l1 guest. This context is always used to * handle faults. */ -struct kvm_mmu mmu; +struct kvm_mmu *mmu; + +/* Non-nested MMU for L1 */ +struct kvm_mmu root_mmu;/* * Paging state of an L2 guest (used for nested npt)
arch/x86/kvm/mmu.c +76 -74
450 lines not on this page · raw message
arch/x86/kvm/mmu.h +1 -1
@@ -79,7 +79,7 @@ static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm)static inline int kvm_mmu_reload (struct kvm_vcpu *vcpu) { -if (likely (vcpu->arch.mmu.root_hpa !=INVALID_PAGE )) +if (likely (vcpu->arch.mmu->root_hpa !=INVALID_PAGE ))return 0 ;return kvm_mmu_load (vcpu);
arch/x86/kvm/mmu_audit.c +6 -6
@@ -59,19 +59,19 @@ static void mmu_spte_walk(struct kvm_vcpu *vcpu, inspect_spte_fn fn)int i;struct kvm_mmu_page *sp; -if (!VALID_PAGE (vcpu->arch.mmu.root_hpa)) +if (!VALID_PAGE (vcpu->arch.mmu->root_hpa))return ; -if (vcpu->arch.mmu.root_level >=PT64_ROOT_4LEVEL ) { -hpa_t root = vcpu->arch.mmu.root_hpa; +if (vcpu->arch.mmu->root_level >=PT64_ROOT_4LEVEL ) { +hpa_t root = vcpu->arch.mmu->root_hpa; sp =page_header (root); -__mmu_spte_walk (vcpu, sp, fn, vcpu->arch.mmu.root_level); +__mmu_spte_walk (vcpu, sp, fn, vcpu->arch.mmu->root_level);return ; }for (i =0 ; i <4 ; ++i) { -hpa_t root = vcpu->arch.mmu.pae_root[i]; +hpa_t root = vcpu->arch.mmu->pae_root[i];if (root &&VALID_PAGE (root)) { root &=PT64_BASE_ADDR_MASK ;
@@ -122,7 +122,7 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) hpa = pfn <<PAGE_SHIFT ;if ((*sptep &PT64_BASE_ADDR_MASK ) != hpa)audit_printk (vcpu->kvm,"levels %d pfn %llx hpa %llx " -"ent %llxn" , vcpu->arch.mmu.root_level, pfn, +"ent %llxn" , vcpu->arch.mmu->root_level, pfn, hpa, *sptep); }
arch/x86/kvm/paging_tmpl.h +9 -8
@@ -158,14 +158,15 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, u64 gpte) { - if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)) + if (is_rsvd_bits_set(vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)) goto no_present;if (!FNAME (is_present_gpte)(gpte)) goto no_present;/* if accessed bit is not supported prefetch non accessed gpte */ -if (PT_HAVE_ACCESSED_DIRTY (&vcpu->arch.mmu) && !(gpte &PT_GUEST_ACCESSED_MASK )) +if (PT_HAVE_ACCESSED_DIRTY (vcpu->arch.mmu) && + !(gpte &PT_GUEST_ACCESSED_MASK )) goto no_present;return false;
@@ -480,7 +481,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,static int FNAME (walk_addr )(struct guest_walker *walker,struct kvm_vcpu *vcpu,gva_t addr,u32 access) { -return FNAME (walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu, addr, +return FNAME (walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr, access); }
@@ -509,7 +510,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, gfn =gpte_to_gfn (gpte); pte_access = sp->role.access &FNAME (gpte_access)(vcpu, gpte); -FNAME (protect_clean_gpte)(&vcpu->arch.mmu, &pte_access, gpte); +FNAME (protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte); pfn =pte_prefetch_gfn_to_pfn (vcpu, gfn, no_dirty_log && (pte_access &ACC_WRITE_MASK ));if (is_error_pfn (pfn))
@@ -604,7 +605,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, direct_access = gw->pte_access; - top_level = vcpu->arch.mmu.root_level; + top_level = vcpu->arch.mmu->root_level;if (top_level ==PT32E_ROOT_LEVEL ) top_level =PT32_ROOT_LEVEL ;/*
@@ -616,7 +617,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,if (FNAME (gpte_changed)(vcpu, gw, top_level)) goto out_gpte_changed; -if (!VALID_PAGE (vcpu->arch.mmu.root_hpa)) +if (!VALID_PAGE (vcpu->arch.mmu->root_hpa)) goto out_gpte_changed; for (shadow_walk_init(&it, vcpu, addr);
@@ -871,7 +872,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)*/ mmu_topup_memory_caches (vcpu); -if (!VALID_PAGE (vcpu->arch.mmu.root_hpa)) { +if (!VALID_PAGE (vcpu->arch.mmu->root_hpa)) {WARN_ON (1 );return ; }
@@ -1003,7 +1004,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) gfn =gpte_to_gfn (gpte); pte_access = sp->role.access; pte_access &=FNAME (gpte_access)(vcpu, gpte); -FNAME (protect_clean_gpte)(&vcpu->arch.mmu, &pte_access, gpte); +FNAME (protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);if (sync_mmio_spte (vcpu, &sp->spt[i], gfn, pte_access, &nr_present))
arch/x86/kvm/svm.c +7 -7
@@ -2920,18 +2920,18 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) {WARN_ON (mmu_is_nested (vcpu));kvm_init_shadow_mmu (vcpu); - vcpu->arch.mmu.set_cr3 = nested_svm_set_tdp_cr3; - vcpu->arch.mmu.get_cr3 = nested_svm_get_tdp_cr3; - vcpu->arch.mmu.get_pdptr = nested_svm_get_tdp_pdptr; - vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit; - vcpu->arch.mmu.shadow_root_level =get_npt_level (vcpu); -reset_shadow_zero_bits_mask (vcpu, &vcpu->arch.mmu); + vcpu->arch.mmu->set_cr3 = nested_svm_set_tdp_cr3; + vcpu->arch.mmu->get_cr3 = nested_svm_get_tdp_cr3; + vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; + vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; + vcpu->arch.mmu->shadow_root_level =get_npt_level (vcpu); +reset_shadow_zero_bits_mask (vcpu, vcpu->arch.mmu); vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; }static void nested_svm_uninit_mmu_context (struct kvm_vcpu *vcpu) { - vcpu->arch.walk_mmu = &vcpu->arch.mmu; + vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; }static int nested_svm_check_permissions (struct vcpu_svm *svm)
arch/x86/kvm/vmx.c +7 -6
@@ -4755,9 +4755,10 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid, bool invalidate_gpa) {if (enable_ept && (invalidate_gpa || !enable_vpid)) { -if (!VALID_PAGE (vcpu->arch.mmu.root_hpa)) +if (!VALID_PAGE (vcpu->arch.mmu->root_hpa))return ; -ept_sync_context (construct_eptp (vcpu, vcpu->arch.mmu.root_hpa)); +ept_sync_context (construct_eptp (vcpu, + vcpu->arch.mmu->root_hpa)); }else {vpid_sync_context (vpid); }
@@ -10577,9 +10578,9 @@ static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)to_vmx (vcpu)->nested.msrs.ept_caps &VMX_EPT_EXECUTE_ONLY_BIT , nested_ept_ad_enabled(vcpu)); - vcpu->arch.mmu.set_cr3 = vmx_set_cr3; - vcpu->arch.mmu.get_cr3 = nested_ept_get_cr3; - vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault; + vcpu->arch.mmu->set_cr3 = vmx_set_cr3; + vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; + vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;return 0 ;
@@ -10587,7 +10588,7 @@ static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)static void nested_ept_uninit_mmu_context (struct kvm_vcpu *vcpu) { - vcpu->arch.walk_mmu = &vcpu->arch.mmu; + vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; } static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
arch/x86/kvm/x86.c +10 -10
@@ -502,7 +502,7 @@ static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fauif (mmu_is_nested (vcpu) && !fault->nested_page_fault) vcpu->arch.nested_mmu.inject_page_fault (vcpu, fault);else - vcpu->arch.mmu.inject_page_fault (vcpu, fault); + vcpu->arch.mmu->inject_page_fault (vcpu, fault);return fault->nested_page_fault; }
@@ -601,7 +601,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)for (i =0 ; i <ARRAY_SIZE (pdpte); ++i) {if ((pdpte[i] &PT_PRESENT_MASK ) && (pdpte[i] & - vcpu->arch.mmu.guest_rsvd_check.rsvd_bits_mask[0 ][2 ])) { + vcpu->arch.mmu->guest_rsvd_check.rsvd_bits_mask[0 ][2 ])) { ret =0 ; goto out; }
@@ -4700,7 +4700,7 @@ gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,/* NPT walks are always user-walks */ access |=PFERR_USER_MASK ; - t_gpa = vcpu->arch.mmu.gva_to_gpa (vcpu, gpa, access, exception); + t_gpa = vcpu->arch.mmu->gva_to_gpa (vcpu, gpa, access, exception);return t_gpa; }
@@ -5780,7 +5780,7 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,if (emulation_type &EMULTYPE_NO_REEXECUTE )return false; -if (!vcpu->arch.mmu.direct_map) { +if (!vcpu->arch.mmu->direct_map) {/* * Write permission should be allowed since only * write access need to be emulated.
@@ -5813,7 +5813,7 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,kvm_release_pfn_clean (pfn);/* The instructions are well-emulated on direct mmu. */ -if (vcpu->arch.mmu.direct_map) { +if (vcpu->arch.mmu->direct_map) {unsigned int indirect_shadow_pages;spin_lock (&vcpu->kvm->mmu_lock);
@@ -5877,7 +5877,7 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt, vcpu->arch.last_retry_eip = ctxt->eip; vcpu->arch.last_retry_addr = cr2; -if (!vcpu->arch.mmu.direct_map) +if (!vcpu->arch.mmu->direct_map) gpa =kvm_mmu_gva_to_gpa_write (vcpu, cr2,NULL );kvm_mmu_unprotect_page (vcpu->kvm,gpa_to_gfn (gpa));
@@ -9171,7 +9171,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) {int r; -if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) || +if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) || work->wakeup_all)return ;
@@ -9179,11 +9179,11 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)if (unlikely (r))return ; -if (!vcpu->arch.mmu.direct_map && - work->arch.cr3 != vcpu->arch.mmu.get_cr3 (vcpu)) +if (!vcpu->arch.mmu->direct_map && + work->arch.cr3 != vcpu->arch.mmu->get_cr3 (vcpu))return ; - vcpu->arch.mmu.page_fault (vcpu, work->gva,0 , true); + vcpu->arch.mmu->page_fault (vcpu, work->gva,0 , true); }static inline u32 kvm_async_pf_hash_fn (gfn_t gfn)
Signature · 1 line
2.14.4
RFC PATCH 2/7 x86/kvm/mmu.c: set get_pdptr hook in kvm_init_shadow_ept_mmu()
In reply to Vitaly Kuznetsovkvm_init_shadow_ept_mmu() doesn't set get_pdptr() hook and is this not a problem just because MMU context is already initialized and this hook points to kvm_pdptr_read(). As we're intended to use a dedicated MMU for shadow EPT MMU set this hook explicitly.
--- arch/x86/kvm/mmu.c | 2 ++ 1 file changed, 2 insertions(+)
arch/x86/kvm/mmu.c +2 -0
@@ -4570,6 +4570,8 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->direct_map = false; context->base_role.ad_disabled = !accessed_dirty; context->base_role.guest_mode =1 ; + context->get_pdptr = kvm_pdptr_read; +update_permission_bitmask (vcpu, context, true);update_pkru_bitmask (vcpu, context, true);update_last_nonleaf_level (vcpu, context);
Signature · 1 line
2.14.4
RFC PATCH 3/7 x86/kvm/mmu.c: add kvm_mmu parameter to kvm_mmu_free_roots()
In reply to Vitaly KuznetsovAdd an option to specify which MMU root we want to free. This will be used when nested and non-nested MMUs for L1 are split.
--- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/mmu.c | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
@@ -1282,7 +1282,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);int kvm_mmu_load (struct kvm_vcpu *vcpu);void kvm_mmu_unload (struct kvm_vcpu *vcpu);void kvm_mmu_sync_roots (struct kvm_vcpu *vcpu); -void kvm_mmu_free_roots (struct kvm_vcpu *vcpu); +void kvm_mmu_free_roots (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu);gpa_t translate_nested_gpa (struct kvm_vcpu *vcpu,gpa_t gpa,u32 access,struct x86_exception *exception); gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
arch/x86/kvm/mmu.c +3 -4
@@ -3358,11 +3358,10 @@ static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa, *root_hpa =INVALID_PAGE ; } -void kvm_mmu_free_roots (struct kvm_vcpu *vcpu) +void kvm_mmu_free_roots (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu) {int i;LIST_HEAD (invalid_list); -struct kvm_mmu *mmu = vcpu->arch.mmu;if (!VALID_PAGE (mmu->root_hpa))return ;
@@ -3954,7 +3953,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu,void kvm_mmu_new_cr3 (struct kvm_vcpu *vcpu) { -kvm_mmu_free_roots (vcpu); +kvm_mmu_free_roots (vcpu, vcpu->arch.mmu); }static unsigned long get_cr3 (struct kvm_vcpu *vcpu)
@@ -4671,7 +4670,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);void kvm_mmu_unload (struct kvm_vcpu *vcpu) { -kvm_mmu_free_roots (vcpu); +kvm_mmu_free_roots (vcpu, vcpu->arch.mmu);WARN_ON (VALID_PAGE (vcpu->arch.mmu->root_hpa)); }EXPORT_SYMBOL_GPL (kvm_mmu_unload);
Signature · 1 line
2.14.4
RFC PATCH 4/7 x86/kvm/mmu: introduce guest_mmu
In reply to Vitaly KuznetsovWhen EPT is used for nested guest we need to re-init MMU as shadow EPT MMU (nested_ept_init_mmu_context() does that). When we return back from L2 to L1 kvm_mmu_reset_context() in nested_vmx_load_cr3() resets MMU back to normal TDP mode. Add a special 'guest_mmu' so we can avoid structure re-initialization and MMU root reload (in future, this is a preparatory patch).
--- arch/x86/include/asm/kvm_host.h | 3 +++ arch/x86/kvm/mmu.c | 12 ++++++++---- arch/x86/kvm/vmx.c | 29 ++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 13 deletions(-)
arch/x86/include/asm/kvm_host.h +3 -0
@@ -526,6 +526,9 @@ struct kvm_vcpu_arch {/* Non-nested MMU for L1 */ struct kvm_mmu root_mmu; +/* L1 MMU when running nested */ +struct kvm_mmu guest_mmu; +/* * Paging state of an L2 guest (used for nested npt) *
arch/x86/kvm/mmu.c +8 -4
@@ -4670,8 +4670,10 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);void kvm_mmu_unload (struct kvm_vcpu *vcpu) { -kvm_mmu_free_roots (vcpu, vcpu->arch.mmu); -WARN_ON (VALID_PAGE (vcpu->arch.mmu->root_hpa)); +kvm_mmu_free_roots (vcpu, &vcpu->arch.root_mmu); +WARN_ON (VALID_PAGE (vcpu->arch.root_mmu.root_hpa)); +kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu); +WARN_ON (VALID_PAGE (vcpu->arch.guest_mmu.root_hpa)); }EXPORT_SYMBOL_GPL (kvm_mmu_unload);
@@ -5051,8 +5053,10 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) { vcpu->arch.mmu = &vcpu->arch.root_mmu; vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; - vcpu->arch.mmu->root_hpa =INVALID_PAGE ; - vcpu->arch.mmu->translate_gpa = translate_gpa; + vcpu->arch.root_mmu.root_hpa =INVALID_PAGE ; + vcpu->arch.root_mmu.translate_gpa = translate_gpa; + vcpu->arch.guest_mmu.root_hpa =INVALID_PAGE ; + vcpu->arch.guest_mmu.translate_gpa = translate_gpa; vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;return alloc_mmu_pages (vcpu);
arch/x86/kvm/vmx.c +20 -9
@@ -8012,8 +8012,10 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)* Free whatever needs to be freed from vmx->nested when L1 goes down, or * just stops using VMX. */ -static void free_nested (struct vcpu_vmx *vmx) +static void free_nested (struct kvm_vcpu *vcpu) { +struct vcpu_vmx *vmx =to_vmx (vcpu); +if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)return ;
@@ -8045,6 +8047,8 @@ static void free_nested(struct vcpu_vmx *vmx) vmx->nested.pi_desc =NULL ; } +kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu); +free_loaded_vmcs (&vmx->nested.vmcs02); }
@@ -8053,7 +8057,7 @@ static int handle_vmoff(struct kvm_vcpu *vcpu) {if (!nested_vmx_check_permission (vcpu))return 1 ; -free_nested (to_vmx (vcpu)); +free_nested (vcpu);nested_vmx_succeed (vcpu);return kvm_skip_emulated_instruction (vcpu); }
@@ -8084,6 +8088,8 @@ static int handle_vmclear(struct kvm_vcpu *vcpu)if (vmptr == vmx->nested.current_vmptr)nested_release_vmcs12 (vmx); +kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu); +kvm_vcpu_write_guest (vcpu, vmptr + offsetof(struct vmcs12 , launch_state), &zero,sizeof (zero));
@@ -8428,6 +8434,8 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) }nested_release_vmcs12 (vmx); + +kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu);/* * Load VMCS12 from guest memory since it is not already * cached.
@@ -10244,12 +10252,12 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)*/ static void vmx_free_vcpu_nested (struct kvm_vcpu *vcpu) { -struct vcpu_vmx *vmx =to_vmx (vcpu); +struct vcpu_vmx *vmx =to_vmx (vcpu); -vcpu_load (vcpu); -vmx_switch_vmcs (vcpu, &vmx->vmcs01); -free_nested (vmx); -vcpu_put (vcpu); +vcpu_load (vcpu); +vmx_switch_vmcs (vcpu, &vmx->vmcs01); +free_nested (vcpu); +vcpu_put (vcpu); }static void vmx_free_vcpu (struct kvm_vcpu *vcpu)
@@ -10573,7 +10581,9 @@ static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)if (!valid_ept_address (vcpu,nested_ept_get_cr3 (vcpu)))return 1 ; -kvm_mmu_unload (vcpu); + vcpu->arch.mmu = &vcpu->arch.guest_mmu; +kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu); + kvm_init_shadow_ept_mmu(vcpu, to_vmx(vcpu)->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT,
@@ -10588,6 +10598,7 @@ static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)static void nested_ept_uninit_mmu_context (struct kvm_vcpu *vcpu) { + vcpu->arch.mmu = &vcpu->arch.root_mmu; vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; }
@@ -12444,7 +12455,7 @@ static void vmx_leave_nested(struct kvm_vcpu *vcpu)to_vmx (vcpu)->nested.nested_run_pending =0 ;nested_vmx_vmexit (vcpu,-1 ,0 ,0 ); } -free_nested (to_vmx (vcpu)); +free_nested (vcpu); }/*
Signature · 1 line
2.14.4
RFC PATCH 5/7 x86/kvm/mmu: get rid of redundant kvm_mmu_setup()
In reply to Vitaly Kuznetsovkvm_mmu_setup() is used only once in kvm_arch_vcpu_setup() and kvm_mmu_reset_context() is good enough: we have no root to free so the end result is not any different. init_kvm_mmu() can now be dropped too.
This is a preparatory change to optimizing kvm_mmu_reset_context().
--- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu.c | 17 +++-------------- arch/x86/kvm/x86.c | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-)
arch/x86/include/asm/kvm_host.h +0 -1
@@ -1128,7 +1128,6 @@ void kvm_mmu_module_exit(void);void kvm_mmu_destroy (struct kvm_vcpu *vcpu);int kvm_mmu_create (struct kvm_vcpu *vcpu); -void kvm_mmu_setup (struct kvm_vcpu *vcpu);void kvm_mmu_init_vm (struct kvm *kvm);void kvm_mmu_uninit_vm (struct kvm *kvm); void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
arch/x86/kvm/mmu.c +3 -14
@@ -4633,8 +4633,10 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)update_last_nonleaf_level (vcpu, g_context); } -static void init_kvm_mmu (struct kvm_vcpu *vcpu) +void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) { +kvm_mmu_unload (vcpu); +if (mmu_is_nested (vcpu))init_kvm_nested_mmu (vcpu);else if (tdp_enabled)
@@ -4642,12 +4644,6 @@ static void init_kvm_mmu(struct kvm_vcpu *vcpu)else init_kvm_softmmu (vcpu); } - -void kvm_mmu_reset_context (struct kvm_vcpu *vcpu) -{ -kvm_mmu_unload (vcpu); -init_kvm_mmu (vcpu); -}EXPORT_SYMBOL_GPL (kvm_mmu_reset_context);int kvm_mmu_load (struct kvm_vcpu *vcpu)
@@ -5062,13 +5058,6 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)return alloc_mmu_pages (vcpu); } -void kvm_mmu_setup (struct kvm_vcpu *vcpu) -{ -MMU_WARN_ON (VALID_PAGE (vcpu->arch.mmu->root_hpa)); - -init_kvm_mmu (vcpu); -} -static void kvm_mmu_invalidate_zap_pages_in_memslot (struct kvm *kvm,struct kvm_memory_slot *slot,struct kvm_page_track_notifier_node *node)
arch/x86/kvm/x86.c +1 -1
@@ -8333,7 +8333,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)kvm_vcpu_mtrr_init (vcpu);vcpu_load (vcpu);kvm_vcpu_reset (vcpu, false); -kvm_mmu_setup (vcpu); +kvm_mmu_reset_context (vcpu);vcpu_put (vcpu);return 0 ; }
Signature · 1 line
2.14.4
RFC PATCH 6/7 x86/kvm/nVMX: introduce scache for kvm_init_shadow_ept_mmu
In reply to Vitaly KuznetsovMMU re-initialization is expensive, in particular, update_permission_bitmask() and update_pkru_bitmask() are.
Cache the data used to setup shadow EPT MMU and avoid full re-init when it is unchanged.
kvm_mmu_free_roots() can be dropped from nested_ept_init_mmu_context() as we always do kvm_mmu_reset_context() in nested_vmx_load_cr3().
--- arch/x86/include/asm/kvm_host.h | 15 +++++++++++++ arch/x86/kvm/mmu.c | 50 ++++++++++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx.c | 5 +++-- 3 files changed, 67 insertions(+), 3 deletions(-)
arch/x86/include/asm/kvm_host.h +15 -0
@@ -325,6 +325,19 @@ struct rsvd_bits_validate {u64 bad_mt_xwr; }; +/* Source data used to setup MMU */ +struct kvm_mmu_sdata_cache { +unsigned int valid:1 ; +unsigned int ept_ad:1 ; +unsigned int execonly:1 ; +unsigned int cr0_wp:1 ; +unsigned int cr4_pae:1 ; +unsigned int cr4_pse:1 ; +unsigned int cr4_pke:1 ; +unsigned int cr4_smap:1 ; +unsigned int cr4_smep:1 ; +}; +/* * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit, * and 2-level 32-bit). The kvm_mmu structure abstracts the details of the
@@ -387,6 +400,8 @@ struct kvm_mmu {bool nx;u64 pdptrs[4 ];/* pae */ + +struct kvm_mmu_sdata_cache scache; };enum pmc_type {
arch/x86/kvm/mmu.c +49 -1
61 lines not on this page · raw message
arch/x86/kvm/vmx.c +3 -2
@@ -10577,12 +10577,13 @@ static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)static int nested_ept_init_mmu_context (struct kvm_vcpu *vcpu) { +unsigned long cr3 =nested_ept_get_cr3 (vcpu); +WARN_ON (mmu_is_nested (vcpu)); -if (!valid_ept_address (vcpu,nested_ept_get_cr3 (vcpu))) +if (!valid_ept_address (vcpu, cr3))return 1 ; vcpu->arch.mmu = &vcpu->arch.guest_mmu; -kvm_mmu_free_roots (vcpu, &vcpu->arch.guest_mmu); kvm_init_shadow_ept_mmu(vcpu, to_vmx(vcpu)->nested.msrs.ept_caps &
Signature · 1 line
2.14.4
RFC PATCH 7/7 x86/kvm/nVMX: optimize MMU switch from nested_vmx_load_cr3()
In reply to Vitaly KuznetsovNow we have everything in place to stop doing MMU reload when we switch from L1 to L2 and back. Generalize shadow_ept_mmu_update_needed() making it suitable for kvm_mmu_reset_context().
--- arch/x86/include/asm/kvm_host.h | 10 +++++- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/mmu.c | 74 +++++++++++++++++++++++++++++++---------- arch/x86/kvm/svm.c | 6 ++-- arch/x86/kvm/vmx.c | 7 ++-- arch/x86/kvm/x86.c | 14 ++++---- 6 files changed, 81 insertions(+), 32 deletions(-)
arch/x86/include/asm/kvm_host.h +9 -1
@@ -327,15 +327,22 @@ struct rsvd_bits_validate {/* Source data used to setup MMU */ struct kvm_mmu_sdata_cache { +unsigned long cr3; +unsigned int valid:1 ; +unsigned int smm:1 ;unsigned int ept_ad:1 ;unsigned int execonly:1 ; +unsigned int cr0_pg:1 ;unsigned int cr0_wp:1 ;unsigned int cr4_pae:1 ;unsigned int cr4_pse:1 ;unsigned int cr4_pke:1 ;unsigned int cr4_smap:1 ;unsigned int cr4_smep:1 ; +unsigned int cr4_la57:1 ; +unsigned int efer_lma:1 ; +unsigned int efer_nx:1 ; };/*
@@ -1149,7 +1156,8 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask, u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 p_mask, u64 acc_track_mask, u64 me_mask); -void kvm_mmu_reset_context (struct kvm_vcpu *vcpu); +void kvm_mmu_reset_context (struct kvm_vcpu *vcpu,bool check_if_unchanged); +void kvm_mmu_slot_remove_write_access (struct kvm *kvm,struct kvm_memory_slot *memslot); void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
arch/x86/kvm/cpuid.c +1 -1
@@ -142,7 +142,7 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu)/* Update physical-address width */ vcpu->arch.maxphyaddr =cpuid_query_maxphyaddr (vcpu); -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false);kvm_pmu_refresh (vcpu);return 0 ;
arch/x86/kvm/mmu.c +56 -18
140 lines not on this page · raw message
arch/x86/kvm/svm.c +3 -3
@@ -1574,7 +1574,7 @@ static void init_vmcb(struct vcpu_svm *svm)* It also updates the guest-visible cr0 value. */ svm_set_cr0 (&svm->vcpu,X86_CR0_NW |X86_CR0_CD |X86_CR0_ET ); -kvm_mmu_reset_context (&svm->vcpu); +kvm_mmu_reset_context (&svm->vcpu, false); save->cr4 =X86_CR4_PAE ;/* rdx = ?? */
@@ -3380,7 +3380,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm)nested_svm_unmap (page);nested_svm_uninit_mmu_context (&svm->vcpu); -kvm_mmu_reset_context (&svm->vcpu); +kvm_mmu_reset_context (&svm->vcpu, false);kvm_mmu_load (&svm->vcpu);return 0 ;
@@ -3466,7 +3466,7 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, (void )kvm_set_cr3 (&svm->vcpu, nested_vmcb->save.cr3);/* Guest paging mode is active - reset mmu */ -kvm_mmu_reset_context (&svm->vcpu); +kvm_mmu_reset_context (&svm->vcpu, false); svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;kvm_register_write (&svm->vcpu,VCPU_REGS_RAX , nested_vmcb->save.rax);
arch/x86/kvm/vmx.c +5 -2
@@ -4696,7 +4696,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu)fix_rmode_seg (VCPU_SREG_GS , &vmx->rmode.segs[VCPU_SREG_GS ]);fix_rmode_seg (VCPU_SREG_FS , &vmx->rmode.segs[VCPU_SREG_FS ]); -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false); }static void vmx_set_efer (struct kvm_vcpu *vcpu,u64 efer)
@@ -11123,6 +11123,8 @@ static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)static int nested_vmx_load_cr3 (struct kvm_vcpu *vcpu,unsigned long cr3,bool nested_ept,u32 *entry_failure_code) { +bool mmu_reset_force = false; +if (cr3 !=kvm_read_cr3 (vcpu) || (!nested_ept &&pdptrs_changed (vcpu))) {if (!nested_cr3_valid (vcpu, cr3)) { *entry_failure_code =ENTRY_FAIL_DEFAULT ;
@@ -11135,6 +11137,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne*/ if (!is_long_mode (vcpu) &&is_pae (vcpu) &&is_paging (vcpu) && !nested_ept) { + mmu_reset_force = true;if (!load_pdptrs (vcpu, vcpu->arch.walk_mmu, cr3)) { *entry_failure_code =ENTRY_FAIL_PDPTE ;return 1 ;
@@ -11145,7 +11148,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne__set_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail); } -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, !mmu_reset_force);return 0 ; }
arch/x86/kvm/x86.c +7 -7
@@ -695,7 +695,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) }if ((cr0 ^ old_cr0) & update_bits) -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false); if (((cr0 ^ old_cr0) & X86_CR0_CD) && kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
@@ -836,7 +836,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)if (((cr4 ^ old_cr4) & pdptr_bits) || (!(cr4 &X86_CR4_PCIDE ) && (old_cr4 &X86_CR4_PCIDE ))) -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false);if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE |X86_CR4_PKE ))kvm_update_cpuid (vcpu);
@@ -1162,7 +1162,7 @@ static int set_efer(struct kvm_vcpu *vcpu, u64 efer)/* Update reserved bits */ if ((efer ^ old_efer) &EFER_NX ) -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false);return 0 ; }
@@ -5898,7 +5898,7 @@ static void kvm_smm_changed(struct kvm_vcpu *vcpu)kvm_make_request (KVM_REQ_EVENT , vcpu); } -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false); }static void kvm_set_hflags (struct kvm_vcpu *vcpu,unsigned emul_flags)
@@ -7156,7 +7156,7 @@ static void enter_smm(struct kvm_vcpu *vcpu) kvm_x86_ops->set_efer (vcpu,0 );kvm_update_cpuid (vcpu); -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false); }static void process_smi (struct kvm_vcpu *vcpu)
@@ -8058,7 +8058,7 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)srcu_read_unlock (&vcpu->kvm->srcu, idx);if (mmu_reset_needed) -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false); max_bits =KVM_NR_INTERRUPTS ; pending_vec = find_first_bit(
@@ -8333,7 +8333,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)kvm_vcpu_mtrr_init (vcpu);vcpu_load (vcpu);kvm_vcpu_reset (vcpu, false); -kvm_mmu_reset_context (vcpu); +kvm_mmu_reset_context (vcpu, false);vcpu_put (vcpu);return 0 ; }
Signature · 1 line
2.14.4