kvm: x86/mmu: Share the same code to invalidate each vTLB entry
v2 cover kvm: x86/mmu: Share the same code to invalidate each vTLB entry
From: Lai Jiangshan <[email protected]>
FNAME(invlpg) and FNAME(sync_page) invalidate vTLB entries but in slightly different methods.
Make them use the same method and share the same code.
[V1]: https://lore.kernel.org/lkml/[email protected]/
Lai Jiangshan (8):
kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in
nested_ept_invalidate_addr()
kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging
kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into
mmu.c
kvm: x86/mmu: Remove FNAME(invlpg)
kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)
kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)
arch/x86/include/asm/kvm_host.h | 7 +- arch/x86/kvm/mmu/mmu.c | 177 +++++++++++++++++---------- arch/x86/kvm/mmu/paging_tmpl.h | 209 +++++++++----------------------- arch/x86/kvm/vmx/nested.c | 5 +- arch/x86/kvm/x86.c | 2 +- 5 files changed, 178 insertions(+), 222 deletions(-)
Signature · 1 line
2.19.1.6.gb485710b
PATCH v2 1/8 kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
From: Lai Jiangshan <[email protected]>
The @root_hpa for kvm_mmu_invalidate_gva() is called with @mmu->root.hpa or INVALID_PAGE where @mmu->root.hpa is to invalidate gva for the current root (the same meaning as KVM_MMU_ROOT_CURRENT) and INVALID_PAGE is to invalidate gva for all roots (the same meaning as KVM_MMU_ROOTS_ALL).
Change the argument type of kvm_mmu_invalidate_gva() and use KVM_MMU_ROOT_XXX instead so that we can reuse the function for kvm_mmu_invpcid_gva() and nested_ept_invalidate_addr() for invalidating gva for different set of roots.
No fuctionalities changed.
--- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/mmu/mmu.c | 39 +++++++++++++++++---------------- arch/x86/kvm/x86.c | 2 +- 3 files changed, 22 insertions(+), 21 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
@@ -2026,7 +2026,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len);void kvm_mmu_invlpg (struct kvm_vcpu *vcpu,gva_t gva);void kvm_mmu_invalidate_gva (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu, -gva_t gva,hpa_t root_hpa); +gva_t gva,unsigned long roots);void kvm_mmu_invpcid_gva (struct kvm_vcpu *vcpu,gva_t gva,unsigned long pcid);void kvm_mmu_new_pgd (struct kvm_vcpu *vcpu,gpa_t new_pgd);
arch/x86/kvm/mmu/mmu.c +20 -19
@@ -5707,10 +5707,12 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 errEXPORT_SYMBOL_GPL (kvm_mmu_page_fault);void kvm_mmu_invalidate_gva (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu, -gva_t gva,hpa_t root_hpa) +gva_t gva,unsigned long roots) {int i; +WARN_ON_ONCE (roots & ~KVM_MMU_ROOTS_ALL ); +/* It's actually a GPA for vcpu->arch.guest_mmu. */ if (mmu != &vcpu->arch.guest_mmu) {/* INVLPG on a non-canonical address is a NOP according to the SDM. */
@@ -5723,31 +5725,30 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,if (!mmu->invlpg)return ; -if (root_hpa ==INVALID_PAGE ) { +if ((roots &KVM_MMU_ROOT_CURRENT ) &&VALID_PAGE (mmu->root.hpa)) mmu->invlpg (vcpu, gva, mmu->root.hpa); -/* -* INVLPG is required to invalidate any global mappings for the VA, -* irrespective of PCID. Since it would take us roughly similar amount -* of work to determine whether any of the prev_root mappings of the VA -* is marked global, or to just sync it blindly, so we might as well -* just always sync it. -* -* Mappings not reachable via the current cr3 or the prev_roots will be -* synced when switching to that cr3, so nothing needs to be done here -* for them. -*/ -for (i =0 ; i <KVM_MMU_NUM_PREV_ROOTS ; i++) -if (VALID_PAGE (mmu->prev_roots[i].hpa)) - mmu->invlpg (vcpu, gva, mmu->prev_roots[i].hpa); - }else { - mmu->invlpg (vcpu, gva, root_hpa); +for (i =0 ; i <KVM_MMU_NUM_PREV_ROOTS ; i++) { +if ((roots &KVM_MMU_ROOT_PREVIOUS (i)) && +VALID_PAGE (mmu->prev_roots[i].hpa)) + mmu->invlpg (vcpu, gva, mmu->prev_roots[i].hpa); } }void kvm_mmu_invlpg (struct kvm_vcpu *vcpu,gva_t gva) { -kvm_mmu_invalidate_gva (vcpu, vcpu->arch.walk_mmu, gva,INVALID_PAGE ); +/* +* INVLPG is required to invalidate any global mappings for the VA, +* irrespective of PCID. Since it would take us roughly similar amount +* of work to determine whether any of the prev_root mappings of the VA +* is marked global, or to just sync it blindly, so we might as well +* just always sync it. +* +* Mappings not reachable via the current cr3 or the prev_roots will be +* synced when switching to that cr3, so nothing needs to be done here +* for them. +*/ +kvm_mmu_invalidate_gva (vcpu, vcpu->arch.walk_mmu, gva,KVM_MMU_ROOTS_ALL ); ++vcpu->stat.invlpg; }EXPORT_SYMBOL_GPL (kvm_mmu_invlpg);
arch/x86/kvm/x86.c +1 -1
@@ -799,7 +799,7 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,if ((fault->error_code &PFERR_PRESENT_MASK ) && !(fault->error_code &PFERR_RSVD_MASK ))kvm_mmu_invalidate_gva (vcpu, fault_mmu, fault->address, - fault_mmu->root.hpa); +KVM_MMU_ROOT_CURRENT ); fault_mmu->inject_page_fault (vcpu, fault); }
Signature · 1 line
2.19.1.6.gb485710b
Sean Christopherson
maintainerRe: [PATCH V2 1/8] kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
Quoted 5 lines · Lai Jiangshan
arch/x86/kvm/mmu/mmu.c +0
-0
arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5714
Quoted 9 lines
arch/x86/kvm/mmu/mmu.c +2
-1@@ -5707,10 +5707,12 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 errEXPORT_SYMBOL_GPL (kvm_mmu_page_fault);void kvm_mmu_invalidate_gva (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu, -gva_t gva,hpa_t root_hpa) +gva_t gva,unsigned long roots) {int i; +WARN_ON_ONCE (roots & ~KVM_MMU_ROOTS_ALL );
This does nothing since KVM_MMU_ROOTS_ALL is ~0ul. What I wanted is below, can you slot that it? Compile tested only.
--
From: Sean Christopherson <[email protected]>
Date: Thu, 9 Feb 2023 16:37:43 -0800
Subject: [PATCH 2/8] KVM: x86/mmu: Sanity check input to kvm_mmu_free_roots()
Tweak KVM_MMU_ROOTS_ALL to precisely cover all current+previous root flags, and add a sanity in kvm_mmu_free_roots() to verify that the set of roots to free doesn't stray outside KVM_MMU_ROOTS_ALL.
--- arch/x86/include/asm/kvm_host.h | 8 ++++---- arch/x86/kvm/mmu/mmu.c | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-)
arch/x86/include/asm/kvm_host.h +4 -4
@@ -422,6 +422,10 @@ struct kvm_mmu_root_info {#define KVM_MMU_NUM_PREV_ROOTS 3 +#define KVM_MMU_ROOT_CURRENT BIT(0) +#define KVM_MMU_ROOT_PREVIOUS (i) BIT(1+i) +#define KVM_MMU_ROOTS_ALL (BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1) +#define KVM_HAVE_MMU_RWLOCK struct kvm_mmu_page ;
@@ -1978,10 +1982,6 @@ static inline int __kvm_irq_line_state(unsigned long *irq_state,return !!(*irq_state); } -#define KVM_MMU_ROOT_CURRENT BIT(0) -#define KVM_MMU_ROOT_PREVIOUS (i) BIT(1+i) -#define KVM_MMU_ROOTS_ALL (~0UL) -int kvm_pic_set_irq (struct kvm_pic *pic,int irq,int irq_source_id,int level);void kvm_pic_clear_all (struct kvm_pic *pic,int irq_source_id);
arch/x86/kvm/mmu/mmu.c +2 -0
@@ -3513,6 +3513,8 @@ void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,LIST_HEAD (invalid_list);bool free_active_root; +WARN_ON_ONCE (roots_to_free & ~KVM_MMU_ROOTS_ALL ); +BUILD_BUG_ON (KVM_MMU_NUM_PREV_ROOTS >=BITS_PER_LONG );/* Before acquiring the MMU lock, see if we need to do any real work. */
PATCH v2 2/8 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().
No functional change intended.
--- arch/x86/kvm/mmu/mmu.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-)
arch/x86/kvm/mmu/mmu.c +7 -14
@@ -5757,27 +5757,20 @@ EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);void kvm_mmu_invpcid_gva (struct kvm_vcpu *vcpu,gva_t gva,unsigned long pcid) {struct kvm_mmu *mmu = vcpu->arch.mmu; -bool tlb_flush = false; +unsigned long roots =0 ;uint i; -if (pcid ==kvm_get_active_pcid (vcpu)) { -if (mmu->invlpg) - mmu->invlpg (vcpu, gva, mmu->root.hpa); - tlb_flush = true; - } +if (pcid ==kvm_get_active_pcid (vcpu)) + roots |=KVM_MMU_ROOT_CURRENT ;for (i =0 ; i <KVM_MMU_NUM_PREV_ROOTS ; i++) {if (VALID_PAGE (mmu->prev_roots[i].hpa) && - pcid ==kvm_get_pcid (vcpu, mmu->prev_roots[i].pgd)) { -if (mmu->invlpg) - mmu->invlpg (vcpu, gva, mmu->prev_roots[i].hpa); - tlb_flush = true; - } + pcid ==kvm_get_pcid (vcpu, mmu->prev_roots[i].pgd)) + roots |=KVM_MMU_ROOT_PREVIOUS (i); } -if (tlb_flush) -static_call (kvm_x86_flush_tlb_gva)(vcpu, gva); - +if (roots) +kvm_mmu_invalidate_gva (vcpu, mmu, gva, roots); ++vcpu->stat.invlpg;/*
Signature · 1 line
2.19.1.6.gb485710b
PATCH v2 3/8 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().
No functional change intended.
--- arch/x86/kvm/mmu/mmu.c | 1 + arch/x86/kvm/vmx/nested.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/mmu.c +1 -0
@@ -5734,6 +5734,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, mmu->invlpg (vcpu, gva, mmu->prev_roots[i].hpa); } } +EXPORT_SYMBOL_GPL (kvm_mmu_invalidate_gva);void kvm_mmu_invlpg (struct kvm_vcpu *vcpu,gva_t gva) {
arch/x86/kvm/vmx/nested.c +4 -1
@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)static void nested_ept_invalidate_addr (struct kvm_vcpu *vcpu,gpa_t eptp,gpa_t addr) { +unsigned long roots =0 ;uint i;struct kvm_mmu_root_info *cached_root;
@@ -368,8 +369,10 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,if (nested_ept_root_matches (cached_root->hpa, cached_root->pgd, eptp)) - vcpu->arch.mmu->invlpg (vcpu, addr, cached_root->hpa); + roots |=KVM_MMU_ROOT_PREVIOUS (i); } +if (roots) +kvm_mmu_invalidate_gva (vcpu, vcpu->arch.mmu, addr, roots); } static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
Signature · 1 line
2.19.1.6.gb485710b
Sean Christopherson
maintainerRe: [PATCH V2 3/8] kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()
Quoted 37 lines · Lai Jiangshan
From: Lai Jiangshan <[email protected]>
Use
kvm_mmu_invalidate_gva()instead open calls tommu->invlpg().No functional change intended.
Signed-off-by Lai Jiangshan <[email protected]>--- arch/x86/kvm/mmu/mmu.c | 1 + arch/x86/kvm/vmx/nested.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-)arch/x86/kvm/mmu/mmu.c +1
-0@@ -5734,6 +5734,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, mmu->invlpg (vcpu, gva, mmu->prev_roots[i].hpa); } } +EXPORT_SYMBOL_GPL (kvm_mmu_invalidate_gva);void kvm_mmu_invlpg (struct kvm_vcpu *vcpu,gva_t gva) {arch/x86/kvm/vmx/nested.c +1
-0@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)static void nested_ept_invalidate_addr (struct kvm_vcpu *vcpu,gpa_t eptp,gpa_t addr) { +unsigned long roots =0 ;uint i;struct kvm_mmu_root_info *cached_root;
arch/x86/kvm/vmx/nested.c static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, · line 375
Quoted 8 lines
arch/x86/kvm/vmx/nested.c +3
-1@@ -368,8 +369,10 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,if (nested_ept_root_matches (cached_root->hpa, cached_root->pgd, eptp)) - vcpu->arch.mmu->invlpg (vcpu, addr, cached_root->hpa); + roots |=KVM_MMU_ROOT_PREVIOUS (i); } +if (roots) +kvm_mmu_invalidate_gva (vcpu, vcpu->arch.mmu, addr, roots);
This is subtly buggy. On 32-bit kernels, the upper 32 bits of the GPA will get dropped. "addr" here is a gpa_t, i.e. u64, whereas kvm_mmu_invalidate_gva() takes a gva_t, i.e. unsigned long.
FNAME(gva_to_gpa) just eats the misnomer, but I think I'd rather rename this path, e.g. kvm_mmu_invalidate_addr()?
PATCH v2 4/8 kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
mmu->sync_page for direct paging is never called.
And both mmu->sync_page and mm->invlpg only make sense in shadowpaging. Setting mmu->sync_page as NULL for direct paging makes it consistent with mm->invlpg which is set NULL for the case.
--- arch/x86/kvm/mmu/mmu.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-)
arch/x86/kvm/mmu/mmu.c +2 -8
@@ -1789,12 +1789,6 @@ static void mark_unsync(u64 *spte)kvm_mmu_mark_parents_unsync (sp); } -static int nonpaging_sync_page (struct kvm_vcpu *vcpu, -struct kvm_mmu_page *sp) -{ -return -1 ; -} -#define KVM_PAGE_ARRAY_NR 16struct kvm_mmu_pages {
@@ -4469,7 +4463,7 @@ static void nonpaging_init_context(struct kvm_mmu *context) { context->page_fault = nonpaging_page_fault; context->gva_to_gpa = nonpaging_gva_to_gpa; - context->sync_page = nonpaging_sync_page; + context->sync_page =NULL ; context->invlpg =NULL ; }
@@ -5157,7 +5151,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, context->cpu_role.as_u64 = cpu_role.as_u64; context->root_role.word = root_role.word; context->page_fault = kvm_tdp_page_fault; - context->sync_page = nonpaging_sync_page; + context->sync_page =NULL ; context->invlpg =NULL ; context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read;
Signature · 1 line
2.19.1.6.gb485710b
Paolo Bonzini
maintainerRe: [PATCH V2 4/8] kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging
Quoted 9 lines · Lai Jiangshan
From: Lai Jiangshan<[email protected]>
mmu->sync_pagefor direct paging is never called.And both
mmu->sync_pageandmm->invlpgonly make sense in shadowpaging. Settingmmu->sync_pageas NULL for direct paging makes it consistent withmm->invlpgwhich is set NULL for the case.Signed-off-by Lai Jiangshan<[email protected]>
I'd rather have a WARN_ON_ONCE in kvm_sync_page(), otherwise looks good.
Paolo
Sean Christopherson
maintainerRe: [PATCH V2 4/8] kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging
Quoted 5 lines · Paolo Bonzini
Quoted 4 lines · Lai Jiangshan
From: Lai Jiangshan<[email protected]>
mmu->sync_pagefor direct paging is never called.
Quoted 7 lines
Quoted 5 lines
And both
mmu->sync_pageandmm->invlpgonly make sense in shadowpaging. Settingmmu->sync_pageas NULL for direct paging makes it consistent withmm->invlpgwhich is set NULL for the case.Signed-off-by Lai Jiangshan<[email protected]>I'd rather have a
WARN_ON_ONCEinkvm_sync_page(), otherwise looks good.
Agreed, there's even a WARN_ON_ONCE() to piggyback:
if (WARN_ON_ONCE(sp->role.direct || !vcpu->arch.mmu->sync_spte || (sp->role.word ^ root_role.word) & ~sync_role_ign.word)) return -1;
Of course, the direct + match checks on the role effectively do the same, but I would rather be explicit.
PATCH v2 5/8 kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into mmu.c
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Rename mmu->sync_page to mmu->sync_spte and move the code out of FNAME(sync_page)'s loop body into mmu.c.
No functionalities change intended.
--- arch/x86/include/asm/kvm_host.h | 4 +- arch/x86/kvm/mmu/mmu.c | 64 +++++++++++++-- arch/x86/kvm/mmu/paging_tmpl.h | 139 +++++++++++--------------------- 3 files changed, 106 insertions(+), 101 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -2
@@ -441,8 +441,8 @@ struct kvm_mmu {gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu,struct kvm_mmu *mmu,gpa_t gva_or_gpa,u64 access,struct x86_exception *exception); -int (*sync_page)(struct kvm_vcpu *vcpu, -struct kvm_mmu_page *sp); +int (*sync_spte)(struct kvm_vcpu *vcpu, +struct kvm_mmu_page *sp,int i);void (*invlpg)(struct kvm_vcpu *vcpu,gva_t gva,hpa_t root_hpa);struct kvm_mmu_root_info root;union kvm_cpu_role cpu_role;
arch/x86/kvm/mmu/mmu.c +58 -6
103 lines not on this page · raw message
arch/x86/kvm/mmu/paging_tmpl.h +46 -93
160 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
PATCH v2 6/8 kvm: x86/mmu: Remove FNAME(invlpg)
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Use FNAME(sync_spte) to invalidate vTLB instead.
In hardware TLB, invalidating TLB entries means the translations are removed from the TLB.
In kvm shadowed vTLB, vTLB translations (combinations of shadowpaging and hardware TLB) are usually kept as long as they are clean when flushing TLB of an address space (a PCID or all) with the help of write-protections, sp->unsync, kvm_sync_page().
But a single vTLB entry is always removed in FNAME(invlpg) if sp->unsync and then prefetched. And clean vTLB entry is always removed and a new one is recreated. The new one might be failed to be created or different (with more permission) and a remote flush is always requred.
Above all, it is duplicate implementation of FNAME(sync_spte) to invalidate a vTLB entry.
Use FNAME(sync_spte) to share the code which has a slight semantics
changed: clean vTLB entry is kept.
--- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu/mmu.c | 48 +++++++++++++++++---------- arch/x86/kvm/mmu/paging_tmpl.h | 58 --------------------------------- 3 files changed, 31 insertions(+), 76 deletions(-)
arch/x86/include/asm/kvm_host.h +0 -1
@@ -443,7 +443,6 @@ struct kvm_mmu { struct x86_exception *exception);int (*sync_spte)(struct kvm_vcpu *vcpu,struct kvm_mmu_page *sp,int i); -void (*invlpg)(struct kvm_vcpu *vcpu,gva_t gva,hpa_t root_hpa);struct kvm_mmu_root_info root;union kvm_cpu_role cpu_role;union kvm_mmu_page_role root_role;
arch/x86/kvm/mmu/mmu.c +31 -17
109 lines not on this page · raw message
arch/x86/kvm/mmu/paging_tmpl.h +0 -58
64 lines not on this page · raw message
Signature · 1 line
2.19.1.6.gb485710b
Sean Christopherson
maintainerRe: [PATCH V2 6/8] kvm: x86/mmu: Remove FNAME(invlpg)
Quoted 2 lines · Lai Jiangshan
Use
FNAME(sync_spte)to share the code which has a slight semantics
changed: clean vTLB entry is kept.
...
arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5753
Quoted 12 lines
arch/x86/kvm/mmu/mmu.c +12
-0+static void __kvm_mmu_invalidate_gva (struct kvm_vcpu *vcpu,struct kvm_mmu *mmu, +gva_t gva,hpa_t root_hpa) +{ +struct kvm_shadow_walk_iterator iterator; + +vcpu_clear_mmio_info (vcpu, gva); + +write_lock (&vcpu->kvm->mmu_lock); +for_each_shadow_entry_using_root (vcpu, root_hpa, gva, iterator) { +struct kvm_mmu_page *sp =sptep_to_sp (iterator.sptep); + +if (sp->unsync && *iterator.sptep) {
Please make the !0 change in a separate patch. It took me a while to connect the dots, and to also understand what I suspect is a major motivation: sync_spte() already has this check, i.e. the change is happening regardless, so might as well avoid the indirect branch.
arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5760
Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7
-0+gfn_t gfn =kvm_mmu_page_get_gfn (sp, iterator.index); +int ret = mmu->sync_spte (vcpu, sp, iterator.index); + +if (ret <0 ) +mmu_page_zap_pte (vcpu->kvm, sp, iterator.sptep,NULL ); +if (ret) +kvm_flush_remote_tlbs_with_address (vcpu->kvm, gfn,1 );
Why open code kvm_flush_remote_tlbs_sptep()? Does it actually shave enough cycles to be visible?
If open coding is really justified, can you rebase on one of the two branches? And then change this to kvm_flush_remote_tlbs_gfn().
https://github.com/kvm-x86/linux/tree/next
https://github.com/kvm-x86/linux/tree/mmu
Re: [PATCH V2 6/8] kvm: x86/mmu: Remove FNAME(invlpg)
Quoted 11 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5760
Quoted 8 lines
arch/x86/kvm/mmu/mmu.c +7
-0+gfn_t gfn =kvm_mmu_page_get_gfn (sp, iterator.index); +int ret = mmu->sync_spte (vcpu, sp, iterator.index); + +if (ret <0 ) +mmu_page_zap_pte (vcpu->kvm, sp, iterator.sptep,NULL ); +if (ret) +kvm_flush_remote_tlbs_with_address (vcpu->kvm, gfn,1 );Why open code
kvm_flush_remote_tlbs_sptep()? Does it actually shave enough cycles to be visible?
Although I have read the code of sync_page() many times,
I don't know why I had been having the assumption that it can possibly change the sp->gfns[] (now sp->shadowed_translation[]).
I will add the following comments before calling kvm_mmu_page_get_gfn():
Get the gfn beforehand for later flushing. Although mmu->sync_spte() doesn't change it, but just avoid dependence.
Or I will use kvm_flush_remote_tlbs_sptep() with comments stating that the gfn will not be changed.
Quoted 6 lines
If open coding is really justified, can you rebase on one of the two branches? And then change this to
kvm_flush_remote_tlbs_gfn().https://github.com/kvm-x86/linux/tree/next
https://github.com/kvm-x86/linux/tree/mmu
The code was based on https://github.com/kvm-x86/linux/tree/mmu.
Thanks
Lai
PATCH v2 7/8 kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Sometimes when the guest updates its pagetable, it adds only new gptes to it without changing any existed one, so there is no point to update the sptes for these existed gptes.
Also when the sptes for these unchanged gptes are updated, the AD bits are also removed since make_spte() is called with prefetch=true which might result unneeded TLB flushing.
Do nothing if the gpte's permissions are unchanged.
--- arch/x86/kvm/mmu/paging_tmpl.h | 5 +++++ 1 file changed, 5 insertions(+)
arch/x86/kvm/mmu/paging_tmpl.h +5 -0
@@ -967,6 +967,11 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, intdrop_spte (vcpu->kvm, &sp->spt[i]);return 1 ; } +/* +* Do nothing if the permissions are unchanged. +*/ +if (kvm_mmu_page_get_access (sp, i) == pte_access) +return 0 ;/* Update the shadowed access bits in case they changed. */ kvm_mmu_page_set_access (sp, i, pte_access);
Signature · 1 line
2.19.1.6.gb485710b
PATCH v2 8/8 kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
FNAME(prefetch_gpte) is always called with @no_dirty_log=true.
--- arch/x86/kvm/mmu/paging_tmpl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)
arch/x86/kvm/mmu/paging_tmpl.h +3 -4
@@ -519,7 +519,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker,static bool FNAME (prefetch_gpte )(struct kvm_vcpu *vcpu,struct kvm_mmu_page *sp, -u64 *spte,pt_element_t gpte,bool no_dirty_log) +u64 *spte,pt_element_t gpte) {struct kvm_memory_slot *slot;unsigned pte_access;
@@ -535,8 +535,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, pte_access = sp->role.access &FNAME (gpte_access)(gpte);FNAME (protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte); - slot =gfn_to_memslot_dirty_bitmap (vcpu, gfn, - no_dirty_log && (pte_access &ACC_WRITE_MASK )); + slot =gfn_to_memslot_dirty_bitmap (vcpu, gfn, pte_access &ACC_WRITE_MASK );if (!slot)return false;
@@ -605,7 +604,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,if (is_shadow_present_pte (*spte))continue ; -if (!FNAME (prefetch_gpte)(vcpu, sp, spte, gptep[i], true)) +if (!FNAME (prefetch_gpte)(vcpu, sp, spte, gptep[i]))break ; } }
Signature · 1 line
2.19.1.6.gb485710b