Skip to contentSkip to conversation

kvm: x86/mmu: Share the same code to invalidate each vTLB entry

PATCH Superseded Lai Jiangshan 2 participants 16 messages Jan 5, 2023 – Feb 3, 2023

Subsystems (3)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover kvm: x86/mmu: Share the same code to invalidate each vTLB entry

From: Lai Jiangshan <[email protected]>

FNAME(invlpg) and FNAME(sync_page) invalidate vTLB entries but in slightly different methods.

Make them use the same method and share the same code.

Lai Jiangshan (7):
kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in
nested_ept_invalidate_addr()
kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)
kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into
mmu.c
kvm: x86/mmu: Remove FNAME(invlpg)
kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)

 arch/x86/include/asm/kvm_host.h |   7 +-
 arch/x86/kvm/mmu/mmu.c          | 177 +++++++++++++++++----------
 arch/x86/kvm/mmu/paging_tmpl.h  | 207 ++++++++------------------------
 arch/x86/kvm/vmx/nested.c       |   5 +-
 arch/x86/kvm/x86.c              |   2 +-
 5 files changed, 176 insertions(+), 222 deletions(-)
Signature · 1 line
2.19.1.6.gb485710b
#1 lore raw

PATCH 1/7 kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

From: Lai Jiangshan <[email protected]>

The @root_hpa for kvm_mmu_invalidate_gva() is called with @mmu->root.hpa or INVALID_PAGE.

Replace them with KVM_MMU_ROOT_XXX.

No fuctionalities changed.

---
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/mmu/mmu.c          | 39 ++++++++++++++++-----------------
 arch/x86/kvm/x86.c              |  2 +-
 3 files changed, 21 insertions(+), 22 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
index 2f5bf581d00a..dbea616bccce 100644
@@ -2026,7 +2026,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa);
+			    gva_t gva, ulong roots_to_invalidate);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
 
arch/x86/kvm/mmu/mmu.c +19 -20
index 5407649de547..90339b71bd56 100644
@@ -5693,8 +5693,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
 
+/* roots_to_invalidte must be some combination of the KVM_MMU_ROOT_* flags */
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa)
+			    gva_t gva, ulong roots_to_invalidate)
 {
 	int i;
 
@@ -5710,31 +5711,29 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	if (!mmu->invlpg)
 		return;
 
-	if (root_hpa == INVALID_PAGE) {
+	if ((roots_to_invalidate & KVM_MMU_ROOT_CURRENT) && VALID_PAGE(mmu->root.hpa))
 		mmu->invlpg(vcpu, gva, mmu->root.hpa);
 
-		/*
-		 * INVLPG is required to invalidate any global mappings for the VA,
-		 * irrespective of PCID. Since it would take us roughly similar amount
-		 * of work to determine whether any of the prev_root mappings of the VA
-		 * is marked global, or to just sync it blindly, so we might as well
-		 * just always sync it.
-		 *
-		 * Mappings not reachable via the current cr3 or the prev_roots will be
-		 * synced when switching to that cr3, so nothing needs to be done here
-		 * for them.
-		 */
-		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
-			if (VALID_PAGE(mmu->prev_roots[i].hpa))
-				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
-	} else {
-		mmu->invlpg(vcpu, gva, root_hpa);
-	}
+	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
+		if ((roots_to_invalidate & KVM_MMU_ROOT_PREVIOUS(i)) &&
+		    VALID_PAGE(mmu->prev_roots[i].hpa))
+			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 }
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
-	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
+	/*
+	 * INVLPG is required to invalidate any global mappings for the VA,
+	 * irrespective of PCID. Since it would take us roughly similar amount
+	 * of work to determine whether any of the prev_root mappings of the VA
+	 * is marked global, or to just sync it blindly, so we might as well
+	 * just always sync it.
+	 *
+	 * Mappings not reachable via the current cr3 or the prev_roots will be
+	 * synced when switching to that cr3, so nothing needs to be done here
+	 * for them.
+	 */
+	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);
 	++vcpu->stat.invlpg;
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
arch/x86/kvm/x86.c +1 -1
index c936f8d28a53..4696cbb40545 100644
@@ -799,7 +799,7 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
 	    !(fault->error_code & PFERR_RSVD_MASK))
 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
-				       fault_mmu->root.hpa);
+				       KVM_MMU_ROOT_CURRENT);
 
 	fault_mmu->inject_page_fault(vcpu, fault);
 }
Signature · 1 line
2.19.1.6.gb485710b
#2 lore raw

Re: [PATCH 1/7] kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

Quoted 6 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

The @root_hpa for kvm_mmu_invalidate_gva() is called with @mmu->root.hpa or INVALID_PAGE.

Replace them with KVM_MMU_ROOT_XXX.

Please explain _why_. I can (and did) figure it out on my own, but doing that takes time and slows down reviews.

Quoted 27 lines

No fuctionalities changed.

---
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/mmu/mmu.c          | 39 ++++++++++++++++-----------------
 arch/x86/kvm/x86.c              |  2 +-
 3 files changed, 21 insertions(+), 22 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
index 2f5bf581d00a..dbea616bccce 100644
@@ -2026,7 +2026,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa);
+			    gva_t gva, ulong roots_to_invalidate);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
 
arch/x86/kvm/mmu/mmu.c +0 -0
index 5407649de547..90339b71bd56 100644

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5696

Quoted 4 lines
arch/x86/kvm/mmu/mmu.c +1 -0
@@ -5693,8 +5693,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
 
+/* roots_to_invalidte must be some combination of the KVM_MMU_ROOT_* flags */

Typo, though I would just drop this comment. If we want some form of sanity check, it should be totally doable to add a WARN_ON_ONCE() that verifies the parameter is a subset of all possible root flags.

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5698

Quoted 3 lines
arch/x86/kvm/mmu/mmu.c +1 -1
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa)
+			    gva_t gva, ulong roots_to_invalidate)

s/ulong/unsigned long

And I got confused by "roots_to_invalidate"; I thought it meant "invalidate these entire trees" as opposed to "invalidate the gva in these trees". Best I can come up with is simply "roots".

Quoted 19 lines
arch/x86/kvm/mmu/mmu.c +1 -9
 {
 	int i;
 
@@ -5710,31 +5711,29 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	if (!mmu->invlpg)
 		return;
 
-	if (root_hpa == INVALID_PAGE) {
+	if ((roots_to_invalidate & KVM_MMU_ROOT_CURRENT) && VALID_PAGE(mmu->root.hpa))
 		mmu->invlpg(vcpu, gva, mmu->root.hpa);
 
-		/*
-		 * INVLPG is required to invalidate any global mappings for the VA,
-		 * irrespective of PCID. Since it would take us roughly similar amount
-		 * of work to determine whether any of the prev_root mappings of the VA
-		 * is marked global, or to just sync it blindly, so we might as well
-		 * just always sync it.
-		 *
-		 * Mappings not reachable via the current cr3 or the prev_roots will be

arch/x86/kvm/mmu/mmu.c void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, · line 5717

Quoted 10 lines
arch/x86/kvm/mmu/mmu.c +1 -9
-		 * synced when switching to that cr3, so nothing needs to be done here
-		 * for them.
-		 */
-		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
-			if (VALID_PAGE(mmu->prev_roots[i].hpa))
-				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
-	} else {
-		mmu->invlpg(vcpu, gva, root_hpa);
-	}
+	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)

for-loop needs curly braces.

arch/x86/kvm/mmu/mmu.c void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, · line 5720

Quoted 3 lines
arch/x86/kvm/mmu/mmu.c +3 -0
+		if ((roots_to_invalidate & KVM_MMU_ROOT_PREVIOUS(i)) &&
+		    VALID_PAGE(mmu->prev_roots[i].hpa))
+			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);

I think it has to go at the end of this series, but please add a patch somewhere to move the VALID_PAGE() check into __kvm_mmu_invalidate_gva(), e.g. end up with

	if (roots & KVM_MMU_ROOT_CURRENT)
		__kvm_mmu_invalidate_gva(vcpu, mmu, gva, mmu->root.hpa);

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		if (roots & KVM_MMU_ROOT_PREVIOUS(i))
			__kvm_mmu_invalidate_gva(vcpu, mmu, gva,
						 mmu->prev_roots[i].hpa);
	}

arch/x86/kvm/x86.c void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, · line 802

Quoted 10 lines
arch/x86/kvm/x86.c +1 -1
index c936f8d28a53..4696cbb40545 100644
@@ -799,7 +799,7 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
 	    !(fault->error_code & PFERR_RSVD_MASK))
 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
-				       fault_mmu->root.hpa);
+				       KVM_MMU_ROOT_CURRENT);

This is logically correct, but there's potential (weird) functional change here. If this is called with an invalid root, then KVM will invalidate the GVA in all roots prior to this patch, but in no roots after this patch.

I _think_ it should be impossible get here with an invalid root. Can you try adding a prep patch to assert that the root is valid so that this patch can reasonably assert that there's no functional change?

arch/x86/kvm/x86.c +2 -0
index 508074e47bc0..fffd9b610196 100644
@@ -792,6 +792,8 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
        fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
                                               vcpu->arch.walk_mmu;
 
+       WARN_ON_ONCE(!VALID_PAGE(fault_mmu->root.hpa));
+
        /*
         * Invalidate the TLB entry for the faulting address, if it exists,
         * else the access will fault indefinitely (and to emulate hardware).
#3 lore raw

Re: [PATCH 1/7] kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

Quoted 15 lines · Sean Christopherson

This is logically correct, but there's potential (weird) functional change here. If this is called with an invalid root, then KVM will invalidate the GVA in all roots prior to this patch, but in no roots after this patch.

I _think_ it should be impossible get here with an invalid root. Can you try adding a prep patch to assert that the root is valid so that this patch can reasonably assert that there's no functional change?

arch/x86/kvm/x86.c +0 -0
index 508074e47bc0..fffd9b610196 100644

arch/x86/kvm/x86.c void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, · line 796

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -792,6 +792,8 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
        fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
                                               vcpu->arch.walk_mmu;
 
+       WARN_ON_ONCE(!VALID_PAGE(fault_mmu->root.hpa));
+

I've been updating the patches as per your suggestions.

And I suddenly realized that when fault->nested_page_fault=false with nested EPT, fault_mmu->root.hpa is always unset.

fault_mmu->root.hpa is just meaningless when fault_mmu is not vcpu->arch.mmu.

I will add it as one of the reasons for replacing the argument with KVM_MMU_ROOT_XXX.

Quoted 4 lines
arch/x86/kvm/x86.c +0 -0
        /*
         * Invalidate the TLB entry for the faulting address, if it exists,
         * else the access will fault indefinitely (and to emulate hardware).
#4 lore raw

Re: [PATCH 1/7] kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

Quoted 25 lines · Lai Jiangshan
Quoted 15 lines · Sean Christopherson

This is logically correct, but there's potential (weird) functional change here. If this is called with an invalid root, then KVM will invalidate the GVA in all roots prior to this patch, but in no roots after this patch.

I _think_ it should be impossible get here with an invalid root. Can you try adding a prep patch to assert that the root is valid so that this patch can reasonably assert that there's no functional change?

arch/x86/kvm/x86.c +0 -0
index 508074e47bc0..fffd9b610196 100644

arch/x86/kvm/x86.c void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, · line 796

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -792,6 +792,8 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
        fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
                                               vcpu->arch.walk_mmu;
 
+       WARN_ON_ONCE(!VALID_PAGE(fault_mmu->root.hpa));
+

I've been updating the patches as per your suggestions.

Quoted 5 lines

And I suddenly realized that when fault->nested_page_fault=false with nested EPT, fault_mmu->root.hpa is always unset.

fault_mmu->root.hpa is just meaningless when fault_mmu is not vcpu->arch.mmu.

Right, because there's no KVM-managed MMU.

Quoted 2 lines

I will add it as one of the reasons for replacing the argument with KVM_MMU_ROOT_XXX.

And maybe call out that when using walk_mmu, the ->invlpg() implementation is NULL, i.e. using CURRENT root is a nop.

Thanks!

#5 lore raw

PATCH 2/7 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)
arch/x86/kvm/mmu/mmu.c +7 -14
index 90339b71bd56..b0e7ac6d4e88 100644
@@ -5742,27 +5742,20 @@ EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
-	bool tlb_flush = false;
+	ulong roots_to_invalidate = 0;
 	uint i;
 
-	if (pcid == kvm_get_active_pcid(vcpu)) {
-		if (mmu->invlpg)
-			mmu->invlpg(vcpu, gva, mmu->root.hpa);
-		tlb_flush = true;
-	}
+	if (pcid == kvm_get_active_pcid(vcpu))
+		roots_to_invalidate |= KVM_MMU_ROOT_CURRENT;
 
 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
 		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
-		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
-			if (mmu->invlpg)
-				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
-			tlb_flush = true;
-		}
+		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd))
+			roots_to_invalidate |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
 
-	if (tlb_flush)
-		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
-
+	if (roots_to_invalidate)
+		kvm_mmu_invalidate_gva(vcpu, mmu, gva, roots_to_invalidate);
 	++vcpu->stat.invlpg;
 
 	/*
Signature · 1 line
2.19.1.6.gb485710b
#6 lore raw

Re: [PATCH 2/7] kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()

Quoted 16 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)
arch/x86/kvm/mmu/mmu.c +0 -0
index 90339b71bd56..b0e7ac6d4e88 100644

arch/x86/kvm/mmu/mmu.c EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); · line 5745

Quoted 5 lines
arch/x86/kvm/mmu/mmu.c +1 -1
@@ -5742,27 +5742,20 @@ EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
-	bool tlb_flush = false;
+	ulong roots_to_invalidate = 0;

"unsigned long roots" again, unless someone has a better idea.

#7 lore raw

PATCH 3/7 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c    | 1 +
 arch/x86/kvm/vmx/nested.c | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/mmu.c +1 -0
index b0e7ac6d4e88..ffef9fe0c853 100644
@@ -5719,6 +5719,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 		    VALID_PAGE(mmu->prev_roots[i].hpa))
 			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva);
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
arch/x86/kvm/vmx/nested.c +4 -1
index 557b9c468734..daf3138bafd1 100644
@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 				       gpa_t addr)
 {
+	ulong roots_to_invalidate = 0;
 	uint i;
 	struct kvm_mmu_root_info *cached_root;
 
@@ -368,8 +369,10 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 
 		if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd,
 					    eptp))
-			vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa);
+			roots_to_invalidate |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
+	if (roots_to_invalidate)
+		kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, addr, roots_to_invalidate);
 }
 
 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
Signature · 1 line
2.19.1.6.gb485710b
#8 lore raw

Re: [PATCH 3/7] kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()

Quoted 29 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c    | 1 +
 arch/x86/kvm/vmx/nested.c | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/mmu.c +1 -0
index b0e7ac6d4e88..ffef9fe0c853 100644
@@ -5719,6 +5719,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 		    VALID_PAGE(mmu->prev_roots[i].hpa))
 			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva);
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
arch/x86/kvm/vmx/nested.c +0 -0
index 557b9c468734..daf3138bafd1 100644

arch/x86/kvm/vmx/nested.c static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) · line 361

Quoted 4 lines
arch/x86/kvm/vmx/nested.c +1 -0
@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 				       gpa_t addr)
 {
+	ulong roots_to_invalidate = 0;

Same thing here.

#9 lore raw

PATCH 4/7 kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Sometimes when the guest updates its pagetable, it adds only new gptes to it without changing any existed one, so there is no point to update the sptes for these existed gptes.

Also when the sptes for these unchanged gptes are updated, the AD bits are also removed since make_spte() is called with prefetch=true which might result unneeded TLB flushing.

Do nothing if the permissions are unchanged.

---
 arch/x86/kvm/mmu/paging_tmpl.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/paging_tmpl.h +7 -1
index 11f17efbec97..ab0b031d4825 100644
@@ -985,7 +985,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 	for (i = 0; i < SPTE_ENT_PER_PAGE; i++) {
 		u64 *sptep, spte;
 		struct kvm_memory_slot *slot;
-		unsigned pte_access;
+		unsigned old_pte_access, pte_access;
 		pt_element_t gpte;
 		gpa_t pte_gpa;
 		gfn_t gfn;
@@ -1025,6 +1025,12 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 			flush = true;
 			continue;
 		}
+		/*
+		 * Do nothing if the permissions are unchanged.
+		 */
+		old_pte_access = kvm_mmu_page_get_access(sp, i);
+		if (old_pte_access == pte_access)
+			continue;
 
 		/* Update the shadowed access bits in case they changed. */
 		kvm_mmu_page_set_access(sp, i, pte_access);
Signature · 1 line
2.19.1.6.gb485710b
#10 lore raw

PATCH 5/7 kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into mmu.c

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Rename mmu->sync_page to mmu->sync_spte and move the code out of FNAME(sync_page)'s loop body into mmu.c.

Also initialize mmu->sync_spte as NULL for direct paging.

No functionalities change intended.

---
 arch/x86/include/asm/kvm_host.h |   4 +-
 arch/x86/kvm/mmu/mmu.c          |  70 ++++++++++++---
 arch/x86/kvm/mmu/paging_tmpl.h  | 147 +++++++++++---------------------
 3 files changed, 110 insertions(+), 111 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -2
index dbea616bccce..69b7967cd743 100644
@@ -441,8 +441,8 @@ struct kvm_mmu {
 	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 			    gpa_t gva_or_gpa, u64 access,
 			    struct x86_exception *exception);
-	int (*sync_page)(struct kvm_vcpu *vcpu,
-			 struct kvm_mmu_page *sp);
+	int (*sync_spte)(struct kvm_vcpu *vcpu,
+			 struct kvm_mmu_page *sp, int i);
 	void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
 	struct kvm_mmu_root_info root;
 	union kvm_cpu_role cpu_role;
arch/x86/kvm/mmu/mmu.c +58 -12

115 lines not on this page · raw message

arch/x86/kvm/mmu/paging_tmpl.h +50 -97

170 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#11 lore raw

Re: [PATCH 5/7] kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into mmu.c

Quoted 6 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Rename mmu->sync_page to mmu->sync_spte and move the code out of FNAME(sync_page)'s loop body into mmu.c.

Also initialize mmu->sync_spte as NULL for direct paging.

Please split this up, we got burned before by making one of the mmu hooks NULL, I don't want a repeat of that. I.e. nullify the direct hook only when it's very clear it can't be dereferenced.

#12 lore raw

PATCH 6/7 kvm: x86/mmu: Remove FNAME(invlpg)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Replace it with FNAME(sync_spte).

FNAME(sync_spte) combined with the shadow pagetable walk meets the semantics of the instruction INVLPG.

Using FNAME(sync_spte) can share the code with flushing vTLB (kvm_sync_page()) on invalidating each vTLB entry.

---
 arch/x86/include/asm/kvm_host.h |  1 -
 arch/x86/kvm/mmu/mmu.c          | 48 +++++++++++++++++----------
 arch/x86/kvm/mmu/paging_tmpl.h  | 59 ---------------------------------
 3 files changed, 31 insertions(+), 77 deletions(-)
arch/x86/include/asm/kvm_host.h +0 -1
index 69b7967cd743..b80de8f53130 100644
@@ -443,7 +443,6 @@ struct kvm_mmu {
 			    struct x86_exception *exception);
 	int (*sync_spte)(struct kvm_vcpu *vcpu,
 			 struct kvm_mmu_page *sp, int i);
-	void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
 	struct kvm_mmu_root_info root;
 	union kvm_cpu_role cpu_role;
 	union kvm_mmu_page_role root_role;
arch/x86/kvm/mmu/mmu.c +31 -17

109 lines not on this page · raw message

arch/x86/kvm/mmu/paging_tmpl.h +0 -59

65 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#13 lore raw

Re: [PATCH 6/7] kvm: x86/mmu: Remove FNAME(invlpg)

Quoted 6 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Replace it with FNAME(sync_spte).

FNAME(sync_spte) combined with the shadow pagetable walk meets the semantics of the instruction INVLPG.

Please call out the differences (I assume the two aren't perfectly identical), and explain why those differences are benign.

#14 lore raw

PATCH 7/7 kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

FNAME(prefetch_gpte) is always called with @no_dirty_log=true.

---
 arch/x86/kvm/mmu/paging_tmpl.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)
arch/x86/kvm/mmu/paging_tmpl.h +3 -4
index 62aac5d7d38c..2db844d5d33c 100644
@@ -519,7 +519,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker,
 
 static bool
 FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
-		     u64 *spte, pt_element_t gpte, bool no_dirty_log)
+		     u64 *spte, pt_element_t gpte)
 {
 	struct kvm_memory_slot *slot;
 	unsigned pte_access;
@@ -535,8 +535,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	pte_access = sp->role.access & FNAME(gpte_access)(gpte);
 	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
 
-	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn,
-			no_dirty_log && (pte_access & ACC_WRITE_MASK));
+	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, pte_access & ACC_WRITE_MASK);
 	if (!slot)
 		return false;
 
@@ -605,7 +604,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 		if (is_shadow_present_pte(*spte))
 			continue;
 
-		if (!FNAME(prefetch_gpte)(vcpu, sp, spte, gptep[i], true))
+		if (!FNAME(prefetch_gpte)(vcpu, sp, spte, gptep[i]))
 			break;
 	}
 }
Signature · 1 line
2.19.1.6.gb485710b
#15 lore raw
In reply to Lai Jiangshan
Quoted 26 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

FNAME(invlpg) and FNAME(sync_page) invalidate vTLB entries but in slightly different methods.

Make them use the same method and share the same code.

Lai Jiangshan (7):
kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in
nested_ept_invalidate_addr()
kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)
kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into
mmu.c
kvm: x86/mmu: Remove FNAME(invlpg)
kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)

 arch/x86/include/asm/kvm_host.h |   7 +-
 arch/x86/kvm/mmu/mmu.c          | 177 +++++++++++++++++----------
 arch/x86/kvm/mmu/paging_tmpl.h  | 207 ++++++++------------------------
 arch/x86/kvm/vmx/nested.c       |   5 +-
 arch/x86/kvm/x86.c              |   2 +-
 5 files changed, 176 insertions(+), 222 deletions(-)
Quoted 3 lines

--
2.19.1.6.gb485710b

Hello

Ping.

Cheers,
Lai