Skip to contentSkip to conversation

kvm: x86/mmu: Share the same code to invalidate each vTLB entry

PATCH Superseded Lai Jiangshan 3 participants 15 messages Feb 7, 2023 – Feb 16, 2023

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover kvm: x86/mmu: Share the same code to invalidate each vTLB entry

From: Lai Jiangshan <[email protected]>

FNAME(invlpg) and FNAME(sync_page) invalidate vTLB entries but in slightly different methods.

Make them use the same method and share the same code.

[V1]: https://lore.kernel.org/lkml/[email protected]/

Lai Jiangshan (8):
kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()
kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in
nested_ept_invalidate_addr()
kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging
kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into
mmu.c
kvm: x86/mmu: Remove FNAME(invlpg)
kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)
kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)

 arch/x86/include/asm/kvm_host.h |   7 +-
 arch/x86/kvm/mmu/mmu.c          | 177 +++++++++++++++++----------
 arch/x86/kvm/mmu/paging_tmpl.h  | 209 +++++++++-----------------------
 arch/x86/kvm/vmx/nested.c       |   5 +-
 arch/x86/kvm/x86.c              |   2 +-
 5 files changed, 178 insertions(+), 222 deletions(-)
Signature · 1 line
2.19.1.6.gb485710b
#1 lore raw

PATCH v2 1/8 kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

From: Lai Jiangshan <[email protected]>

The @root_hpa for kvm_mmu_invalidate_gva() is called with @mmu->root.hpa or INVALID_PAGE where @mmu->root.hpa is to invalidate gva for the current root (the same meaning as KVM_MMU_ROOT_CURRENT) and INVALID_PAGE is to invalidate gva for all roots (the same meaning as KVM_MMU_ROOTS_ALL).

Change the argument type of kvm_mmu_invalidate_gva() and use KVM_MMU_ROOT_XXX instead so that we can reuse the function for kvm_mmu_invpcid_gva() and nested_ept_invalidate_addr() for invalidating gva for different set of roots.

No fuctionalities changed.

---
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/mmu/mmu.c          | 39 +++++++++++++++++----------------
 arch/x86/kvm/x86.c              |  2 +-
 3 files changed, 22 insertions(+), 21 deletions(-)
arch/x86/include/asm/kvm_host.h +1 -1
index 4d2bc08794e4..81429a5640d6 100644
@@ -2026,7 +2026,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa);
+			    gva_t gva, unsigned long roots);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
 
arch/x86/kvm/mmu/mmu.c +20 -19
index c91ee2927dd7..958e8eb977ed 100644
@@ -5707,10 +5707,12 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
 
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa)
+			    gva_t gva, unsigned long roots)
 {
 	int i;
 
+	WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
+
 	/* It's actually a GPA for vcpu->arch.guest_mmu.  */
 	if (mmu != &vcpu->arch.guest_mmu) {
 		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */
@@ -5723,31 +5725,30 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	if (!mmu->invlpg)
 		return;
 
-	if (root_hpa == INVALID_PAGE) {
+	if ((roots & KVM_MMU_ROOT_CURRENT) && VALID_PAGE(mmu->root.hpa))
 		mmu->invlpg(vcpu, gva, mmu->root.hpa);
 
-		/*
-		 * INVLPG is required to invalidate any global mappings for the VA,
-		 * irrespective of PCID. Since it would take us roughly similar amount
-		 * of work to determine whether any of the prev_root mappings of the VA
-		 * is marked global, or to just sync it blindly, so we might as well
-		 * just always sync it.
-		 *
-		 * Mappings not reachable via the current cr3 or the prev_roots will be
-		 * synced when switching to that cr3, so nothing needs to be done here
-		 * for them.
-		 */
-		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
-			if (VALID_PAGE(mmu->prev_roots[i].hpa))
-				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
-	} else {
-		mmu->invlpg(vcpu, gva, root_hpa);
+	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
+		if ((roots & KVM_MMU_ROOT_PREVIOUS(i)) &&
+		    VALID_PAGE(mmu->prev_roots[i].hpa))
+			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 	}
 }
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
-	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
+	/*
+	 * INVLPG is required to invalidate any global mappings for the VA,
+	 * irrespective of PCID. Since it would take us roughly similar amount
+	 * of work to determine whether any of the prev_root mappings of the VA
+	 * is marked global, or to just sync it blindly, so we might as well
+	 * just always sync it.
+	 *
+	 * Mappings not reachable via the current cr3 or the prev_roots will be
+	 * synced when switching to that cr3, so nothing needs to be done here
+	 * for them.
+	 */
+	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);
 	++vcpu->stat.invlpg;
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
arch/x86/kvm/x86.c +1 -1
index 508074e47bc0..a81937a8fe0c 100644
@@ -799,7 +799,7 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
 	    !(fault->error_code & PFERR_RSVD_MASK))
 		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
-				       fault_mmu->root.hpa);
+				       KVM_MMU_ROOT_CURRENT);
 
 	fault_mmu->inject_page_fault(vcpu, fault);
 }
Signature · 1 line
2.19.1.6.gb485710b
#2 lore raw

Re: [PATCH V2 1/8] kvm: x86/mmu: Use KVM_MMU_ROOT_XXX for kvm_mmu_invalidate_gva()

Quoted 5 lines · Lai Jiangshan
arch/x86/kvm/mmu/mmu.c +0 -0
index c91ee2927dd7..958e8eb977ed 100644

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5714

Quoted 9 lines
arch/x86/kvm/mmu/mmu.c +2 -1
@@ -5707,10 +5707,12 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
 
 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			    gva_t gva, hpa_t root_hpa)
+			    gva_t gva, unsigned long roots)
 {
 	int i;
 
+	WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);

This does nothing since KVM_MMU_ROOTS_ALL is ~0ul. What I wanted is below, can you slot that it? Compile tested only.

--
From: Sean Christopherson <[email protected]>
Date: Thu, 9 Feb 2023 16:37:43 -0800
Subject: [PATCH 2/8] KVM: x86/mmu: Sanity check input to kvm_mmu_free_roots()

Tweak KVM_MMU_ROOTS_ALL to precisely cover all current+previous root flags, and add a sanity in kvm_mmu_free_roots() to verify that the set of roots to free doesn't stray outside KVM_MMU_ROOTS_ALL.

---
 arch/x86/include/asm/kvm_host.h | 8 ++++----
 arch/x86/kvm/mmu/mmu.c          | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)
arch/x86/include/asm/kvm_host.h +4 -4
index 4d2bc08794e4..01d34f7d009d 100644
@@ -422,6 +422,10 @@ struct kvm_mmu_root_info {
 
 #define KVM_MMU_NUM_PREV_ROOTS 3
 
+#define KVM_MMU_ROOT_CURRENT		BIT(0)
+#define KVM_MMU_ROOT_PREVIOUS(i)	BIT(1+i)
+#define KVM_MMU_ROOTS_ALL		(BIT(1 + KVM_MMU_NUM_PREV_ROOTS) - 1)
+
 #define KVM_HAVE_MMU_RWLOCK
 
 struct kvm_mmu_page;
@@ -1978,10 +1982,6 @@ static inline int __kvm_irq_line_state(unsigned long *irq_state,
 	return !!(*irq_state);
 }
 
-#define KVM_MMU_ROOT_CURRENT		BIT(0)
-#define KVM_MMU_ROOT_PREVIOUS(i)	BIT(1+i)
-#define KVM_MMU_ROOTS_ALL		(~0UL)
-
 int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
 void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
 
arch/x86/kvm/mmu/mmu.c +2 -0
index c91ee2927dd7..896abf9c1126 100644
@@ -3513,6 +3513,8 @@ void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
 	LIST_HEAD(invalid_list);
 	bool free_active_root;
 
+	WARN_ON_ONCE(roots_to_free & ~KVM_MMU_ROOTS_ALL);
+
 	BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
 
 	/* Before acquiring the MMU lock, see if we need to do any real work. */
#3 lore raw

PATCH v2 2/8 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in kvm_mmu_invpcid_gva()

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)
arch/x86/kvm/mmu/mmu.c +7 -14
index 958e8eb977ed..8563b52b8bb7 100644
@@ -5757,27 +5757,20 @@ EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
-	bool tlb_flush = false;
+	unsigned long roots = 0;
 	uint i;
 
-	if (pcid == kvm_get_active_pcid(vcpu)) {
-		if (mmu->invlpg)
-			mmu->invlpg(vcpu, gva, mmu->root.hpa);
-		tlb_flush = true;
-	}
+	if (pcid == kvm_get_active_pcid(vcpu))
+		roots |= KVM_MMU_ROOT_CURRENT;
 
 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
 		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
-		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
-			if (mmu->invlpg)
-				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
-			tlb_flush = true;
-		}
+		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd))
+			roots |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
 
-	if (tlb_flush)
-		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
-
+	if (roots)
+		kvm_mmu_invalidate_gva(vcpu, mmu, gva, roots);
 	++vcpu->stat.invlpg;
 
 	/*
Signature · 1 line
2.19.1.6.gb485710b
#4 lore raw

PATCH v2 3/8 kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c    | 1 +
 arch/x86/kvm/vmx/nested.c | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/mmu.c +1 -0
index 8563b52b8bb7..e03cf5558773 100644
@@ -5734,6 +5734,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 	}
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva);
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
arch/x86/kvm/vmx/nested.c +4 -1
index 557b9c468734..f552f3c454b1 100644
@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 				       gpa_t addr)
 {
+	unsigned long roots = 0;
 	uint i;
 	struct kvm_mmu_root_info *cached_root;
 
@@ -368,8 +369,10 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 
 		if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd,
 					    eptp))
-			vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa);
+			roots |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
+	if (roots)
+		kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, addr, roots);
 }
 
 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
Signature · 1 line
2.19.1.6.gb485710b
#5 lore raw

Re: [PATCH V2 3/8] kvm: x86/mmu: Use kvm_mmu_invalidate_gva() in nested_ept_invalidate_addr()

Quoted 37 lines · Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use kvm_mmu_invalidate_gva() instead open calls to mmu->invlpg().

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c    | 1 +
 arch/x86/kvm/vmx/nested.c | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
arch/x86/kvm/mmu/mmu.c +1 -0
index 8563b52b8bb7..e03cf5558773 100644
@@ -5734,6 +5734,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
 	}
 }
+EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva);
 
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 {
arch/x86/kvm/vmx/nested.c +1 -0
index 557b9c468734..f552f3c454b1 100644
@@ -358,6 +358,7 @@ static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 				       gpa_t addr)
 {
+	unsigned long roots = 0;
 	uint i;
 	struct kvm_mmu_root_info *cached_root;
 

arch/x86/kvm/vmx/nested.c static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, · line 375

Quoted 8 lines
arch/x86/kvm/vmx/nested.c +3 -1
@@ -368,8 +369,10 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 
 		if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd,
 					    eptp))
-			vcpu->arch.mmu->invlpg(vcpu, addr, cached_root->hpa);
+			roots |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
+	if (roots)
+		kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, addr, roots);

This is subtly buggy. On 32-bit kernels, the upper 32 bits of the GPA will get dropped. "addr" here is a gpa_t, i.e. u64, whereas kvm_mmu_invalidate_gva() takes a gva_t, i.e. unsigned long.

FNAME(gva_to_gpa) just eats the misnomer, but I think I'd rather rename this path, e.g. kvm_mmu_invalidate_addr()?

#6 lore raw

PATCH v2 4/8 kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

mmu->sync_page for direct paging is never called.

And both mmu->sync_page and mm->invlpg only make sense in shadowpaging. Setting mmu->sync_page as NULL for direct paging makes it consistent with mm->invlpg which is set NULL for the case.

---
 arch/x86/kvm/mmu/mmu.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)
arch/x86/kvm/mmu/mmu.c +2 -8
index e03cf5558773..e30ca652f6ff 100644
@@ -1789,12 +1789,6 @@ static void mark_unsync(u64 *spte)
 	kvm_mmu_mark_parents_unsync(sp);
 }
 
-static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
-			       struct kvm_mmu_page *sp)
-{
-	return -1;
-}
-
 #define KVM_PAGE_ARRAY_NR 16
 
 struct kvm_mmu_pages {
@@ -4469,7 +4463,7 @@ static void nonpaging_init_context(struct kvm_mmu *context)
 {
 	context->page_fault = nonpaging_page_fault;
 	context->gva_to_gpa = nonpaging_gva_to_gpa;
-	context->sync_page = nonpaging_sync_page;
+	context->sync_page = NULL;
 	context->invlpg = NULL;
 }
 
@@ -5157,7 +5151,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
 	context->cpu_role.as_u64 = cpu_role.as_u64;
 	context->root_role.word = root_role.word;
 	context->page_fault = kvm_tdp_page_fault;
-	context->sync_page = nonpaging_sync_page;
+	context->sync_page = NULL;
 	context->invlpg = NULL;
 	context->get_guest_pgd = get_cr3;
 	context->get_pdptr = kvm_pdptr_read;
Signature · 1 line
2.19.1.6.gb485710b
maintainer
#7 lore raw

Re: [PATCH V2 4/8] kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging

Quoted 9 lines · Lai Jiangshan

From: Lai Jiangshan<[email protected]>

mmu->sync_page for direct paging is never called.

And both mmu->sync_page and mm->invlpg only make sense in shadowpaging. Setting mmu->sync_page as NULL for direct paging makes it consistent with mm->invlpg which is set NULL for the case.

I'd rather have a WARN_ON_ONCE in kvm_sync_page(), otherwise looks good.

Paolo

#8 lore raw

Re: [PATCH V2 4/8] kvm: x86/mmu: Set mmu->sync_page as NULL for direct paging

Quoted 5 lines · Paolo Bonzini
Quoted 4 lines · Lai Jiangshan

From: Lai Jiangshan<[email protected]>

mmu->sync_page for direct paging is never called.

Quoted 7 lines
Quoted 5 lines

And both mmu->sync_page and mm->invlpg only make sense in shadowpaging. Setting mmu->sync_page as NULL for direct paging makes it consistent with mm->invlpg which is set NULL for the case.

I'd rather have a WARN_ON_ONCE in kvm_sync_page(), otherwise looks good.

Agreed, there's even a WARN_ON_ONCE() to piggyback:

	if (WARN_ON_ONCE(sp->role.direct || !vcpu->arch.mmu->sync_spte ||
			 (sp->role.word ^ root_role.word) & ~sync_role_ign.word))
		return -1;

Of course, the direct + match checks on the role effectively do the same, but I would rather be explicit.

#9 lore raw

PATCH v2 5/8 kvm: x86/mmu: Move the code out of FNAME(sync_page)'s loop body into mmu.c

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Rename mmu->sync_page to mmu->sync_spte and move the code out of FNAME(sync_page)'s loop body into mmu.c.

No functionalities change intended.

---
 arch/x86/include/asm/kvm_host.h |   4 +-
 arch/x86/kvm/mmu/mmu.c          |  64 +++++++++++++--
 arch/x86/kvm/mmu/paging_tmpl.h  | 139 +++++++++++---------------------
 3 files changed, 106 insertions(+), 101 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -2
index 81429a5640d6..6c64ebfbd778 100644
@@ -441,8 +441,8 @@ struct kvm_mmu {
 	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 			    gpa_t gva_or_gpa, u64 access,
 			    struct x86_exception *exception);
-	int (*sync_page)(struct kvm_vcpu *vcpu,
-			 struct kvm_mmu_page *sp);
+	int (*sync_spte)(struct kvm_vcpu *vcpu,
+			 struct kvm_mmu_page *sp, int i);
 	void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
 	struct kvm_mmu_root_info root;
 	union kvm_cpu_role cpu_role;
arch/x86/kvm/mmu/mmu.c +58 -6

103 lines not on this page · raw message

arch/x86/kvm/mmu/paging_tmpl.h +46 -93

160 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#10 lore raw

PATCH v2 6/8 kvm: x86/mmu: Remove FNAME(invlpg)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Use FNAME(sync_spte) to invalidate vTLB instead.

In hardware TLB, invalidating TLB entries means the translations are removed from the TLB.

In kvm shadowed vTLB, vTLB translations (combinations of shadowpaging and hardware TLB) are usually kept as long as they are clean when flushing TLB of an address space (a PCID or all) with the help of write-protections, sp->unsync, kvm_sync_page().

But a single vTLB entry is always removed in FNAME(invlpg) if sp->unsync and then prefetched. And clean vTLB entry is always removed and a new one is recreated. The new one might be failed to be created or different (with more permission) and a remote flush is always requred.

Above all, it is duplicate implementation of FNAME(sync_spte) to invalidate a vTLB entry.

Use FNAME(sync_spte) to share the code which has a slight semantics
changed: clean vTLB entry is kept.

---
 arch/x86/include/asm/kvm_host.h |  1 -
 arch/x86/kvm/mmu/mmu.c          | 48 +++++++++++++++++----------
 arch/x86/kvm/mmu/paging_tmpl.h  | 58 ---------------------------------
 3 files changed, 31 insertions(+), 76 deletions(-)
arch/x86/include/asm/kvm_host.h +0 -1
index 6c64ebfbd778..86ae8f6419f1 100644
@@ -443,7 +443,6 @@ struct kvm_mmu {
 			    struct x86_exception *exception);
 	int (*sync_spte)(struct kvm_vcpu *vcpu,
 			 struct kvm_mmu_page *sp, int i);
-	void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
 	struct kvm_mmu_root_info root;
 	union kvm_cpu_role cpu_role;
 	union kvm_mmu_page_role root_role;
arch/x86/kvm/mmu/mmu.c +31 -17

109 lines not on this page · raw message

arch/x86/kvm/mmu/paging_tmpl.h +0 -58

64 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#11 lore raw

Re: [PATCH V2 6/8] kvm: x86/mmu: Remove FNAME(invlpg)

Quoted 2 lines · Lai Jiangshan

Use FNAME(sync_spte) to share the code which has a slight semantics
changed: clean vTLB entry is kept.

...

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5753

Quoted 12 lines
arch/x86/kvm/mmu/mmu.c +12 -0
+static void __kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+				     gva_t gva, hpa_t root_hpa)
+{
+	struct kvm_shadow_walk_iterator iterator;
+
+	vcpu_clear_mmio_info(vcpu, gva);
+
+	write_lock(&vcpu->kvm->mmu_lock);
+	for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) {
+		struct kvm_mmu_page *sp = sptep_to_sp(iterator.sptep);
+
+		if (sp->unsync && *iterator.sptep) {

Please make the !0 change in a separate patch. It took me a while to connect the dots, and to also understand what I suspect is a major motivation: sync_spte() already has this check, i.e. the change is happening regardless, so might as well avoid the indirect branch.

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5760

Quoted 7 lines
arch/x86/kvm/mmu/mmu.c +7 -0
+			gfn_t gfn = kvm_mmu_page_get_gfn(sp, iterator.index);
+			int ret = mmu->sync_spte(vcpu, sp, iterator.index);
+
+			if (ret < 0)
+				mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);
+			if (ret)
+				kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);

Why open code kvm_flush_remote_tlbs_sptep()? Does it actually shave enough cycles to be visible?

If open coding is really justified, can you rebase on one of the two branches? And then change this to kvm_flush_remote_tlbs_gfn().

https://github.com/kvm-x86/linux/tree/next
https://github.com/kvm-x86/linux/tree/mmu

#12 lore raw

Re: [PATCH V2 6/8] kvm: x86/mmu: Remove FNAME(invlpg)

Quoted 11 lines · Sean Christopherson

arch/x86/kvm/mmu/mmu.c int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err · line 5760

Quoted 8 lines
arch/x86/kvm/mmu/mmu.c +7 -0
+                     gfn_t gfn = kvm_mmu_page_get_gfn(sp, iterator.index);
+                     int ret = mmu->sync_spte(vcpu, sp, iterator.index);
+
+                     if (ret < 0)
+                             mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);
+                     if (ret)
+                             kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);

Why open code kvm_flush_remote_tlbs_sptep()? Does it actually shave enough cycles to be visible?

Although I have read the code of sync_page() many times,
I don't know why I had been having the assumption that it can possibly change the sp->gfns[] (now sp->shadowed_translation[]).

I will add the following comments before calling kvm_mmu_page_get_gfn():

Get the gfn beforehand for later flushing. Although mmu->sync_spte() doesn't change it, but just avoid dependence.

Or I will use kvm_flush_remote_tlbs_sptep() with comments stating that the gfn will not be changed.

Quoted 6 lines

If open coding is really justified, can you rebase on one of the two branches? And then change this to kvm_flush_remote_tlbs_gfn().

https://github.com/kvm-x86/linux/tree/next
https://github.com/kvm-x86/linux/tree/mmu

The code was based on https://github.com/kvm-x86/linux/tree/mmu.

Thanks
Lai

#13 lore raw

PATCH v2 7/8 kvm: x86/mmu: Reduce the update to the spte in FNAME(sync_page)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

Sometimes when the guest updates its pagetable, it adds only new gptes to it without changing any existed one, so there is no point to update the sptes for these existed gptes.

Also when the sptes for these unchanged gptes are updated, the AD bits are also removed since make_spte() is called with prefetch=true which might result unneeded TLB flushing.

Do nothing if the gpte's permissions are unchanged.

---
 arch/x86/kvm/mmu/paging_tmpl.h | 5 +++++
 1 file changed, 5 insertions(+)
arch/x86/kvm/mmu/paging_tmpl.h +5 -0
index 0031fe22af3d..fca5ce349d9d 100644
@@ -967,6 +967,11 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int
 		drop_spte(vcpu->kvm, &sp->spt[i]);
 		return 1;
 	}
+	/*
+	 * Do nothing if the permissions are unchanged.
+	 */
+	if (kvm_mmu_page_get_access(sp, i) == pte_access)
+		return 0;
 
 	/* Update the shadowed access bits in case they changed. */
 	kvm_mmu_page_set_access(sp, i, pte_access);
Signature · 1 line
2.19.1.6.gb485710b
#14 lore raw

PATCH v2 8/8 kvm: x86/mmu: Remove @no_dirty_log from FNAME(prefetch_gpte)

In reply to Lai Jiangshan

From: Lai Jiangshan <[email protected]>

FNAME(prefetch_gpte) is always called with @no_dirty_log=true.

---
 arch/x86/kvm/mmu/paging_tmpl.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)
arch/x86/kvm/mmu/paging_tmpl.h +3 -4
index fca5ce349d9d..e04950015dc4 100644
@@ -519,7 +519,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker,
 
 static bool
 FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
-		     u64 *spte, pt_element_t gpte, bool no_dirty_log)
+		     u64 *spte, pt_element_t gpte)
 {
 	struct kvm_memory_slot *slot;
 	unsigned pte_access;
@@ -535,8 +535,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	pte_access = sp->role.access & FNAME(gpte_access)(gpte);
 	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
 
-	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn,
-			no_dirty_log && (pte_access & ACC_WRITE_MASK));
+	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, pte_access & ACC_WRITE_MASK);
 	if (!slot)
 		return false;
 
@@ -605,7 +604,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 		if (is_shadow_present_pte(*spte))
 			continue;
 
-		if (!FNAME(prefetch_gpte)(vcpu, sp, spte, gptep[i], true))
+		if (!FNAME(prefetch_gpte)(vcpu, sp, spte, gptep[i]))
 			break;
 	}
 }
Signature · 1 line
2.19.1.6.gb485710b