Skip to contentSkip to conversation

KVM: x86/mmu: Add RET_PF_CONTINUE to eliminate bool+int* "returns"

PATCH In review Sean Christopherson 3 participants 4 messages Apr 15, 2022 – Apr 18, 2022

Subsystems (3)
lore
Share & tools
lore

Sean Christopherson

author maintainer
#0 lore raw

PATCH 1/1 KVM: x86/mmu: Add RET_PF_CONTINUE to eliminate bool+int* "returns"

Add RET_PF_CONTINUE and use it in handle_abnormal_pfn() and kvm_faultin_pfn() to signal that the page fault handler should continue doing its thing. Aside from being gross and inefficient, using a boolean return to signal continue vs. stop makes it extremely difficult to add more helpers and/or move existing code to a helper.

E.g. hypothetically, if nested MMUs were to gain a separate page fault handler in the future, everything up to the "is self-modifying PTE" check can be shared by all shadow MMUs, but communicating up the stack whether to continue on or stop becomes a nightmare.

More concretely, proposed support for private guest memory ran into a similar issue, where it'll be forced to forego a helper in order to yield sane code: https://lore.kernel.org/all/YkJbxiL%[email protected].

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c          | 51 ++++++++++++++-------------------
 arch/x86/kvm/mmu/mmu_internal.h |  9 +++++-
 arch/x86/kvm/mmu/paging_tmpl.h  |  6 ++--
 3 files changed, 34 insertions(+), 32 deletions(-)
arch/x86/kvm/mmu/mmu.c +22 -29
index 69a30d6d1e2b..cb2982c6b513 100644
@@ -2972,14 +2972,12 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
 	return -EFAULT;
 }
 
-static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
-				unsigned int access, int *ret_val)
+static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
+			       unsigned int access)
 {
 	/* The pfn is invalid, report the error! */
-	if (unlikely(is_error_pfn(fault->pfn))) {
-		*ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
-		return true;
-	}
+	if (unlikely(is_error_pfn(fault->pfn)))
+		return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
 
 	if (unlikely(!fault->slot)) {
 		gva_t gva = fault->is_tdp ? 0 : fault->addr;
@@ -2991,13 +2989,11 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa
 		 * touching the shadow page tables as attempting to install an
 		 * MMIO SPTE will just be an expensive nop.
 		 */
-		if (unlikely(!shadow_mmio_value)) {
-			*ret_val = RET_PF_EMULATE;
-			return true;
-		}
+		if (unlikely(!shadow_mmio_value))
+			return RET_PF_EMULATE;
 	}
 
-	return false;
+	return RET_PF_CONTINUE;
 }
 
 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
@@ -3888,7 +3884,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 				  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
 }
 
-static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r)
+static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 	struct kvm_memory_slot *slot = fault->slot;
 	bool async;
@@ -3899,7 +3895,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 	 * be zapped before KVM inserts a new MMIO SPTE for the gfn.
 	 */
 	if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
-		goto out_retry;
+		return RET_PF_RETRY;
 
 	if (!kvm_is_visible_memslot(slot)) {
 		/* Don't expose private memslots to L2. */
@@ -3907,7 +3903,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 			fault->slot = NULL;
 			fault->pfn = KVM_PFN_NOSLOT;
 			fault->map_writable = false;
-			return false;
+			return RET_PF_CONTINUE;
 		}
 		/*
 		 * If the APIC access page exists but is disabled, go directly
@@ -3916,10 +3912,8 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 		 * when the AVIC is re-enabled.
 		 */
 		if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
-		    !kvm_apicv_activated(vcpu->kvm)) {
-			*r = RET_PF_EMULATE;
-			return true;
-		}
+		    !kvm_apicv_activated(vcpu->kvm))
+			return RET_PF_EMULATE;
 	}
 
 	async = false;
@@ -3927,26 +3921,23 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 					  fault->write, &fault->map_writable,
 					  &fault->hva);
 	if (!async)
-		return false; /* *pfn has correct page already */
+		return RET_PF_CONTINUE; /* *pfn has correct page already */
 
 	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
 		trace_kvm_try_async_get_page(fault->addr, fault->gfn);
 		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
 			trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
 			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
-			goto out_retry;
-		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn))
-			goto out_retry;
+			return RET_PF_RETRY;
+		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
+			return RET_PF_RETRY;
+		}
 	}
 
 	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
 					  fault->write, &fault->map_writable,
 					  &fault->hva);
-	return false;
-
-out_retry:
-	*r = RET_PF_RETRY;
-	return true;
+	return RET_PF_CONTINUE;
 }
 
 /*
@@ -4001,10 +3992,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (kvm_faultin_pfn(vcpu, fault, &r))
+	r = kvm_faultin_pfn(vcpu, fault);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
-	if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r))
+	r = handle_abnormal_pfn(vcpu, fault, ACC_ALL);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
 	r = RET_PF_RETRY;
arch/x86/kvm/mmu/mmu_internal.h +8 -1
index 1bff453f7cbe..c0e502b17ef7 100644
@@ -143,6 +143,7 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
 /*
  * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault().
  *
+ * RET_PF_CONTINUE: So far, so good, keep handling the page fault.
  * RET_PF_RETRY: let CPU fault again on the address.
  * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
  * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.
@@ -151,9 +152,15 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
  *
  * Any names added to this enum should be exported to userspace for use in
  * tracepoints via TRACE_DEFINE_ENUM() in mmutrace.h
+ *
+ * Note, all values must be greater than or equal to zero so as not to encroach
+ * on -errno return values.  Somewhat arbitrarily use '0' for CONTINUE, which
+ * will allow for efficient machine code when checking for CONTINUE, e.g.
+ * "TEST %rax, %rax, JNZ", as all "stop!" values are non-zero.
  */
 enum {
-	RET_PF_RETRY = 0,
+	RET_PF_CONTINUE = 0,
+	RET_PF_RETRY,
 	RET_PF_EMULATE,
 	RET_PF_INVALID,
 	RET_PF_FIXED,
arch/x86/kvm/mmu/paging_tmpl.h +4 -2
index 66f1acf153c4..7038273d04ab 100644
@@ -838,10 +838,12 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (kvm_faultin_pfn(vcpu, fault, &r))
+	r = kvm_faultin_pfn(vcpu, fault);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
-	if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r))
+	r = handle_abnormal_pfn(vcpu, fault, walker.pte_access);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
 	/*

base-commit: 150866cd0ec871c765181d145aa0912628289c8a

Signature · 1 line
2.36.0.rc0.470.gd361397f0d-goog
#1 lore raw
Quoted 160 lines · Sean Christopherson

Add RET_PF_CONTINUE and use it in handle_abnormal_pfn() and kvm_faultin_pfn() to signal that the page fault handler should continue doing its thing. Aside from being gross and inefficient, using a boolean return to signal continue vs. stop makes it extremely difficult to add more helpers and/or move existing code to a helper.

E.g. hypothetically, if nested MMUs were to gain a separate page fault handler in the future, everything up to the "is self-modifying PTE" check can be shared by all shadow MMUs, but communicating up the stack whether to continue on or stop becomes a nightmare.

More concretely, proposed support for private guest memory ran into a similar issue, where it'll be forced to forego a helper in order to yield sane code: https://lore.kernel.org/all/YkJbxiL%[email protected].

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c          | 51 ++++++++++++++-------------------
 arch/x86/kvm/mmu/mmu_internal.h |  9 +++++-
 arch/x86/kvm/mmu/paging_tmpl.h  |  6 ++--
 3 files changed, 34 insertions(+), 32 deletions(-)
arch/x86/kvm/mmu/mmu.c +22 -29
index 69a30d6d1e2b..cb2982c6b513 100644
@@ -2972,14 +2972,12 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
        return -EFAULT;
 }
 
-static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
-                               unsigned int access, int *ret_val)
+static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
+                              unsigned int access)
 {
        /* The pfn is invalid, report the error! */
-       if (unlikely(is_error_pfn(fault->pfn))) {
-               *ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
-               return true;
-       }
+       if (unlikely(is_error_pfn(fault->pfn)))
+               return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
 
        if (unlikely(!fault->slot)) {
                gva_t gva = fault->is_tdp ? 0 : fault->addr;
@@ -2991,13 +2989,11 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa
                 * touching the shadow page tables as attempting to install an
                 * MMIO SPTE will just be an expensive nop.
                 */
-               if (unlikely(!shadow_mmio_value)) {
-                       *ret_val = RET_PF_EMULATE;
-                       return true;
-               }
+               if (unlikely(!shadow_mmio_value))
+                       return RET_PF_EMULATE;
        }
 
-       return false;
+       return RET_PF_CONTINUE;
 }
 
 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
@@ -3888,7 +3884,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
                                  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
 }
 
-static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r)
+static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
        struct kvm_memory_slot *slot = fault->slot;
        bool async;
@@ -3899,7 +3895,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
         * be zapped before KVM inserts a new MMIO SPTE for the gfn.
         */
        if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
-               goto out_retry;
+               return RET_PF_RETRY;
 
        if (!kvm_is_visible_memslot(slot)) {
                /* Don't expose private memslots to L2. */
@@ -3907,7 +3903,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
                        fault->slot = NULL;
                        fault->pfn = KVM_PFN_NOSLOT;
                        fault->map_writable = false;
-                       return false;
+                       return RET_PF_CONTINUE;
                }
                /*
                 * If the APIC access page exists but is disabled, go directly
@@ -3916,10 +3912,8 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
                 * when the AVIC is re-enabled.
                 */
                if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
-                   !kvm_apicv_activated(vcpu->kvm)) {
-                       *r = RET_PF_EMULATE;
-                       return true;
-               }
+                   !kvm_apicv_activated(vcpu->kvm))
+                       return RET_PF_EMULATE;
        }
 
        async = false;
@@ -3927,26 +3921,23 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
                                          fault->write, &fault->map_writable,
                                          &fault->hva);
        if (!async)
-               return false; /* *pfn has correct page already */
+               return RET_PF_CONTINUE; /* *pfn has correct page already */
 
        if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
                trace_kvm_try_async_get_page(fault->addr, fault->gfn);
                if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
                        trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
                        kvm_make_request(KVM_REQ_APF_HALT, vcpu);
-                       goto out_retry;
-               } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn))
-                       goto out_retry;
+                       return RET_PF_RETRY;
+               } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
+                       return RET_PF_RETRY;
+               }
        }
 
        fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
                                          fault->write, &fault->map_writable,
                                          &fault->hva);
-       return false;
-
-out_retry:
-       *r = RET_PF_RETRY;
-       return true;
+       return RET_PF_CONTINUE;
 }
 
 /*
@@ -4001,10 +3992,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
        mmu_seq = vcpu->kvm->mmu_notifier_seq;
        smp_rmb();
 
-       if (kvm_faultin_pfn(vcpu, fault, &r))
+       r = kvm_faultin_pfn(vcpu, fault);
+       if (r != RET_PF_CONTINUE)
                return r;
 
-       if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r))
+       r = handle_abnormal_pfn(vcpu, fault, ACC_ALL);
+       if (r != RET_PF_CONTINUE)
                return r;
 
        r = RET_PF_RETRY;
arch/x86/kvm/mmu/mmu_internal.h +0 -0
index 1bff453f7cbe..c0e502b17ef7 100644

arch/x86/kvm/mmu/mmu_internal.h unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); · line 149

Quoted 7 lines
arch/x86/kvm/mmu/mmu_internal.h +1 -0
@@ -143,6 +143,7 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
 /*
  * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault().
  *
+ * RET_PF_CONTINUE: So far, so good, keep handling the page fault.
  * RET_PF_RETRY: let CPU fault again on the address.
  * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
  * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.

RET_PF_CONTINUE and RET_PF_INVALID are pretty similar, they both indicate to the PF handler that it should keep going. What do you think about taking this a step further and removing RET_PF_INVALID and RET_PF_CONTINUE and using 0 instead? That way we can replace:

if (ret != RET_PF_CONTINUE)

          return r;

and

if (ret != RET_PF_INVALID)

          return r;

with

if (r)

          return r;

?

arch/x86/kvm/mmu/mmu_internal.h unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); · line 154

Quoted 4 lines
arch/x86/kvm/mmu/mmu_internal.h +0 -0
@@ -151,9 +152,15 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
  *
  * Any names added to this enum should be exported to userspace for use in
  * tracepoints via TRACE_DEFINE_ENUM() in mmutrace.h

Please add RET_PF_CONTINUE to mmutrace.h, e.g.

  TRACE_DEFINE_ENUM(RET_PF_CONTINUE);

It doesn't matter in practice (yet) since the trace enums are only used for trace_fast_page_fault, which does not return RET_PF_CONTINUE. But it would be good to keep it up to date.

Quoted 37 lines
arch/x86/kvm/mmu/mmu_internal.h +7 -1
+ *
+ * Note, all values must be greater than or equal to zero so as not to encroach
+ * on -errno return values.  Somewhat arbitrarily use '0' for CONTINUE, which
+ * will allow for efficient machine code when checking for CONTINUE, e.g.
+ * "TEST %rax, %rax, JNZ", as all "stop!" values are non-zero.
  */
 enum {
-       RET_PF_RETRY = 0,
+       RET_PF_CONTINUE = 0,
+       RET_PF_RETRY,
        RET_PF_EMULATE,
        RET_PF_INVALID,
        RET_PF_FIXED,
arch/x86/kvm/mmu/paging_tmpl.h +4 -2
index 66f1acf153c4..7038273d04ab 100644
@@ -838,10 +838,12 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
        mmu_seq = vcpu->kvm->mmu_notifier_seq;
        smp_rmb();
 
-       if (kvm_faultin_pfn(vcpu, fault, &r))
+       r = kvm_faultin_pfn(vcpu, fault);
+       if (r != RET_PF_CONTINUE)
                return r;
 
-       if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r))
+       r = handle_abnormal_pfn(vcpu, fault, walker.pte_access);
+       if (r != RET_PF_CONTINUE)
                return r;
 
        /*

base-commit: 150866cd0ec871c765181d145aa0912628289c8a --
2.36.0.rc0.470.gd361397f0d-goog

author maintainer
#2 lore raw
Quoted 10 lines · David Matlack
Quoted 9 lines · Sean Christopherson
arch/x86/kvm/mmu/mmu_internal.h +1 -0
index 1bff453f7cbe..c0e502b17ef7 100644
@@ -143,6 +143,7 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
 /*
  * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault().
  *
+ * RET_PF_CONTINUE: So far, so good, keep handling the page fault.
Quoted 8 lines

arch/x86/kvm/mmu/mmu_internal.h unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); · line 149

Quoted 3 lines
arch/x86/kvm/mmu/mmu_internal.h +0 -0
  * RET_PF_RETRY: let CPU fault again on the address.
  * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
  * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.

RET_PF_CONTINUE and RET_PF_INVALID are pretty similar, they both indicate to the PF handler that it should keep going. What do you think about taking this a step further and removing RET_PF_INVALID and RET_PF_CONTINUE and using 0 instead?

RET_PF_INVALID is useful because it allows kvm_mmu_page_fault() to sanity check that the page fault handler didn't barf:

		r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
					  lower_32_bits(error_code), false);
		if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
			return -EIO;

It's a bit odd that fast_page_fault() returns RET_PF_INVALID instead of RET_PF_CONTINUE, but at the same time the fault _is_ indeed invalid for fast handling.

Quoted 8 lines

That way we can replace:

if (ret != RET_PF_CONTINUE)

          return r;

and

if (ret != RET_PF_INVALID)

Quoted 8 lines
          return r;

with

if (r)

          return r;

?

We could actually do that now, at least for RET_PF_CONTINUE, but I deliberately avoided doing that so as to not take a hard dependency on the underlying value of RET_PF_CONTINUE. KVM's magic "0 == exit to userspace, 1 == resume guest" logic is dangerous, e.g. it's caused real bugs in the past. But in that case, the return value is multiplexed with -errno, i.e. there's a good reason for using magic values.

For this code, there's no such requirement because the caller must convert the RET_PF_* return to the aforementioned magic returns.

And the even bigger motivation was to discourage "if (r)" for this case because that suggests that this code _does_ utilize KVM's magic return value. I actually wrote it that way at first and then got completely turned around and forgot what value was being returned :-)

Heh, and thinking about it, I'd be tempted to assign RET_PF_CONTINUE a non-zero value if some debug Kconfig is enabled to really enforce that KVM explicitly checks for RET_PF_CONTINUE instead of assuming a non-zero value means "bail".

Quoted 7 lines

arch/x86/kvm/mmu/mmu_internal.h unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); · line 154

Quoted 4 lines
arch/x86/kvm/mmu/mmu_internal.h +0 -0
@@ -151,9 +152,15 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
  *
  * Any names added to this enum should be exported to userspace for use in
  * tracepoints via TRACE_DEFINE_ENUM() in mmutrace.h

Please add RET_PF_CONTINUE to mmutrace.h, e.g.

Quoted 5 lines
  TRACE_DEFINE_ENUM(RET_PF_CONTINUE);

It doesn't matter in practice (yet) since the trace enums are only used for trace_fast_page_fault, which does not return RET_PF_CONTINUE. But it would be good to keep it up to date.

Drat, missed that. Thanks!

#3 lore raw
In reply to Sean Christopherson
Quoted 6 lines · Sean Christopherson

Add RET_PF_CONTINUE and use it in handle_abnormal_pfn() and kvm_faultin_pfn() to signal that the page fault handler should continue doing its thing. Aside from being gross and inefficient, using a boolean return to signal continue vs. stop makes it extremely difficult to add more helpers and/or move existing code to a helper.

Quoted 8 lines

E.g. hypothetically, if nested MMUs were to gain a separate page fault handler in the future, everything up to the "is self-modifying PTE" check can be shared by all shadow MMUs, but communicating up the stack whether to continue on or stop becomes a nightmare.

More concretely, proposed support for private guest memory ran into a similar issue, where it'll be forced to forego a helper in order to yield sane code: https://lore.kernel.org/all/YkJbxiL%[email protected].

Thanks for cooking this patch, it makes private memory patch much easier.

Quoted 159 lines

No functional change intended.

---
 arch/x86/kvm/mmu/mmu.c          | 51 ++++++++++++++-------------------
 arch/x86/kvm/mmu/mmu_internal.h |  9 +++++-
 arch/x86/kvm/mmu/paging_tmpl.h  |  6 ++--
 3 files changed, 34 insertions(+), 32 deletions(-)
arch/x86/kvm/mmu/mmu.c +22 -29
index 69a30d6d1e2b..cb2982c6b513 100644
@@ -2972,14 +2972,12 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
 	return -EFAULT;
 }
 
-static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
-				unsigned int access, int *ret_val)
+static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
+			       unsigned int access)
 {
 	/* The pfn is invalid, report the error! */
-	if (unlikely(is_error_pfn(fault->pfn))) {
-		*ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
-		return true;
-	}
+	if (unlikely(is_error_pfn(fault->pfn)))
+		return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
 
 	if (unlikely(!fault->slot)) {
 		gva_t gva = fault->is_tdp ? 0 : fault->addr;
@@ -2991,13 +2989,11 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa
 		 * touching the shadow page tables as attempting to install an
 		 * MMIO SPTE will just be an expensive nop.
 		 */
-		if (unlikely(!shadow_mmio_value)) {
-			*ret_val = RET_PF_EMULATE;
-			return true;
-		}
+		if (unlikely(!shadow_mmio_value))
+			return RET_PF_EMULATE;
 	}
 
-	return false;
+	return RET_PF_CONTINUE;
 }
 
 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
@@ -3888,7 +3884,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 				  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
 }
 
-static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r)
+static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 	struct kvm_memory_slot *slot = fault->slot;
 	bool async;
@@ -3899,7 +3895,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 	 * be zapped before KVM inserts a new MMIO SPTE for the gfn.
 	 */
 	if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
-		goto out_retry;
+		return RET_PF_RETRY;
 
 	if (!kvm_is_visible_memslot(slot)) {
 		/* Don't expose private memslots to L2. */
@@ -3907,7 +3903,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 			fault->slot = NULL;
 			fault->pfn = KVM_PFN_NOSLOT;
 			fault->map_writable = false;
-			return false;
+			return RET_PF_CONTINUE;
 		}
 		/*
 		 * If the APIC access page exists but is disabled, go directly
@@ -3916,10 +3912,8 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 		 * when the AVIC is re-enabled.
 		 */
 		if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
-		    !kvm_apicv_activated(vcpu->kvm)) {
-			*r = RET_PF_EMULATE;
-			return true;
-		}
+		    !kvm_apicv_activated(vcpu->kvm))
+			return RET_PF_EMULATE;
 	}
 
 	async = false;
@@ -3927,26 +3921,23 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 					  fault->write, &fault->map_writable,
 					  &fault->hva);
 	if (!async)
-		return false; /* *pfn has correct page already */
+		return RET_PF_CONTINUE; /* *pfn has correct page already */
 
 	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
 		trace_kvm_try_async_get_page(fault->addr, fault->gfn);
 		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
 			trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
 			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
-			goto out_retry;
-		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn))
-			goto out_retry;
+			return RET_PF_RETRY;
+		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
+			return RET_PF_RETRY;
+		}
 	}
 
 	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
 					  fault->write, &fault->map_writable,
 					  &fault->hva);
-	return false;
-
-out_retry:
-	*r = RET_PF_RETRY;
-	return true;
+	return RET_PF_CONTINUE;
 }
 
 /*
@@ -4001,10 +3992,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (kvm_faultin_pfn(vcpu, fault, &r))
+	r = kvm_faultin_pfn(vcpu, fault);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
-	if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r))
+	r = handle_abnormal_pfn(vcpu, fault, ACC_ALL);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
 	r = RET_PF_RETRY;
arch/x86/kvm/mmu/mmu_internal.h +4 -0
index 1bff453f7cbe..c0e502b17ef7 100644
@@ -143,6 +143,7 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
 /*
  * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault().
  *
+ * RET_PF_CONTINUE: So far, so good, keep handling the page fault.
  * RET_PF_RETRY: let CPU fault again on the address.
  * RET_PF_EMULATE: mmio page fault, emulate the instruction directly.
  * RET_PF_INVALID: the spte is invalid, let the real page fault path update it.
@@ -151,9 +152,15 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head);
  *
  * Any names added to this enum should be exported to userspace for use in
  * tracepoints via TRACE_DEFINE_ENUM() in mmutrace.h
+ *
+ * Note, all values must be greater than or equal to zero so as not to encroach
+ * on -errno return values.  Somewhat arbitrarily use '0' for CONTINUE, which

arch/x86/kvm/mmu/mmu_internal.h unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); · line 166

Quoted 10 lines
arch/x86/kvm/mmu/mmu_internal.h +4 -1
+ * will allow for efficient machine code when checking for CONTINUE, e.g.
+ * "TEST %rax, %rax, JNZ", as all "stop!" values are non-zero.
  */
 enum {
-	RET_PF_RETRY = 0,
+	RET_PF_CONTINUE = 0,
+	RET_PF_RETRY,
 	RET_PF_EMULATE,
 	RET_PF_INVALID,
 	RET_PF_FIXED,

Unrelated to this patch but related to private memory patch, when implicit conversion happens, I'm thinking which return value I should use. Current -1 does not make much sense since it causes KVM_RUN returns an error while it's expected to be 0. None of the above including RET_PF_CONTINUE seems appropriate. Maybe we should go further to introduce another RET_PF_USER indicating we should exit to userspace for handling with a return value of 0 instead of -error in kvm_mmu_page_fault().

Thanks,
Chao

Quoted 23 lines
arch/x86/kvm/mmu/paging_tmpl.h +4 -2
index 66f1acf153c4..7038273d04ab 100644
@@ -838,10 +838,12 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (kvm_faultin_pfn(vcpu, fault, &r))
+	r = kvm_faultin_pfn(vcpu, fault);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
-	if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r))
+	r = handle_abnormal_pfn(vcpu, fault, walker.pte_access);
+	if (r != RET_PF_CONTINUE)
 		return r;
 
 	/*

base-commit: 150866cd0ec871c765181d145aa0912628289c8a

Signature · 1 line
2.36.0.rc0.470.gd361397f0d-goog