Skip to contentSkip to conversation

x86/ept: batch PML type-changes into single locked region

PATCH In review Roger Pau Monne 2 participants 5 messages Jul 15, 2025 – Jul 16, 2025

lore
Share & tools
lore
#0 lore raw

cover x86/ept: batch PML type-changes into single locked region

Hello,

Following series attempts to batch PML related type-changes into a single locked region, so that the lock is not individually taken for each gfn that requires changing, and to also defer the EPT flush until all type-changes are done. Doing an EPT flush for each gfn changed is extremely wasteful

First patch moves the implementation of vmx_domain_flush_pml_buffers() into the EPT file, so it has access to the mm locking primitives. Patch 2 introduces p2m locking around the type change loop.

PML related code is already split between EPT and VMX, it could certainly do with some cleanup. The patch here doesn't make things better, but I don't think it makes them any worse either.

Thanks, Roger.

Roger Pau Monne (2):
x86/ept: move vmx_domain_flush_pml_buffers() to p2m-ept.c x86/ept: batch PML p2m type-changes into single locked region

 xen/arch/x86/hvm/vmx/vmcs.c             | 59 +---------------------
 xen/arch/x86/hvm/vmx/vmx.c              |  2 +-
 xen/arch/x86/include/asm/hvm/vmx/vmcs.h |  1 -
 xen/arch/x86/include/asm/hvm/vmx/vmx.h  |  1 +
 xen/arch/x86/mm/p2m-ept.c               | 65 +++++++++++++++++++++++++
 5 files changed, 69 insertions(+), 59 deletions(-)
Signature · 1 line
2.49.0
#1 lore raw

PATCH 1/2 x86/ept: move vmx_domain_flush_pml_buffers() to p2m-ept.c

No functional change intended.

---
 xen/arch/x86/hvm/vmx/vmcs.c             | 59 +------------------------
 xen/arch/x86/hvm/vmx/vmx.c              |  2 +-
 xen/arch/x86/include/asm/hvm/vmx/vmcs.h |  1 -
 xen/arch/x86/include/asm/hvm/vmx/vmx.h  |  1 +
 xen/arch/x86/mm/p2m-ept.c               | 56 +++++++++++++++++++++++
 5 files changed, 60 insertions(+), 59 deletions(-)
xen/arch/x86/hvm/vmx/vmcs.c +2 -57
index e7fca4bf33db..ab8b1c87ec0f 100644
@@ -1670,7 +1670,7 @@ void vmx_vcpu_disable_pml(struct vcpu *v)
         return;
 
     /* Make sure we don't lose any logged GPAs. */
-    vmx_vcpu_flush_pml_buffer(v);
+    ept_vcpu_flush_pml_buffer(v);
 
     vmx_vmcs_enter(v);
 
@@ -1684,61 +1684,6 @@ void vmx_vcpu_disable_pml(struct vcpu *v)
     v->arch.hvm.vmx.pml_pg = NULL;
 }
 
-void vmx_vcpu_flush_pml_buffer(struct vcpu *v)
-{
-    uint64_t *pml_buf;
-    unsigned long pml_idx;
-
-    ASSERT((v == current) || (!vcpu_runnable(v) && !v->is_running));
-    ASSERT(vmx_vcpu_pml_enabled(v));
-
-    vmx_vmcs_enter(v);
-
-    __vmread(GUEST_PML_INDEX, &pml_idx);
-
-    /* Do nothing if PML buffer is empty. */
-    if ( pml_idx == (NR_PML_ENTRIES - 1) )
-        goto out;
-
-    pml_buf = __map_domain_page(v->arch.hvm.vmx.pml_pg);
-
-    /*
-     * PML index can be either 2^16-1 (buffer is full), or 0 ~ NR_PML_ENTRIES-1
-     * (buffer is not full), and in latter case PML index always points to next
-     * available entity.
-     */
-    if ( pml_idx >= NR_PML_ENTRIES )
-        pml_idx = 0;
-    else
-        pml_idx++;
-
-    for ( ; pml_idx < NR_PML_ENTRIES; pml_idx++ )
-    {
-        unsigned long gfn = pml_buf[pml_idx] >> PAGE_SHIFT;
-
-        /*
-         * Need to change type from log-dirty to normal memory for logged GFN.
-         * hap_track_dirty_vram depends on it to work. And we mark all logged
-         * GFNs to be dirty, as we cannot be sure whether it's safe to ignore
-         * GFNs on which p2m_change_type_one returns failure. The failure cases
-         * are very rare, and additional cost is negligible, but a missing mark
-         * is extremely difficult to debug.
-         */
-        p2m_change_type_one(v->domain, gfn, p2m_ram_logdirty, p2m_ram_rw);
-
-        /* HVM guest: pfn == gfn */
-        paging_mark_pfn_dirty(v->domain, _pfn(gfn));
-    }
-
-    unmap_domain_page(pml_buf);
-
-    /* Reset PML index */
-    __vmwrite(GUEST_PML_INDEX, NR_PML_ENTRIES - 1);
-
- out:
-    vmx_vmcs_exit(v);
-}
-
 bool vmx_domain_pml_enabled(const struct domain *d)
 {
     return d->arch.hvm.vmx.status & VMX_DOMAIN_PML_ENABLED;
@@ -1812,7 +1757,7 @@ void vmx_domain_flush_pml_buffers(struct domain *d)
         return;
 
     for_each_vcpu ( d, v )
-        vmx_vcpu_flush_pml_buffer(v);
+        ept_vcpu_flush_pml_buffer(v);
 }
 
 static void vmx_vcpu_update_eptp(struct vcpu *v, u64 eptp)
xen/arch/x86/hvm/vmx/vmx.c +1 -1
index 218cb2c1af84..ce538668c737 100644
@@ -4787,7 +4787,7 @@ void asmlinkage vmx_vmexit_handler(struct cpu_user_regs *regs)
              !(idtv_info & INTR_INFO_VALID_MASK) )
             undo_nmis_unblocked_by_iret();
 
-        vmx_vcpu_flush_pml_buffer(v);
+        ept_vcpu_flush_pml_buffer(v);
         break;
 
     case EXIT_REASON_XSAVES:
xen/arch/x86/include/asm/hvm/vmx/vmcs.h +0 -1
index ff5dd66b0ad9..ecd91389302c 100644
@@ -734,7 +734,6 @@ DECLARE_PER_CPU(bool, vmxon);
 bool vmx_vcpu_pml_enabled(const struct vcpu *v);
 int vmx_vcpu_enable_pml(struct vcpu *v);
 void vmx_vcpu_disable_pml(struct vcpu *v);
-void vmx_vcpu_flush_pml_buffer(struct vcpu *v);
 bool vmx_domain_pml_enabled(const struct domain *d);
 int vmx_domain_enable_pml(struct domain *d);
 void vmx_domain_disable_pml(struct domain *d);
xen/arch/x86/include/asm/hvm/vmx/vmx.h +1 -0
index 56bea252cc5a..da04752e1752 100644
@@ -491,6 +491,7 @@ void ept_walk_table(struct domain *d, unsigned long gfn);
 bool ept_handle_misconfig(uint64_t gpa);
 int epte_get_entry_emt(struct domain *d, gfn_t gfn, mfn_t mfn,
                        unsigned int order, bool *ipat, p2m_type_t type);
+void ept_vcpu_flush_pml_buffer(struct vcpu *v);
 void setup_ept_dump(void);
 /* Locate an alternate p2m by its EPTP */
 unsigned int p2m_find_altp2m_by_eptp(struct domain *d, uint64_t eptp);
xen/arch/x86/mm/p2m-ept.c +56 -0

62 lines not on this page · raw message

Signature · 1 line
2.49.0
#3 lore raw

PATCH 2/2 x86/ept: batch PML p2m type-changes into single locked region

In reply to Roger Pau Monne

The current p2m type-change loop in ept_vcpu_flush_pml_buffer() relies on each call to p2m_change_type_one() taking the p2m lock, doing the change and then dropping the lock and flushing the p2m. Instead take the p2m lock outside of the loop, so that calls to gfn_{,un}lock() inside p2m_change_type_one() just take the p2m lock recursively, and more importantly, the flush is deferred until the p2m is unlocked in ept_vcpu_flush_pml_buffer().

No functional change intended in the end result of ept_vcpu_flush_pml_buffer(), however a possibly noticeable performance improvement is expected.

---
 xen/arch/x86/mm/p2m-ept.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
xen/arch/x86/mm/p2m-ept.c +11 -2
index 015911ba6c80..62fc8e50689f 100644
@@ -1375,6 +1375,8 @@ static void cf_check ept_flush_pml_buffers(struct p2m_domain *p2m)
 
 void ept_vcpu_flush_pml_buffer(struct vcpu *v)
 {
+    struct domain *d = v->domain;
+    struct p2m_domain *p2m = p2m_get_hostp2m(d);
     uint64_t *pml_buf;
     unsigned long pml_idx;
 
@@ -1401,6 +1403,12 @@ void ept_vcpu_flush_pml_buffer(struct vcpu *v)
     else
         pml_idx++;
 
+    /*
+     * Take the lock outside of the loop, so all the type changes are done
+     * inside of the same locked region and the EPT flush is deferred until the
+     * end of the loop.
+     */
+    p2m_lock(p2m);
     for ( ; pml_idx < NR_PML_ENTRIES; pml_idx++ )
     {
         unsigned long gfn = pml_buf[pml_idx] >> PAGE_SHIFT;
@@ -1413,11 +1421,12 @@ void ept_vcpu_flush_pml_buffer(struct vcpu *v)
          * are very rare, and additional cost is negligible, but a missing mark
          * is extremely difficult to debug.
          */
-        p2m_change_type_one(v->domain, gfn, p2m_ram_logdirty, p2m_ram_rw);
+        p2m_change_type_one(d, gfn, p2m_ram_logdirty, p2m_ram_rw);
 
         /* HVM guest: pfn == gfn */
-        paging_mark_pfn_dirty(v->domain, _pfn(gfn));
+        paging_mark_pfn_dirty(d, _pfn(gfn));
     }
+    p2m_unlock(p2m);
 
     unmap_domain_page(pml_buf);
 
Signature · 1 line
2.49.0
Reviewed-by on v1 2/2
#4 lore raw

Re: [PATCH 2/2] x86/ept: batch PML p2m type-changes into single locked region

Quoted 8 lines · Roger Pau Monne

The current p2m type-change loop in ept_vcpu_flush_pml_buffer() relies on each call to p2m_change_type_one() taking the p2m lock, doing the change and then dropping the lock and flushing the p2m. Instead take the p2m lock outside of the loop, so that calls to gfn_{,un}lock() inside p2m_change_type_one() just take the p2m lock recursively, and more importantly, the flush is deferred until the p2m is unlocked in ept_vcpu_flush_pml_buffer().

Quoted 5 lines

No functional change intended in the end result of ept_vcpu_flush_pml_buffer(), however a possibly noticeable performance improvement is expected.