TDX: Dynamic PAMT + S-EPT Hugepage
Subsystems (6)
Sean Christopherson
maintainerRFC PATCH v5 44/45 KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
In reply to Sean ChristophersonAdd support for splitting S-EPT hugepages in preparation for converting a subset of a hugepage to be shared, as KVM must precisely zap/remove S-EPT entries to avoid clobbering guest memory (the lifetime of guest private memory is tied to the S-EPT). I.e. KVM needs to first split a hugepage so that only the to-be-converted small pages can be zapped.
To avoid unnecessary work, e.g. if only the tail/end page of massive region isn't aligned to the conversion, explicitly detect unaligned head and tail pages relative to the max page size support by KVM, i.e. head/tail pages that will undergo partial conversion.
To support splitting an S-EPT hugepage without a vCPU, add a per-VM PAMT cache, along with a mutex to guard the cache. Using a mutex, e.g. versus a spinlock, is important at it allows KVM to allocate memory *without* dropping the lock, i.e. so that the PAMT cache can be topped-up as needed without needed to juggle arch.tdp_mmu_external_cache_lock.
--- arch/x86/include/asm/kvm_host.h | 8 +++- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/tdp_mmu.c | 72 +++++++++++++++++++++++++++++++-- arch/x86/kvm/vmx/tdx.c | 34 +++++++++++++--- arch/x86/kvm/vmx/tdx.h | 2 + 5 files changed, 107 insertions(+), 11 deletions(-)
arch/x86/include/asm/kvm_host.h +7 -1
@@ -1563,6 +1563,12 @@ struct kvm_arch {* the code to do so. */ spinlock_t tdp_mmu_pages_lock; + +/* +* Protect the per-VM cache of pre-allocate pages used to populate the +* Dynamic PAMT when splitting S-EPT huge pages without a vCPU. +*/ +struct mutex tdp_mmu_external_cache_lock;#endif /* CONFIG_X86_64 */ /*
@@ -1861,7 +1867,7 @@ struct kvm_x86_ops { u64 new_spte, enum pg_level level);void (*reclaim_external_sp)(struct kvm *kvm,gfn_t gfn,struct kvm_mmu_page *sp); -int (*topup_external_cache)(struct kvm_vcpu *vcpu,int min); +int (*topup_external_cache)(struct kvm *kvm,struct kvm_vcpu *vcpu,int min);bool (*has_wbinvd_exit)(void );
arch/x86/kvm/mmu/mmu.c +1 -1
@@ -606,7 +606,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)if (r)return r; - r =kvm_x86_call (topup_external_cache)(vcpu,PT64_ROOT_MAX_LEVEL ); + r =kvm_x86_call (topup_external_cache)(vcpu->kvm, vcpu,PT64_ROOT_MAX_LEVEL );if (r)return r; }
arch/x86/kvm/mmu/tdp_mmu.c +69 -3
99 lines not on this page · raw message
arch/x86/kvm/vmx/tdx.c +28 -6
@@ -607,6 +607,8 @@ void tdx_vm_destroy(struct kvm *kvm) {struct kvm_tdx *kvm_tdx =to_kvm_tdx (kvm); +tdx_free_pamt_cache (&kvm_tdx->pamt_cache); +tdx_reclaim_td_control_pages (kvm); kvm_tdx->state =TD_STATE_UNINITIALIZED ;
@@ -629,6 +631,8 @@ int tdx_vm_init(struct kvm *kvm) {struct kvm_tdx *kvm_tdx =to_kvm_tdx (kvm); +tdx_init_pamt_cache (&kvm_tdx->pamt_cache); + kvm->arch.has_protected_state = true;/* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap,
@@ -1621,15 +1625,32 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)td_vmcs_write64 (to_tdx (vcpu),SHARED_EPT_POINTER , root_hpa); } -static int tdx_topup_external_pamt_cache (struct kvm_vcpu *vcpu,int min) +static struct tdx_pamt_cache *tdx_get_pamt_cache (struct kvm *kvm, +struct kvm_vcpu *vcpu) { +if (KVM_BUG_ON (vcpu && vcpu->kvm != kvm, kvm)) +return NULL ; + +if (vcpu) +return &to_tdx (vcpu)->pamt_cache; + +lockdep_assert_held (&kvm->arch.tdp_mmu_external_cache_lock); +return &to_kvm_tdx (kvm)->pamt_cache; +} + +static int tdx_topup_external_pamt_cache (struct kvm *kvm, +struct kvm_vcpu *vcpu,int min) +{ +struct tdx_pamt_cache *pamt_cache; +if (!tdx_supports_dynamic_pamt (tdx_sysinfo))return 0 ; -if (WARN_ON_ONCE (!vcpu)) + pamt_cache =tdx_get_pamt_cache (kvm, vcpu); +if (!pamt_cache)return -EIO ; -return tdx_topup_pamt_cache (&to_tdx (vcpu)->pamt_cache, min); +return tdx_topup_pamt_cache (pamt_cache, min); } static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
@@ -1792,8 +1813,8 @@ static struct page *tdx_spte_to_external_spt(struct kvm *kvm, gfn_t gfn,static int tdx_sept_split_private_spte (struct kvm *kvm,gfn_t gfn,u64 old_spte,u64 new_spte,enum pg_level level) { -struct kvm_vcpu *vcpu =kvm_get_running_vcpu ();struct kvm_tdx *kvm_tdx =to_kvm_tdx (kvm); +struct tdx_pamt_cache *pamt_cache;gpa_t gpa =gfn_to_gpa (gfn);u64 err, entry, level_state;struct page *external_spt;
@@ -1804,7 +1825,8 @@ static int tdx_sept_split_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,if (!external_spt)return -EIO ; -if (KVM_BUG_ON (!vcpu || vcpu->kvm != kvm, kvm)) + pamt_cache =tdx_get_pamt_cache (kvm,kvm_get_running_vcpu ()); +if (!pamt_cache)return -EIO ; err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa,
@@ -1816,7 +1838,7 @@ static int tdx_sept_split_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, err =tdh_do_no_vcpus (tdh_mem_page_demote, kvm, &kvm_tdx->td, gpa, level,spte_to_pfn (old_spte), external_spt, - &to_tdx (vcpu)->pamt_cache, &entry, &level_state); + pamt_cache, &entry, &level_state);if (TDX_BUG_ON_2 (err,TDH_MEM_PAGE_DEMOTE , entry, level_state, kvm))return -EIO ;
arch/x86/kvm/vmx/tdx.h +2 -0
@@ -48,6 +48,8 @@ struct kvm_tdx {* Set/unset is protected with kvm->mmu_lock. */ bool wait_for_sept_zap; + +struct tdx_pamt_cache pamt_cache; };/* TDX module vCPU states */
Signature · 1 line
2.53.0.rc1.217.geba53bf80e-goog
Sean Christopherson
maintainerRe: [RFC PATCH v5 44/45] KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
Quoted 57 lines · Sean Christopherson
#ifdef
CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT(quoted diff) +53
-0+static int __tdp_mmu_split_mirror_huge_pages(struct kvm *kvm, + struct kvm_mmu_page *root, + gfn_t gfn, int target_level) +{ + gfn_t end = gfn + KVM_PAGES_PER_HPAGE(target_level + 1); + + return tdp_mmu_split_huge_pages_root(kvm, root, gfn, end, target_level, false); +} + +static int tdp_mmu_split_mirror_huge_pages(struct kvm *kvm, + struct kvm_mmu_page *root, + gfn_t start, gfn_t end, int level) +{ + + gfn_t head = gfn_round_for_level(start, level + 1); + gfn_t tail = gfn_round_for_level(end, level + 1); + int r; + + if (head != start) { + r = __tdp_mmu_split_mirror_huge_pages(kvm, root, head, level); + if (r) + return r; + } + + if (tail != end && (head != tail || head == start)) { + r = __tdp_mmu_split_mirror_huge_pages(kvm, root, tail, level); + if (r) + return r; + } + + return 0; +} + int kvm_arch_gmem_convert(struct kvm *kvm, gfn_t start, gfn_t end, bool to_private) { + struct kvm_mmu_page *root; + int r; + + /* + * When converting from private=>shared, KVM must first split potential + * hugepages, as KVM mustn't overzap private mappings for TDX guests, + * i.e. must zap _exactly_ [start, end). Split potential hugepages at + * the head and tail of the to-be-converted (and thus zapped) range so + * that KVM doesn't overzap due to dropping a hugepage that doesn't + * fall wholly inside the range. + */ + if (to_private || !kvm_has_mirrored_tdp(kvm)) + return 0; + + /* + * Acquire the external cache lock, a.k.a. the Dynamic PAMT lock, to + * protect the per-VM cache of pre-allocate pages used to populate the + * Dynamic PAMT when splitting S-EPT huge pages. + */ + guard(mutex)(&kvm->arch.tdp_mmu_external_cache_lock);
Review
Quoted 10 lines
(quoted diff) +10
-0+ + guard(write_lock)(&kvm->mmu_lock); + + /* + * TODO: Also split from PG_LEVEL_1G => PG_LEVEL_2M when KVM supports + * 1GiB S-EPT pages. + */ + __for_each_tdp_mmu_root_yield_safe(kvm, root, 0, KVM_MIRROR_ROOTS) { + r = tdp_mmu_split_mirror_huge_pages(kvm, root, start, end, PG_LEVEL_4K); + if (r)
This needs to call kvm_tdp_mmu_put_root() on failure. But if we instead add kvm_tdp_mmu_mirrors_split_huge_pages() for use in handling mismatched ACCEPT, this code goes away.
And then the bulk of this code can live in tdx.c instead of tdp_mmu.c, and the pamt mutex can live in kvm_tdx instead of kvm_arch.
Compile tested only...
---
From: Sean Christopherson <[email protected]>
Date: Thu, 22 Jan 2026 07:36:47 -0800
Subject: [PATCH] KVM: x86/mmu: Add support for splitting S-EPT hugepages on
conversion
Add support for splitting S-EPT hugepages in preparation for converting a subset of a hugepage to be shared, as KVM must precisely zap/remove S-EPT entries to avoid clobbering guest memory (the lifetime of guest private memory is tied to the S-EPT). I.e. KVM needs to first split a hugepage so that only the to-be-converted small pages can be zapped.
To avoid unnecessary work, e.g. if only the tail/end page of massive region isn't aligned to the conversion, explicitly detect unaligned head and tail pages relative to the max page size support by KVM, i.e. head/tail pages that will undergo partial conversion.
To support splitting an S-EPT hugepage without a vCPU, add a per-VM PAMT cache, along with a mutex to guard the cache. Using a mutex, e.g. versus a spinlock, is important at it allows KVM to allocate memory *without* dropping the lock, i.e. so that the PAMT cache can be topped-up as needed without needed to juggle arch.tdp_mmu_external_cache_lock.
--- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 3 +- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/tdp_mmu.c | 7 ++- arch/x86/kvm/vmx/tdx.c | 96 ++++++++++++++++++++++++++++-- arch/x86/kvm/vmx/tdx.h | 3 + arch/x86/kvm/x86.c | 2 +- 7 files changed, 102 insertions(+), 12 deletions(-)
arch/x86/include/asm/kvm-x86-ops.h +1 -0
@@ -149,6 +149,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) KVM_X86_OP_OPTIONAL(gmem_invalidate) +KVM_X86_OP_OPTIONAL_RET0(gmem_convert)#undef KVM_X86_OP#undef KVM_X86_OP_OPTIONAL
arch/x86/include/asm/kvm_host.h +2 -1
@@ -1861,7 +1861,7 @@ struct kvm_x86_ops { u64 new_spte, enum pg_level level);void (*reclaim_external_sp)(struct kvm *kvm,gfn_t gfn,struct kvm_mmu_page *sp); -int (*topup_external_cache)(struct kvm_vcpu *vcpu,int min); +int (*topup_external_cache)(struct kvm *kvm,struct kvm_vcpu *vcpu,int min);bool (*has_wbinvd_exit)(void );
@@ -1950,6 +1950,7 @@ struct kvm_x86_ops {void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);int (*gmem_prepare)(struct kvm *kvm,kvm_pfn_t pfn,gfn_t gfn,int max_order);void (*gmem_invalidate)(kvm_pfn_t start,kvm_pfn_t end); +int (*gmem_convert)(struct kvm *kvm,gfn_t start,gfn_t end,bool to_private);int (*gmem_max_mapping_level)(struct kvm *kvm,kvm_pfn_t pfn,bool is_private); };
arch/x86/kvm/mmu/mmu.c +1 -1
@@ -606,7 +606,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)if (r)return r; - r =kvm_x86_call (topup_external_cache)(vcpu,PT64_ROOT_MAX_LEVEL ); + r =kvm_x86_call (topup_external_cache)(vcpu->kvm, vcpu,PT64_ROOT_MAX_LEVEL );if (r)return r; }
arch/x86/kvm/mmu/tdp_mmu.c +4 -3
@@ -1447,7 +1447,8 @@ bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,return spte_set; } -static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split (struct tdp_iter *iter) +static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split (struct kvm *kvm, +struct tdp_iter *iter) {struct kvm_mmu_page *sp;
@@ -1464,7 +1465,7 @@ static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct tdp_iter *iter)if (!sp->external_spt) goto err_external_spt; -if (kvm_x86_call (topup_external_cache)(kvm_get_running_vcpu (),1 )) +if (kvm_x86_call (topup_external_cache)(kvm,kvm_get_running_vcpu (),1 )) goto err_external_split; }
@@ -1556,7 +1557,7 @@ static int tdp_mmu_split_huge_pages_root(struct kvm *kvm,else write_unlock (&kvm->mmu_lock); - sp =tdp_mmu_alloc_sp_for_split (&iter); + sp =tdp_mmu_alloc_sp_for_split (kvm, &iter);if (shared)read_lock (&kvm->mmu_lock);
arch/x86/kvm/vmx/tdx.c +90 -6
151 lines not on this page · raw message
arch/x86/kvm/vmx/tdx.h +3 -0
@@ -48,6 +48,9 @@ struct kvm_tdx {* Set/unset is protected with kvm->mmu_lock. */ bool wait_for_sept_zap; + +struct tdx_pamt_cache pamt_cache; +struct mutex pamt_cache_lock; };/* TDX module vCPU states */
arch/x86/kvm/x86.c +1 -1
@@ -14061,7 +14061,7 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)int kvm_arch_gmem_convert (struct kvm *kvm,gfn_t start,gfn_t end,bool to_private) { -return 0 ; +return kvm_x86_call (gmem_convert)(kvm, start, end, to_private); }#endif #endif
base-commit: b2791d61e9774d8575525816e864d2e09ee9090a --
Re: [RFC PATCH v5 44/45] KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
Quoted 1 line · Sean Christopherson
Compile tested only...
It passed my local tests with the fix [1].
[1] https://lore.kernel.org/all/[email protected].
arch/x86/include/asm/kvm_host.h struct kvm_x86_ops { · line 1955
Quoted 7 lines
arch/x86/include/asm/kvm_host.h +1
-0@@ -1950,6 +1950,7 @@ struct kvm_x86_ops {void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);int (*gmem_prepare)(struct kvm *kvm,kvm_pfn_t pfn,gfn_t gfn,int max_order);void (*gmem_invalidate)(kvm_pfn_t start,kvm_pfn_t end); +int (*gmem_convert)(struct kvm *kvm,gfn_t start,gfn_t end,bool to_private);int (*gmem_max_mapping_level)(struct kvm *kvm,kvm_pfn_t pfn,bool is_private); };
Since tdx_gmem_convert() only performs S-EPT splitting on the specified range, would it make sense to rename the op gmem_convert() to something like gmem_split_private_mapping()?
(This would also involve renamingkvm_gmem_convert() to kvm_gmem_split_private_mapping(), and kvm_arch_gmem_convert() to kvm_arch_gmem_split_private_mapping()).
This way, it's natural for it to be called by kvm_gmem_set_attributes() for private-to-shared conversions, kvm_gmem_punch_hole(), or kvm_gmem_error_folio().
Quoted 10 lines
arch/x86/kvm/vmx/tdx.c +10
-0+static int tdx_gmem_convert (struct kvm *kvm,gfn_t start,gfn_t end, +bool to_private) +{ +/* +* When converting from private=>shared, KVM must first split potential +* hugepages, as KVM mustn't overzap private mappings for TDX guests, +* i.e. must zap _exactly_ [start, end). Split potential hugepages at +* the head and tail of the to-be-converted (and thus zapped) range so +* that KVM doesn't overzap due to dropping a hugepage that doesn't +* fall wholly inside the range.
arch/x86/kvm/vmx/tdx.c static int tdx_map_gpa(struct kvm_vcpu *vcpu) · line 1341
Quoted 10 lines
arch/x86/kvm/vmx/tdx.c +10
-0+*/ +if (to_private || !kvm_has_mirrored_tdp (kvm)) +return 0 ; + +/* +* Acquire the external cache lock, a.k.a. the Dynamic PAMT lock, to +* protect the per-VM cache of pre-allocate pages used to populate the +* Dynamic PAMT when splitting S-EPT huge pages. +*/ +guard (mutex)(&to_kvm_tdx (kvm)->pamt_cache_lock);
Thanks for the change from spinlock to mutex, which is a smart approach that eliminates the need to release the lock for topup.
However, I have a question about kvm_tdp_mmu_try_split_huge_pages(), which is called by dirty page tracking related functions. I'm not sure if we might want to invoke them from a non-vCPU thread for mirror roots in the future. If that's the case, would they need some way to acquire this lock?
Quoted 39 lines
arch/x86/kvm/vmx/tdx.c +9
-0+guard (write_lock)(&kvm->mmu_lock); + +/* +* TODO: Also split from PG_LEVEL_1G => PG_LEVEL_2M when KVM supports +* 1GiB S-EPT pages. +*/ +return tdx_sept_split_huge_pages (kvm, start, end,PG_LEVEL_4K ); +} +arch/x86/kvm/vmx/tdx.h +3
-0@@ -48,6 +48,9 @@ struct kvm_tdx {* Set/unset is protected with kvm->mmu_lock. */ bool wait_for_sept_zap; + +struct tdx_pamt_cache pamt_cache; +struct mutex pamt_cache_lock; };/* TDX module vCPU states */ arch/x86/kvm/x86.c +1
-1@@ -14061,7 +14061,7 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)int kvm_arch_gmem_convert (struct kvm *kvm,gfn_t start,gfn_t end,bool to_private) { -return 0 ; +return kvm_x86_call (gmem_convert)(kvm, start, end, to_private); }#endif #endif base-commit:
b2791d61e9774d8575525816e864d2e09ee9090a--
Sean Christopherson
maintainerRe: [RFC PATCH v5 44/45] KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
Quoted 4 lines · Yan Zhao
However, I have a question about
kvm_tdp_mmu_try_split_huge_pages(), which is called by dirty page tracking related functions. I'm not sure if we might want to invoke them from a non-vCPU thread for mirror roots in the future. If that's the case, would they need some way to acquire this lock?
More than likely, yes. But that's a far future problem, at least as far as upstream is concerned. I.e. I don't want to plan _that_ far ahead in terms of writing code to avoid churn.
Re: [RFC PATCH v5 44/45] KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
In reply to Sean ChristophersonQuoted 31 lines · Sean Christopherson
Add support for splitting S-EPT hugepages in preparation for converting a subset of a hugepage to be shared, as KVM must precisely zap/remove S-EPT entries to avoid clobbering guest memory (the lifetime of guest private memory is tied to the S-EPT). I.e. KVM needs to first split a hugepage so that only the to-be-converted small pages can be zapped.
To avoid unnecessary work, e.g. if only the tail/end page of massive region isn't aligned to the conversion, explicitly detect unaligned head and tail pages relative to the max page size support by KVM, i.e. head/tail pages that will undergo partial conversion.
To support splitting an S-EPT hugepage without a vCPU, add a per-VM PAMT cache, along with a mutex to guard the cache. Using a mutex, e.g. versus a spinlock, is important at it allows KVM to allocate memory *without* dropping the lock, i.e. so that the PAMT cache can be topped-up as needed without needed to juggle arch.tdp_mmu_external_cache_lock.
Signed-off-by Sean Christopherson <[email protected]>--- arch/x86/include/asm/kvm_host.h | 8 +++- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/tdp_mmu.c | 72 +++++++++++++++++++++++++++++++-- arch/x86/kvm/vmx/tdx.c | 34 +++++++++++++--- arch/x86/kvm/vmx/tdx.h | 2 + 5 files changed, 107 insertions(+), 11 deletions(-)arch/x86/include/asm/kvm_host.h +0
-0
arch/x86/include/asm/kvm_host.h struct kvm_arch { · line 1571
Quoted 9 lines
arch/x86/include/asm/kvm_host.h +6
-0@@ -1563,6 +1563,12 @@ struct kvm_arch {* the code to do so. */ spinlock_t tdp_mmu_pages_lock; + +/* +* Protect the per-VM cache of pre-allocate pages used to populate the +* Dynamic PAMT when splitting S-EPT huge pages without a vCPU. +*/ +struct mutex tdp_mmu_external_cache_lock;
Missing "spin_lock_init(&kvm->arch.tdp_mmu_external_cache_lock);" in kvm_mmu_init_tdp_mmu().
Will check the patch you replied next week.
Quoted 1 line
arch/x86/include/asm/kvm_host.h +0
-0#endif /* CONFIG_X86_64 */
Sean Christopherson
maintainerRe: [RFC PATCH v5 44/45] KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversion
Quoted 44 lines · Yan Zhao
Quoted 31 lines · Sean Christopherson
Add support for splitting S-EPT hugepages in preparation for converting a subset of a hugepage to be shared, as KVM must precisely zap/remove S-EPT entries to avoid clobbering guest memory (the lifetime of guest private memory is tied to the S-EPT). I.e. KVM needs to first split a hugepage so that only the to-be-converted small pages can be zapped.
To avoid unnecessary work, e.g. if only the tail/end page of massive region isn't aligned to the conversion, explicitly detect unaligned head and tail pages relative to the max page size support by KVM, i.e. head/tail pages that will undergo partial conversion.
To support splitting an S-EPT hugepage without a vCPU, add a per-VM PAMT cache, along with a mutex to guard the cache. Using a mutex, e.g. versus a spinlock, is important at it allows KVM to allocate memory *without* dropping the lock, i.e. so that the PAMT cache can be topped-up as needed without needed to juggle arch.tdp_mmu_external_cache_lock.
Signed-off-by Sean Christopherson <[email protected]>--- arch/x86/include/asm/kvm_host.h | 8 +++- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/tdp_mmu.c | 72 +++++++++++++++++++++++++++++++-- arch/x86/kvm/vmx/tdx.c | 34 +++++++++++++--- arch/x86/kvm/vmx/tdx.h | 2 + 5 files changed, 107 insertions(+), 11 deletions(-)arch/x86/include/asm/kvm_host.h +0
-0arch/x86/include/asm/kvm_host.h struct kvm_arch { · line 1571
Quoted 9 lines
arch/x86/include/asm/kvm_host.h +6
-0@@ -1563,6 +1563,12 @@ struct kvm_arch {* the code to do so. */ spinlock_t tdp_mmu_pages_lock; + +/* +* Protect the per-VM cache of pre-allocate pages used to populate the +* Dynamic PAMT when splitting S-EPT huge pages without a vCPU. +*/ +struct mutex tdp_mmu_external_cache_lock;Missing "
spin_lock_init(&kvm->arch.tdp_mmu_external_cache_lock);" inkvm_mmu_init_tdp_mmu().
Quoted 1 line
Will check the patch you replied next week.
It has the same bug. FWIW, I found and fixed the bug on our internal branch, but I either missed the fixup when synchronizing back to the upstream branch, or I found the issue after posting.
@@ -634,6 +642,7 @@ int tdx_vm_init(struct kvm *kvm)
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
tdx_init_pamt_cache(&kvm_tdx->pamt_cache);+ mutex_init(&kvm_tdx->pamt_cache_lock);
kvm->arch.has_protected_state = true;
/*Sean Christopherson
maintainerRFC PATCH v5 45/45 KVM: TDX: Turn on PG_LEVEL_2M
In reply to Sean ChristophersonFrom: Yan Zhao <[email protected]>
Turn on PG_LEVEL_2M in tdx_gmem_private_max_mapping_level() when TDX huge page is enabled and TD is RUNNABLE.
Introduce a module parameter named "tdx_huge_page" for kvm-intel.ko to enable/disable TDX huge page. Turn TDX huge page off if the TDX module does not support TDX_FEATURES0.ENHANCED_DEMOTE_INTERRUPTIBILITY.
Force page size to 4KB during TD build time to simplify code design, since
- tdh_mem_page_add() only adds private pages at 4KB.
- The amount of initial memory pages is usually limited (e.g. ~4MB in a typical linux TD).
Update the warnings and KVM_BUG_ON() info to match the conditions when 2MB mappings are permitted.
--- arch/x86/kvm/vmx/tdx.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-)
arch/x86/kvm/vmx/tdx.c +30 -7
@@ -55,6 +55,8 @@bool enable_tdx__ro_after_init ;module_param_named (tdx, enable_tdx, bool,0444 ); +static bool __read_mostly enable_tdx_huge_page = true; +module_param_named (tdx_huge_page, enable_tdx_huge_page, bool,0444 );#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51))#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47))
@@ -1703,8 +1705,9 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, u64 new_spte,kvm_pfn_t pfn =spte_to_pfn (new_spte);int ret; -/* TODO: handle large pages. */ -if (KVM_BUG_ON (level !=PG_LEVEL_4K , kvm)) +/* TODO: Support hugepages when building the initial TD image. */ +if (KVM_BUG_ON (level !=PG_LEVEL_4K && +to_kvm_tdx (kvm)->state !=TD_STATE_RUNNABLE , kvm))return -EIO ;if (KVM_BUG_ON (!vcpu, kvm))
@@ -1885,10 +1888,6 @@ static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,if (KVM_BUG_ON (!is_hkid_assigned (to_kvm_tdx (kvm)), kvm))return -EIO ; -/* TODO: handle large pages. */ -if (KVM_BUG_ON (level !=PG_LEVEL_4K , kvm)) -return -EIO ; - err =tdh_do_no_vcpus (tdh_mem_range_block, kvm, &kvm_tdx->td, gpa, level, &entry, &level_state);if (TDX_BUG_ON_2 (err,TDH_MEM_RANGE_BLOCK , entry, level_state, kvm))
@@ -3474,12 +3473,34 @@ int tdx_vcpu_ioctl(struct kvm_vcpu *vcpu, void __user *argp)return ret; } +/* +* For private pages: +* +* Force KVM to map at 4KB level when !enable_tdx_huge_page (e.g., due to +* incompatible TDX module) or before TD state is RUNNABLE. +* +* Always allow KVM to map at 2MB level in other cases, though KVM may still map +* the page at 4KB (i.e., passing in PG_LEVEL_4K to AUG) due to +* (1) the backend folio is 4KB, +* (2) disallow_lpage restrictions: +* - mixed private/shared pages in the 2MB range +* - level misalignment due to slot base_gfn, slot size, and ugfn +* - guest_inhibit bit set due to guest's 4KB accept level +* (3) page merging is disallowed (e.g., when part of a 2MB range has been +* mapped at 4KB level during TD build time). +*/ int tdx_gmem_max_mapping_level (struct kvm *kvm,kvm_pfn_t pfn,bool is_private) {if (!is_private)return 0 ; -return PG_LEVEL_4K ; +if (!enable_tdx_huge_page) +return PG_LEVEL_4K ; + +if (unlikely (to_kvm_tdx (kvm)->state !=TD_STATE_RUNNABLE )) +return PG_LEVEL_4K ; + +return PG_LEVEL_2M ; }static int tdx_online_cpu (unsigned int cpu)
@@ -3665,6 +3686,8 @@ static int __init __tdx_bringup(void)if (misc_cg_set_capacity (MISC_CG_RES_TDX ,tdx_get_nr_guest_keyids ())) goto get_sysinfo_err; +if (enable_tdx_huge_page && !tdx_supports_demote_nointerrupt (tdx_sysinfo)) + enable_tdx_huge_page = false;/* * Leave hardware virtualization enabled after TDX is enabled * successfully. TDX CPU hotplug depends on this.
Signature · 1 line
2.53.0.rc1.217.geba53bf80e-goog
Quoted 5 lines · Sean Christopherson
This is a combined series of Dynamic PAMT (from Rick), and S-EPT hugepage support (from Yan). Except for some last minute tweaks to the DPAMT array args stuff, a version of this based on a Google-internal kernel has been moderately well tested (thanks Vishal!). But overall it's still firmly RFC as I have deliberately NOT addressed others feedback from v4 of DPAMT and v3
What does PAMT stand for? Is there a design document somewhere?
Quoted 11 lines
of S-EPT hugepage (mostly lack of cycles), and there's at least one patch in here that shouldn't be merged as-is (the quick-and-dirty switch from struct page to raw pfns).
My immediate goal is to solidify the designs for DPAMT and S-EPT hugepage. Given the substantial design changes I am proposing, posting an end-to-end RFC seemed like a much better method than trying to communicate my thoughts piecemeal.
As for landing these series, I think the fastest overall approach would be to land patches 1-4 asap (tangentially related cleanups and fixes), agree
Should they be split out as non-RFC then?
Quoted 9 lines
on a design (hopefully), and then hand control back to Rick and Yan to polish their respective series for merge.
I also want to land the VMXON series[*] before DPAMT, because there's a nasty wart where KVM wires up a DPAMT-specific hook even if DPAMT is disabled, because KVM's ordering needs to set the vendor hooks before
tdx_sysinfois ready. Decoupling VMXON from KVM solves that problem, because it lets the TDX subsystem parse sysinfo before TDX is loaded.
Quoted 3 lines
Beyond that dependency, I am comfortable landing both DPAMT and S-EPT hugepage support without any other prereqs, i.e. without an in-tree way to light up the S-EPT hugepage code due to lack of hugepage support in
guest_memfd.
Can there be test-cases? Or simple code posted for QEMU which is the tool that 99% of kernel engineers use?
Quoted 125 lines
Outside of the
guest_memfdarch hook for in-place conversion, S-EPT hugepage support doesn't have any direction dependencies/conflicts withguest_memfdhugepage or in-place conversion support (which is great, because it means we didn't totally botch the design!). E.g. Vishal's been able to test this code precisely because it applies relatively cleanly on an internal branch with a whole pile ofguest_memfdchanges.Applies on kvm-x86 next (specifically kvm-x86-next-2026.01.23).
[*] https://lore.kernel.org/all/[email protected]
P.S. I apologize if I clobbered any of the Author attribution or SoBs. I
was moving patches around and synchronizing between an internal tree and this upstream version, so things may have gotten a bit wonky.Isaku Yamahata (1):
KVM: x86/tdp_mmu: Allocexternal_sptpage for mirror page table
splittingKiryl Shutsemau (12):
x86/tdx: Move all TDX error defines into <asm/shared/tdx_errno.h> x86/tdx: Add helpers to check return status codes x86/virt/tdx: Allocate page bitmap for Dynamic PAMT x86/virt/tdx: Allocate reference counters for PAMT memory x86/virt/tdx: Improve PAMT refcounts allocation for sparse memory x86/virt/tdx: Add tdx_alloc/free_control_page() helpers x86/virt/tdx: Optimize tdx_alloc/free_control_page() helpers
KVM: TDX: Allocate PAMT memory for TD and vCPU control structures
KVM: TDX: Get/put PAMT pages when (un)mapping private memory x86/virt/tdx: Enable Dynamic PAMT
Documentation/x86: Add documentation for TDX's Dynamic PAMT x86/virt/tdx: Get/Put DPAMT page pair if and only if mapping size is4KBRick Edgecombe (3):
x86/virt/tdx: Simplifytdmr_get_pamt_sz()
x86/tdx: Add APIs to support get/put of DPAMT entries from KVM, under
spinlock
KVM: x86/mmu: Prevent hugepage promotion for mirror roots in fault
pathSean Christopherson (22):
x86/tdx: Usepg_levelin TDX APIs, not the TDX-Module's 0-based level
KVM: x86/mmu: Updateiter->old_spteif cmpxchg64 on mirror SPTE"fails" KVM: TDX: Account all non-transient page allocations for per-TD structures KVM: x86: Make "external SPTE" ops that can fail RET0 static calls KVM: TDX: Drop kvm_x86_ops.link_external_spt(), use .set_external_spte() for all KVM: x86/mmu: Fold set_external_spte_present() into its sole caller KVM: x86/mmu: Plumb the SPTE _pointer_ into the TDP MMU's handle_changed_spte() KVM: x86/mmu: Propagate mirror SPTE removal to S-EPT in handle_changed_spte() KVM: x86: Rework .free_external_spt() into .reclaim_external_sp() KVM: Allow owner of kvm_mmu_memory_cache to provide a custom page allocator KVM: x86/mmu: Allocate/free S-EPT pages using tdx_{alloc,free}_control_page()*** DO NOT MERGE *** x86/virt/tdx: Don't assume guest memory is backed
by struct page
x86/virt/tdx: Extend "reset page" quirk to support huge pages
KVM: x86/mmu: Plumb theold_spteintokvm_x86_ops.set_external_spte()
KVM: TDX: Hoisttdx_sept_remove_private_spte()above
set_private_spte()
KVM: TDX: Handle removal of leaf SPTEs in .set_private_spte()
KVM: TDX: Add helper to handle mapping leaf SPTE into S-EPT
KVM: TDX: Move S-EPT page demotion TODO totdx_sept_set_private_spte()
KVM: x86/mmu: Add Dynamic PAMT support in TDP MMU for vCPU-induced
page split
KVM:guest_memfd: Add helpers to get start/end gfns give
gmem+slot+pgoff
*** DO NOT MERGE *** KVM:guest_memfd: Add pre-zap arch hook for
shared<=>private conversion
KVM: x86/mmu: Add support for splitting S-EPT hugepages on conversionXiaoyao Li (1):
x86/virt/tdx: Add API to demote a 2MB mapping to 512 4KB mappingsYan Zhao (6):
x86/virt/tdx: Enhancetdh_mem_page_aug()to support huge pages x86/virt/tdx: Enhancetdh_phymem_page_wbinvd_hkid()to invalidate huge
pages
KVM: TDX: Add core support for splitting/demoting 2MiB S-EPT to 4KiB
KVM: x86: Introducehugepage_set_guest_inhibit()
KVM: TDX: Honor the guest's accept level contained in an EPT violation
KVM: TDX: Turn onPG_LEVEL_2MDocumentation/arch/x86/tdx.rst | 21 + arch/x86/coco/tdx/tdx.c | 10 +- arch/x86/include/asm/kvm-x86-ops.h | 9 +- arch/x86/include/asm/kvm_host.h | 36 +- arch/x86/include/asm/shared/tdx.h | 1 + arch/x86/include/asm/shared/tdx_errno.h | 104 +++ arch/x86/include/asm/tdx.h | 127 ++-- arch/x86/include/asm/tdx_global_metadata.h | 1 + arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/mmu.h | 4 + arch/x86/kvm/mmu/mmu.c | 34 +- arch/x86/kvm/mmu/mmu_internal.h | 11 - arch/x86/kvm/mmu/tdp_mmu.c | 315 ++++---- arch/x86/kvm/mmu/tdp_mmu.h | 2 + arch/x86/kvm/vmx/tdx.c | 468 +++++++++--- arch/x86/kvm/vmx/tdx.h | 5 +- arch/x86/kvm/vmx/tdx_arch.h | 3 + arch/x86/kvm/vmx/tdx_errno.h | 40 - arch/x86/virt/vmx/tdx/tdx.c | 762 +++++++++++++++++--- arch/x86/virt/vmx/tdx/tdx.h | 6 +- arch/x86/virt/vmx/tdx/tdx_global_metadata.c | 7 + include/linux/kvm_host.h | 5 + include/linux/kvm_types.h | 2 + virt/kvm/Kconfig | 4 + virt/kvm/guest_memfd.c | 71 +- virt/kvm/kvm_main.c | 7 +- 26 files changed, 1576 insertions(+), 480 deletions(-) create mode 100644 arch/x86/include/asm/shared/tdx_errno.h delete mode 100644 arch/x86/kvm/vmx/tdx_errno.hbase-commit:
e81f7c908e1664233974b9f20beead78cde6343aSignature · 1 line
2.53.0.rc1.217.geba53bf80e-goog
Quoted 7 lines · Konrad Rzeszutek Wilk
Quoted 5 lines · Sean Christopherson
This is a combined series of Dynamic PAMT (from Rick), and S-EPT hugepage support (from Yan). Except for some last minute tweaks to the DPAMT array args stuff, a version of this based on a Google-internal kernel has been moderately well tested (thanks Vishal!). But overall it's still firmly RFC as I have deliberately NOT addressed others feedback from v4 of DPAMT and v3
What does PAMT stand for? Is there a design document somewhere?
It's all in here (I hope):
Sean Christopherson
maintainerQuoted 18 lines · Konrad Rzeszutek Wilk
Quoted 5 lines · Sean Christopherson
This is a combined series of Dynamic PAMT (from Rick), and S-EPT hugepage support (from Yan). Except for some last minute tweaks to the DPAMT array args stuff, a version of this based on a Google-internal kernel has been moderately well tested (thanks Vishal!). But overall it's still firmly RFC as I have deliberately NOT addressed others feedback from v4 of DPAMT and v3
What does PAMT stand for? Is there a design document somewhere?
Quoted 9 lines
of S-EPT hugepage (mostly lack of cycles), and there's at least one patch in here that shouldn't be merged as-is (the quick-and-dirty switch from struct page to raw pfns).
My immediate goal is to solidify the designs for DPAMT and S-EPT hugepage. Given the substantial design changes I am proposing, posting an end-to-end RFC seemed like a much better method than trying to communicate my thoughts piecemeal.
Quoted 4 lines
Quoted 2 lines
As for landing these series, I think the fastest overall approach would be to land patches 1-4 asap (tangentially related cleanups and fixes), agree
Should they be split out as non-RFC then?
Yeah, I'll do that soonish. I posted the kitchen sink so that people could review the entire thing without having to chase down 4+ series/patches.
Quoted 9 lines
Quoted 9 lines
on a design (hopefully), and then hand control back to Rick and Yan to polish their respective series for merge.
I also want to land the VMXON series[*] before DPAMT, because there's a nasty wart where KVM wires up a DPAMT-specific hook even if DPAMT is disabled, because KVM's ordering needs to set the vendor hooks before
tdx_sysinfois ready. Decoupling VMXON from KVM solves that problem, because it lets the TDX subsystem parse sysinfo before TDX is loaded.
Quoted 6 lines
Quoted 3 lines
Beyond that dependency, I am comfortable landing both DPAMT and S-EPT hugepage support without any other prereqs, i.e. without an in-tree way to light up the S-EPT hugepage code due to lack of hugepage support in
guest_memfd.Can there be test-cases? Or simple code posted for QEMU which is the tool that 99% of kernel engineers use?
No? The core limitation is that KVM doesn't yet support hugepages for private memory. No amount userspace code can overcome that limitation.
We can and do have tests and VMM support, but it's all out-of-tree (for now). All I'm saying here is that I'm ok landing the S-EPT hugepage code in advance of guest_memfd hugepage support, e.g. so that we don't end up in a stalemate due to cyclical dependecies, or one big megaseries.
On 2/4/26 06:38, Sean Christopherson wrote:
...
Quoted 4 lines
We can and do have tests and VMM support, but it's all out-of-tree (for now). All I'm saying here is that I'm ok landing the S-EPT hugepage code in advance of
guest_memfdhugepage support, e.g. so that we don't end up in a stalemate due to cyclical dependecies, or one big megaseries.
Does "landing" mean having it sit in some topic branch, or pushing to Linus?
I'm all for getting these hellish dependency chains out of the way, but we usually try pretty hard to avoid having dead/unreachable code in mainline.
If it is something you want to do in mainline, we should probably do a bit of cross-x86/kvm brainstorming to make sure there's no other option.
Sean Christopherson
maintainerQuoted 8 lines · Dave Hansen
On 2/4/26 06:38, Sean Christopherson wrote:
...Quoted 4 lines
We can and do have tests and VMM support, but it's all out-of-tree (for now). All I'm saying here is that I'm ok landing the S-EPT hugepage code in advance of
guest_memfdhugepage support, e.g. so that we don't end up in a stalemate due to cyclical dependecies, or one big megaseries.Does "landing" mean having it sit in some topic branch, or pushing to Linus?
I was thinking pushing to Linus' tree, but a topic branch could likely provide almost as much value?
Quoted 6 lines
I'm all for getting these hellish dependency chains out of the way, but we usually try pretty hard to avoid having dead/unreachable code in mainline.
If it is something you want to do in mainline, we should probably do a bit of cross-x86/kvm brainstorming to make sure there's no other option.
I'm a-ok starting with a topic branch. If maintaining that branch becomes too costly, then we can always revisit things. And that would probably be good motiviation to beat guest_memfd hugepage into shape :-)
Quoted 3 lines · Sean Christopherson
I'm a-ok starting with a topic branch. If maintaining that branch becomes too costly, then we can always revisit things. And that would probably be good motiviation to beat
guest_memfdhugepage into shape 🙂
Sounds like a plan.
Edgecombe, Rick P
maintainerQuoted 4 lines · Sean Christopherson
As for landing these series, I think the fastest overall approach would be to land patches 1-4 asap (tangentially related cleanups and fixes), agree on a design (hopefully), and then hand control back to Rick and Yan to polish their respective series for merge.
Sean, were you still planning to do this? I see you're, I think, asking for Dave to pull patch 1 here:
https://lore.kernel.org/kvm/[email protected]/
But actually he acked the version in this series, so another option is you could just take patches 1-4?
Please let me know if you are ok with that plan. We are growing stacks ahead of DPAMT and the other splintered pre-req work and looking to reduce it. It could be nice if we could take all the pre-req work through the same tree for that reason, and most of it is on the KVM side. Is it reasonable?
Sean Christopherson
maintainerQuoted 11 lines · Rick P Edgecombe
Quoted 4 lines · Sean Christopherson
As for landing these series, I think the fastest overall approach would be to land patches 1-4 asap (tangentially related cleanups and fixes), agree on a design (hopefully), and then hand control back to Rick and Yan to polish their respective series for merge.
Sean, were you still planning to do this? I see you're, I think, asking for Dave to pull patch 1 here:
https://lore.kernel.org/kvm/[email protected]/But actually he acked the version in this series,
Oh, right. I forgot about that.
Quoted 1 line
so another option is you could just take patches 1-4?
Yeah, I think that makes the most sense.
Quoted 4 lines
Please let me know if you are ok with that plan. We are growing stacks ahead of DPAMT and the other splintered pre-req work and looking to reduce it. It could be nice if we could take all the pre-req work through the same tree for that reason, and most of it is on the KVM side. Is it reasonable?
Ya. I'll apply 1-4 and create a stable tag for patch 1's commit for 7.2. If it won't throw a wrench in things for you, I'll wait until 7.1-rc2 to get 'em applied. I've been burned more than a few times when using rc1 as the base for topic branches.
Dave, holler if this doesn't work for you.
Edgecombe, Rick P
maintainerQuoted 4 lines · Sean Christopherson
Quoted 4 lines
Ya. I'll apply 1-4 and create a stable tag for patch 1's commit for 7.2. If it won't throw a wrench in things for you, I'll wait until 7.1-rc2 to get 'em applied. I've been burned more than a few times when using rc1 as the base for topic branches.
Sounds great from my side. Thanks!
Sean Christopherson
maintainerQuoted 10 lines · Sean Christopherson
This is a combined series of Dynamic PAMT (from Rick), and S-EPT hugepage support (from Yan). Except for some last minute tweaks to the DPAMT array args stuff, a version of this based on a Google-internal kernel has been moderately well tested (thanks Vishal!). But overall it's still firmly RFC as I have deliberately NOT addressed others feedback from v4 of DPAMT and v3 of S-EPT hugepage (mostly lack of cycles), and there's at least one patch in here that shouldn't be merged as-is (the quick-and-dirty switch from struct page to raw pfns).
[...]
Applied 1-4 to kvm-x86 mmu. Please yell if this was unexpected in any way. I'm pretty sure this is what we agreed on, but the last few week have been a bit chaotic...
[01/45] x86/tdx: Use pg_level in TDX APIs, not the TDX-Module's 0-based level
https://github.com/kvm-x86/linux/commit/4487492b92a4 [02/45] KVM: x86/mmu: Update iter->old_spte if cmpxchg64 on mirror SPTE "fails" https://github.com/kvm-x86/linux/commit/02eaaffdd865 [03/45] KVM: TDX: Account all non-transient page allocations for per-TD structures https://github.com/kvm-x86/linux/commit/a8b2924676ec [04/45] KVM: x86: Make "external SPTE" ops that can fail RET0 static calls https://github.com/kvm-x86/linux/commit/e1a31ca28c9d