Include MKTME KeyID bits to MMU shadow_zero_check
cover Include MKTME KeyID bits to MMU shadow_zero_check
Host kernel never sets any Intel MKTME KeyID bits (including TDX KeyID bits) in any mapping. KVM should not set any of them to MMU's SPTE which maps to guest memory either. KVM MMU has shadow_zero_check which contains those bits must be 0 for SPTE. This series adds MKTME bits to shadow_zero_check. The first patch is not directly needed but looks better to have IMHO. Patch 02/03 actually do the job.
Sanity tested by creating normal VMs on TDX capable machine (with TDX turned on). However this is not tested on AMD machine since I don't have access to any, though the risk should be very small I suppose.
Kai Huang (3):
KVM: x86/mmu: Rename reset_rsvds_bits_mask()
KVM: x86/mmu: Add shadow_me_value and repurpose shadow_me_mask
KVM: VMX: Include MKTME KeyID bits to shadow_zero_check
arch/x86/kvm/mmu.h | 20 ++++++++++++++++++++ arch/x86/kvm/mmu/mmu.c | 22 +++++++++++++++------- arch/x86/kvm/mmu/spte.c | 40 ++++++++++++++++------------------------ arch/x86/kvm/mmu/spte.h | 1 + arch/x86/kvm/svm/svm.c | 3 +++ arch/x86/kvm/vmx/vmx.c | 31 +++++++++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 31 deletions(-)
Signature · 1 line
2.35.1
PATCH 1/3 KVM: x86/mmu: Rename reset_rsvds_bits_mask()
Rename reset_rsvds_bits_mask() to reset_guest_rsvds_bits_mask() to make it clearer that it resets the reserved bits check for guest's page table entries.
--- arch/x86/kvm/mmu/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/kvm/mmu/mmu.c +3 -3
@@ -4369,8 +4369,8 @@ static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu)guest_cpuid_has (vcpu,X86_FEATURE_GBPAGES ); } -static void reset_rsvds_bits_mask (struct kvm_vcpu *vcpu, -struct kvm_mmu *context) +static void reset_guest_rsvds_bits_mask (struct kvm_vcpu *vcpu, +struct kvm_mmu *context) { __reset_rsvds_bits_mask(&context->guest_rsvd_check, vcpu->arch.reserved_gpa_bits,
@@ -4669,7 +4669,7 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,if (!is_cr0_pg (mmu))return ; -reset_rsvds_bits_mask (vcpu, mmu); +reset_guest_rsvds_bits_mask (vcpu, mmu);update_permission_bitmask (mmu, false);update_pkru_bitmask (mmu); }
Signature · 1 line
2.35.1
PATCH 3/3 KVM: VMX: Include MKTME KeyID bits to shadow_zero_check
In reply to Kai HuangIntel MKTME KeyID bits (including Intel TDX private KeyID bits) should never be set to SPTE. Set shadow_me_value to 0 and shadow_me_mask to include all MKTME KeyID bits to include them to shadow_zero_check.
--- arch/x86/kvm/mmu.h | 19 +++++++++++++++++++ arch/x86/kvm/mmu/spte.c | 19 ------------------- arch/x86/kvm/vmx/vmx.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 19 deletions(-)
arch/x86/kvm/mmu.h +19 -0
@@ -65,6 +65,25 @@ static __always_inline u64 rsvd_bits(int s, int e)return ((2ULL << (e - s)) -1 ) << s; } +static inline u8 kvm_get_shadow_phys_bits (void ) +{ +/* +* boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected +* in CPU detection code, but the processor treats those reduced bits as +* 'keyID' thus they are not reserved bits. Therefore KVM needs to look at +* the physical address bits reported by CPUID. +*/ +if (likely (boot_cpu_data.extended_cpuid_level >=0x80000008 )) +return cpuid_eax (0x80000008 ) &0xff ; + +/* +* Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with +* custom CPUID. Proceed with whatever the kernel found since these features +* aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008). +*/ +return boot_cpu_data.x86_phys_bits; +} +void kvm_mmu_set_mmio_spte_mask (u64 mmio_value,u64 mmio_mask,u64 access_mask);void kvm_mmu_set_me_spte_mask (u64 me_value,u64 me_mask);void kvm_mmu_set_ept_masks (bool has_ad_bits,bool has_exec_only);
arch/x86/kvm/mmu/spte.c +0 -19
@@ -283,25 +283,6 @@ u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn)return new_spte; } -static u8 kvm_get_shadow_phys_bits (void ) -{ -/* -* boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected -* in CPU detection code, but the processor treats those reduced bits as -* 'keyID' thus they are not reserved bits. Therefore KVM needs to look at -* the physical address bits reported by CPUID. -*/ -if (likely (boot_cpu_data.extended_cpuid_level >=0x80000008 )) -return cpuid_eax (0x80000008 ) &0xff ; - -/* -* Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with -* custom CPUID. Proceed with whatever the kernel found since these features -* aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008). -*/ -return boot_cpu_data.x86_phys_bits; -} -u64 mark_spte_for_access_track (u64 spte) {if (spte_ad_enabled (spte))
arch/x86/kvm/vmx/vmx.c +31 -0
@@ -7883,6 +7883,31 @@ static __init void vmx_setup_user_return_msrs(void)kvm_add_user_return_msr (vmx_uret_msrs_list[i]); } +static void __init vmx_setup_me_spte_mask (void ) +{ +u64 me_mask =0 ; + +/* +* kvm_get_shadow_phys_bits() returns shadow_phys_bits. Use +* the former to avoid exposing shadow_phys_bits. +* +* On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to +* shadow_phys_bits. On MKTME and/or TDX capable systems, +* boot_cpu_data.x86_phys_bits holds the actual physical address +* w/o the KeyID bits, and shadow_phys_bits equals to MAXPHYADDR +* reported by CPUID. Those bits between are KeyID bits. +*/ +if (boot_cpu_data.x86_phys_bits !=kvm_get_shadow_phys_bits ()) + me_mask =rsvd_bits (boot_cpu_data.x86_phys_bits, +kvm_get_shadow_phys_bits () -1 ); +/* +* Unlike to SME, host kernel doesn't support setting up any +* MKTME KeyID to any mapping for Intel platforms. No memory +* encryption bits should be included into the SPTE. +*/ +kvm_mmu_set_me_spte_mask (0 , me_mask); +} +static struct kvm_x86_init_ops vmx_init_ops__initdata ;static __init int hardware_setup (void )
@@ -7985,6 +8010,12 @@ static __init int hardware_setup(void)kvm_mmu_set_ept_masks (enable_ept_ad_bits,cpu_has_vmx_ept_execute_only ()); +/* +* Setup shadow_me_value/shadow_me_mask to include MKTME KeyID +* bits to shadow_zero_check. +*/ +vmx_setup_me_spte_mask (); +kvm_configure_mmu (enable_ept,0 ,vmx_get_max_tdp_level (),ept_caps_to_lpage_level (vmx_capability.ept));
Signature · 1 line
2.35.1
PATCH 2/3 KVM: x86/mmu: Add shadow_me_value and repurpose shadow_me_mask
In reply to Kai HuangIntel Multi-Key Total Memory Encryption (MKTME) repurposes couple of high bits of physical address bits as 'KeyID' bits. Intel Trust Domain Extentions (TDX) further steals part of MKTME KeyID bits as TDX private KeyID bits. TDX private KeyID bits cannot be set in any mapping in the host kernel since they can only be accessed by software running inside a new CPU isolated mode. And unlike to AMD's SME, host kernel doesn't set any legacy MKTME KeyID bits to any mapping either. Therefore, it's not legitimate for KVM to set any KeyID bits in SPTE which maps guest memory.
KVM maintains shadow_zero_check bits to represent which bits must be zero for SPTE which maps guest memory. MKTME KeyID bits should be set to shadow_zero_check. Currently, shadow_me_mask is used by AMD to set the sme_me_mask to SPTE, and shadow_me_shadow is excluded from shadow_zero_check. So initializing shadow_me_mask to represent all MKTME keyID bits doesn't work for VMX (as oppositely, they must be set to shadow_zero_check).
Introduce a new 'shadow_me_value' to replace existing shadow_me_mask, and repurpose shadow_me_mask as 'all possible memory encryption bits'. The new schematic of them will be:
- shadow_me_value: the memory encryption bit(s) that will be set to the SPTE (the original shadow_me_mask).
- shadow_me_mask: all possible memory encryption bits (which is a super set of shadow_me_value).
- For now, shadow_me_value is supposed to be set by SVM and VMX respectively, and it is a constant during KVM's life time. This perhaps doesn't fit MKTME but for now host kernel doesn't support it (and perhaps will never do).
- Bits in shadow_me_mask are set to shadow_zero_check, except the bits in shadow_me_value.
Introduce a new helper kvm_mmu_set_me_spte_mask() to initialize them. Replace shadow_me_mask with shadow_me_value in almost all code paths, except the one in PT64_PERM_MASK, which is used by need_remote_flush() to determine whether remote TLB flush is needed. This should still use shadow_me_mask as any encryption bit change should need a TLB flush. And for AMD, move initializing shadow_me_value/shadow_me_mask from kvm_mmu_reset_all_pte_masks() to svm_hardware_setup().
--- arch/x86/kvm/mmu.h | 1 + arch/x86/kvm/mmu/mmu.c | 16 ++++++++++++---- arch/x86/kvm/mmu/spte.c | 21 ++++++++++++++++----- arch/x86/kvm/mmu/spte.h | 1 + arch/x86/kvm/svm/svm.c | 3 +++ 5 files changed, 33 insertions(+), 9 deletions(-)
arch/x86/kvm/mmu.h +1 -0
@@ -66,6 +66,7 @@ static __always_inline u64 rsvd_bits(int s, int e) }void kvm_mmu_set_mmio_spte_mask (u64 mmio_value,u64 mmio_mask,u64 access_mask); +void kvm_mmu_set_me_spte_mask (u64 me_value,u64 me_mask);void kvm_mmu_set_ept_masks (bool has_ad_bits,bool has_exec_only);void kvm_init_mmu (struct kvm_vcpu *vcpu);
arch/x86/kvm/mmu/mmu.c +12 -4
@@ -3489,7 +3489,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)* or a PAE 3-level page table. In either case we need to be aware that * the shadow page table may be a PAE or a long mode page table. */ - pm_mask =PT_PRESENT_MASK | shadow_me_mask; + pm_mask =PT_PRESENT_MASK | shadow_me_value;if (mmu->root_role.level >=PT64_ROOT_4LEVEL ) { pm_mask |=PT_ACCESSED_MASK |PT_WRITABLE_MASK |PT_USER_MASK ;
@@ -4458,8 +4458,16 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,return ;for (i = context->root_role.level; --i >=0 ;) { - shadow_zero_check->rsvd_bits_mask[0 ][i] &= ~shadow_me_mask; - shadow_zero_check->rsvd_bits_mask[1 ][i] &= ~shadow_me_mask; +/* +* So far shadow_me_value is a constant during KVM's life +* time. Bits in shadow_me_value are allowed to be set. +* Bits in shadow_me_mask but not in shadow_me_value are +* not allowed to be set. +*/ + shadow_zero_check->rsvd_bits_mask[0 ][i] |= shadow_me_mask; + shadow_zero_check->rsvd_bits_mask[1 ][i] |= shadow_me_mask; + shadow_zero_check->rsvd_bits_mask[0 ][i] &= ~shadow_me_value; + shadow_zero_check->rsvd_bits_mask[1 ][i] &= ~shadow_me_value; } }
@@ -5546,7 +5554,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)if (!tdp_enabled)set_memory_decrypted ((unsigned long )mmu->pae_root,1 );else -WARN_ON_ONCE (shadow_me_mask); +WARN_ON_ONCE (shadow_me_value);for (i =0 ; i <4 ; ++i) mmu->pae_root[i] =INVALID_PAE_ROOT ;
arch/x86/kvm/mmu/spte.c +16 -5
@@ -33,6 +33,7 @@ u64 __read_mostly shadow_mmio_value;u64 __read_mostly shadow_mmio_mask;u64 __read_mostly shadow_mmio_access_mask;u64 __read_mostly shadow_present_mask; +u64 __read_mostly shadow_me_value;u64 __read_mostly shadow_me_mask;u64 __read_mostly shadow_acc_track_mask;
@@ -140,7 +141,7 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, pte_access &= ~ACC_WRITE_MASK ;if (!kvm_is_mmio_pfn (pfn)) - spte |= shadow_me_mask; + spte |= shadow_me_value; spte |= (u64 )pfn <<PAGE_SHIFT ;
@@ -256,7 +257,7 @@ u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled)u64 spte =SPTE_MMU_PRESENT_MASK ; spte |=__pa (child_pt) | shadow_present_mask |PT_WRITABLE_MASK | - shadow_user_mask | shadow_x_mask | shadow_me_mask; + shadow_user_mask | shadow_x_mask | shadow_me_value;if (ad_disabled) spte |=SPTE_TDP_AD_DISABLED_MASK ;
@@ -357,6 +358,17 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask) }EXPORT_SYMBOL_GPL (kvm_mmu_set_mmio_spte_mask); +void kvm_mmu_set_me_spte_mask (u64 me_value,u64 me_mask) +{ +/* shadow_me_value must be a subset of shadow_me_mask */ +if (WARN_ON (me_value & ~me_mask)) + me_value = me_mask =0 ; + + shadow_me_value = me_value; + shadow_me_mask = me_mask; +} +EXPORT_SYMBOL_GPL (kvm_mmu_set_me_spte_mask); +void kvm_mmu_set_ept_masks (bool has_ad_bits,bool has_exec_only) { shadow_user_mask =VMX_EPT_READABLE_MASK ;
@@ -366,8 +378,6 @@ void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only) shadow_x_mask =VMX_EPT_EXECUTABLE_MASK ; shadow_present_mask = has_exec_only ?0ull :VMX_EPT_READABLE_MASK ; shadow_acc_track_mask =VMX_EPT_RWX_MASK ; - shadow_me_mask =0ull ; - shadow_host_writable_mask =EPT_SPTE_HOST_WRITABLE ; shadow_mmu_writable_mask =EPT_SPTE_MMU_WRITABLE ;
@@ -418,7 +428,8 @@ void kvm_mmu_reset_all_pte_masks(void) shadow_x_mask =0 ; shadow_present_mask =PT_PRESENT_MASK ; shadow_acc_track_mask =0 ; - shadow_me_mask = sme_me_mask; + shadow_me_mask =0 ; + shadow_me_value =0 ; shadow_host_writable_mask =DEFAULT_SPTE_HOST_WRITABLE ; shadow_mmu_writable_mask =DEFAULT_SPTE_MMU_WRITABLE ;
arch/x86/kvm/mmu/spte.h +1 -0
@@ -149,6 +149,7 @@ extern u64 __read_mostly shadow_mmio_value;extern u64 __read_mostly shadow_mmio_mask;extern u64 __read_mostly shadow_mmio_access_mask;extern u64 __read_mostly shadow_present_mask; +extern u64 __read_mostly shadow_me_value;extern u64 __read_mostly shadow_me_mask;/*
arch/x86/kvm/svm/svm.c +3 -0
@@ -4891,6 +4891,9 @@ static __init int svm_hardware_setup(void) get_npt_level(), PG_LEVEL_1G);pr_info ("kvm: Nested Paging %sabled\n" , npt_enabled ?"en" :"dis" ); +/* Setup shadow_me_value and shadow_me_mask */ +kvm_mmu_set_me_spte_mask (sme_me_mask, sme_me_mask); +/* Note, SEV setup consumes npt_enabled. */ sev_hardware_setup ();
Signature · 1 line
2.35.1
Queued, thanks.
Paolo