x86/tdx: Fix one more load_unaligned_zeropad() issue
Subsystems (4)
v2 cover x86/tdx: Fix one more load_unaligned_zeropad() issue
During review of TDX guests on Hyper-V patchset Dave pointed to the potential race between changing page private/shared status and load_unaligned_zeropad().
Fix the issue.
v2:
- Add more info in commit message of the first patch.
- Move enc_status_change_finish_noop() into a separate patch.
- Fix typo in commit message and comment.
Kirill A. Shutemov (3):
x86/mm: Allow guest.enc_status_change_prepare() to fail x86/tdx: Fix race between set_memory_encrypted() and
load_unaligned_zeropad()
x86/mm: Fix enc_status_change_finish_noop()
arch/x86/coco/tdx/tdx.c | 56 +++++++++++++++++++++++++++++++-- arch/x86/include/asm/x86_init.h | 2 +- arch/x86/kernel/x86_init.c | 4 +-- arch/x86/mm/mem_encrypt_amd.c | 4 ++- arch/x86/mm/pat/set_memory.c | 3 +- 5 files changed, 61 insertions(+), 8 deletions(-)
Signature · 1 line
2.39.3
PATCH v2 1/3 x86/mm: Allow guest.enc_status_change_prepare() to fail
TDX code is going to provide guest.enc_status_change_prepare() that is able to fail. TDX will use the call to convert the GPA range from shared to private. This operation can fail.
Add a way to return an error from the callback.
--- arch/x86/include/asm/x86_init.h | 2 +- arch/x86/kernel/x86_init.c | 2 +- arch/x86/mm/mem_encrypt_amd.c | 4 +++- arch/x86/mm/pat/set_memory.c | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-)
arch/x86/include/asm/x86_init.h +1 -1
@@ -150,7 +150,7 @@ struct x86_init_acpi {* @enc_cache_flush_required Returns true if a cache flush is needed before changing page encryption status */ struct x86_guest { -void (*enc_status_change_prepare)(unsigned long vaddr,int npages,bool enc); +bool (*enc_status_change_prepare)(unsigned long vaddr,int npages,bool enc);bool (*enc_status_change_finish)(unsigned long vaddr,int npages,bool enc);bool (*enc_tlb_flush_required)(bool enc);bool (*enc_cache_flush_required)(void );
arch/x86/kernel/x86_init.c +1 -1
@@ -130,7 +130,7 @@ struct x86_cpuinit_ops x86_cpuinit = {static void default_nmi_init (void ) { }; -static void enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) { } +static bool enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) {return true; }static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return false; }static bool enc_tlb_flush_required_noop (bool enc) {return false; }static bool enc_cache_flush_required_noop (void ) {return false; }
arch/x86/mm/mem_encrypt_amd.c +3 -1
@@ -319,7 +319,7 @@ static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)#endif } -static void amd_enc_status_change_prepare (unsigned long vaddr,int npages,bool enc) +static bool amd_enc_status_change_prepare (unsigned long vaddr,int npages,bool enc) {/* * To maintain the security guarantees of SEV-SNP guests, make sure
@@ -327,6 +327,8 @@ static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool*/ if (cc_platform_has (CC_ATTR_GUEST_SEV_SNP ) && !enc)snp_set_memory_shared (vaddr, npages); + +return true; }/* Return true unconditionally: return value doesn't matter for the SEV side */
arch/x86/mm/pat/set_memory.c +2 -1
@@ -2151,7 +2151,8 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc)cpa_flush (&cpa, x86_platform.guest.enc_cache_flush_required ());/* Notify hypervisor that we are about to set/clr encryption attribute. */ - x86_platform.guest.enc_status_change_prepare (addr, numpages, enc); +if (!x86_platform.guest.enc_status_change_prepare (addr, numpages, enc)) +return -EIO ; ret =__change_page_attr_set_clr (&cpa,1 );
Signature · 1 line
2.39.3
Re: [PATCHv2 1/3] x86/mm: Allow guest.enc_status_change_prepare() to fail
Quoted 9 lines · Kirill A. Shutemov
TDX code is going to provide
guest.enc_status_change_prepare()that is able to fail. TDX will use the call to convert the GPA range from shared to private. This operation can fail.Add a way to return an error from the callback.
Signed-off-by Kirill A. Shutemov <[email protected]>---
Looks good to me.
Quoted 67 lines
arch/x86/include/asm/x86_init.h | 2 +- arch/x86/kernel/x86_init.c | 2 +- arch/x86/mm/mem_encrypt_amd.c | 4 +++- arch/x86/mm/pat/set_memory.c | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-)arch/x86/include/asm/x86_init.h +1
-1@@ -150,7 +150,7 @@ struct x86_init_acpi {* @enc_cache_flush_required Returns true if a cache flush is needed before changing page encryption status */ struct x86_guest { -void (*enc_status_change_prepare)(unsigned long vaddr,int npages,bool enc); +bool (*enc_status_change_prepare)(unsigned long vaddr,int npages,bool enc);bool (*enc_status_change_finish)(unsigned long vaddr,int npages,bool enc);bool (*enc_tlb_flush_required)(bool enc);bool (*enc_cache_flush_required)(void );arch/x86/kernel/x86_init.c +1
-1@@ -130,7 +130,7 @@ struct x86_cpuinit_ops x86_cpuinit = {static void default_nmi_init (void ) { }; -static void enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) { } +static bool enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) {return true; }static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return false; }static bool enc_tlb_flush_required_noop (bool enc) {return false; }static bool enc_cache_flush_required_noop (void ) {return false; }arch/x86/mm/mem_encrypt_amd.c +3
-1@@ -319,7 +319,7 @@ static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc)#endif } -static void amd_enc_status_change_prepare (unsigned long vaddr,int npages,bool enc) +static bool amd_enc_status_change_prepare (unsigned long vaddr,int npages,bool enc) {/* * To maintain the security guarantees of SEV-SNP guests, make sure @@ -327,6 +327,8 @@ static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool*/ if (cc_platform_has (CC_ATTR_GUEST_SEV_SNP ) && !enc)snp_set_memory_shared (vaddr, npages); + +return true; }>/* Return true unconditionally: return value doesn't matter for the SEV side */ arch/x86/mm/pat/set_memory.c +2
-1@@ -2151,7 +2151,8 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc)cpa_flush (&cpa, x86_platform.guest.enc_cache_flush_required ());/* Notify hypervisor that we are about to set/clr encryption attribute. */ - x86_platform.guest.enc_status_change_prepare (addr, numpages, enc); +if (!x86_platform.guest.enc_status_change_prepare (addr, numpages, enc)) +return -EIO ; ret =__change_page_attr_set_clr (&cpa,1 );
Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer
PATCH v2 2/3 x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
In reply to Kirill A. ShutemovTouching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad() can touch memory that is not owned by the caller, but just happened to next after the owned memory. This load_unaligned_zeropad() behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.
guest.enc_status_change_prepare() called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
guest.enc_status_change_finish() called after adjusting direct mapping and it converts the memory to shared.
It is okay to have a shared mapping of memory that is not converted properly. handle_mmio() knows how to deal with load_unaligned_zeropad() stepping on it.
--- arch/x86/coco/tdx/tdx.c | 56 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-)
arch/x86/coco/tdx/tdx.c +53 -3
@@ -840,6 +840,30 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)return true; } +static bool tdx_enc_status_change_prepare (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle shared->private conversion here. +* See the comment in tdx_early_init(). +*/ +if (enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} + +static bool tdx_enc_status_change_finish (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle private->shared conversion here. +* See the comment in tdx_early_init(). +*/ +if (!enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} +void __init tdx_early_init (void ) {u64 cc_mask;
@@ -867,9 +891,35 @@ void __init tdx_early_init(void)*/ physical_mask &= cc_mask -1 ; - x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; - x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required; - x86_platform.guest.enc_status_change_finish = tdx_enc_status_changed; +/* +* Touching privately mapped GPA that is not properly converted to +* private with MapGPA and accepted leads to unrecoverable exit +* to VMM. +* +* load_unaligned_zeropad() can touch memory that is not owned by +* the caller, but just happened to next after the owned memory. +* This load_unaligned_zeropad() behaviour makes it important when +* kernel asks VMM to convert a GPA from shared to private or back. +* Kernel must never have a page mapped into direct mapping (and +* aliases) as private when the GPA is already converted to shared or +* when GPA is not yet converted to private. +* +* guest.enc_status_change_prepare() called before adjusting direct +* mapping and therefore it is responsible for converting the memory +* to private. +* +* guest.enc_status_change_finish() called after adjusting direct +* mapping and it converts the memory to shared. +* +* It is okay to have a shared mapping of memory that is not converted +* properly. handle_mmio() knows how to deal with load_unaligned_zeropad() +* stepping on it. +*/ + x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; + + x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; + x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;pr_info ("Guest detected\n" ); }
Signature · 1 line
2.39.3
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 5 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory.
/s/to/to be ?
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.
I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in set_memory_encrypted() where this race exists.
Quoted 55 lines
guest.enc_status_change_prepare()called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
guest.enc_status_change_finish()called after adjusting direct mapping and it converts the memory to shared.It is okay to have a shared mapping of memory that is not converted properly.
handle_mmio()knows how to deal withload_unaligned_zeropad()stepping on it.Signed-off-by Kirill A. Shutemov <[email protected]>Fixes 7dbde7631629 ("x86/mm/cpa: Add support for TDX shared memory")--- arch/x86/coco/tdx/tdx.c | 56 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-)arch/x86/coco/tdx/tdx.c +24
-0@@ -840,6 +840,30 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)return true; } +static bool tdx_enc_status_change_prepare (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle shared->private conversion here. +* See the comment in tdx_early_init(). +*/ +if (enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} + +static bool tdx_enc_status_change_finish (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle private->shared conversion here. +* See the comment in tdx_early_init(). +*/ +if (!enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} +void __init tdx_early_init (void ) {u64 cc_mask;
arch/x86/coco/tdx/tdx.c void __init tdx_early_init(void) · line 871
Quoted 5 lines
arch/x86/coco/tdx/tdx.c +0
-2@@ -867,9 +891,35 @@ void __init tdx_early_init(void)*/ physical_mask &= cc_mask -1 ; - x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; - x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;
I think you don't need to change the order here.
Quoted 33 lines
arch/x86/coco/tdx/tdx.c +29
-1- x86_platform.guest.enc_status_change_finish = tdx_enc_status_changed; +/* +* Touching privately mapped GPA that is not properly converted to +* private with MapGPA and accepted leads to unrecoverable exit +* to VMM. +* +* load_unaligned_zeropad() can touch memory that is not owned by +* the caller, but just happened to next after the owned memory. +* This load_unaligned_zeropad() behaviour makes it important when +* kernel asks VMM to convert a GPA from shared to private or back. +* Kernel must never have a page mapped into direct mapping (and +* aliases) as private when the GPA is already converted to shared or +* when GPA is not yet converted to private. +* +* guest.enc_status_change_prepare() called before adjusting direct +* mapping and therefore it is responsible for converting the memory +* to private. +* +* guest.enc_status_change_finish() called after adjusting direct +* mapping and it converts the memory to shared. +* +* It is okay to have a shared mapping of memory that is not converted +* properly. handle_mmio() knows how to deal with load_unaligned_zeropad() +* stepping on it. +*/ + x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; + + x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; + x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;pr_info ("Guest detected\n" ); }
Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 10 lines · Sathyanarayanan Kuppuswamy
Quoted 5 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 10 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.
I don't think AMD affected by load_unaligned_zeropad() the same way as Intel does. But I'm not sure.
Tom, do you have any comments?
Quoted 59 lines
Quoted 59 lines
guest.enc_status_change_prepare()called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
guest.enc_status_change_finish()called after adjusting direct mapping and it converts the memory to shared.It is okay to have a shared mapping of memory that is not converted properly.
handle_mmio()knows how to deal withload_unaligned_zeropad()stepping on it.Signed-off-by Kirill A. Shutemov <[email protected]>Fixes 7dbde7631629 ("x86/mm/cpa: Add support for TDX shared memory")--- arch/x86/coco/tdx/tdx.c | 56 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-)arch/x86/coco/tdx/tdx.c +24
-0@@ -840,6 +840,30 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)return true; } +static bool tdx_enc_status_change_prepare (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle shared->private conversion here. +* See the comment in tdx_early_init(). +*/ +if (enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} + +static bool tdx_enc_status_change_finish (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle private->shared conversion here. +* See the comment in tdx_early_init(). +*/ +if (!enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} +void __init tdx_early_init (void ) {u64 cc_mask;@@ -867,9 +891,35 @@ void __init tdx_early_init(void)*/ physical_mask &= cc_mask -1 ;
Quoted 4 lines
arch/x86/coco/tdx/tdx.c void __init tdx_early_init(void) · line 871
Quoted 2 lines
arch/x86/coco/tdx/tdx.c +0
-2- x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; - x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;I think you don't need to change the order here.
I wanted to emphasise that the comment is for _prepare/_finish callbacks and I hoped re-order would help with this.
Quoted 37 lines
Quoted 33 lines
arch/x86/coco/tdx/tdx.c +29
-1- x86_platform.guest.enc_status_change_finish = tdx_enc_status_changed; +/* +* Touching privately mapped GPA that is not properly converted to +* private with MapGPA and accepted leads to unrecoverable exit +* to VMM. +* +* load_unaligned_zeropad() can touch memory that is not owned by +* the caller, but just happened to next after the owned memory. +* This load_unaligned_zeropad() behaviour makes it important when +* kernel asks VMM to convert a GPA from shared to private or back. +* Kernel must never have a page mapped into direct mapping (and +* aliases) as private when the GPA is already converted to shared or +* when GPA is not yet converted to private. +* +* guest.enc_status_change_prepare() called before adjusting direct +* mapping and therefore it is responsible for converting the memory +* to private. +* +* guest.enc_status_change_finish() called after adjusting direct +* mapping and it converts the memory to shared. +* +* It is okay to have a shared mapping of memory that is not converted +* properly. handle_mmio() knows how to deal with load_unaligned_zeropad() +* stepping on it. +*/ + x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; + + x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; + x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;pr_info ("Guest detected\n" ); }Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer
Signature · 1 line
Kiryl Shutsemau / Kirill A. Shutemov
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 22 lines · Kirill A. Shutemov
Quoted 10 lines · Sathyanarayanan Kuppuswamy
Quoted 5 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 8 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
Quoted 7 lines
Quoted 2 lines
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.I don't think AMD affected by
load_unaligned_zeropad()the same way as Intel does. But I'm not sure.Tom, do you have any comments?
Right, shouldn't be an issue for SNP.
Thanks,
Tom
Quoted 106 lines
Quoted 60 lines
Quoted 60 lines
guest.enc_status_change_prepare()called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
guest.enc_status_change_finish()called after adjusting direct mapping and it converts the memory to shared.It is okay to have a shared mapping of memory that is not converted properly.
handle_mmio()knows how to deal withload_unaligned_zeropad()stepping on it.Signed-off-by Kirill A. Shutemov <[email protected]>Fixes 7dbde7631629 ("x86/mm/cpa: Add support for TDX shared memory")--- arch/x86/coco/tdx/tdx.c | 56 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-)arch/x86/coco/tdx/tdx.c +24
-0@@ -840,6 +840,30 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc)return true; } +static bool tdx_enc_status_change_prepare (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle shared->private conversion here. +* See the comment in tdx_early_init(). +*/ +if (enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} + +static bool tdx_enc_status_change_finish (unsigned long vaddr,int numpages, +bool enc) +{ +/* +* Only handle private->shared conversion here. +* See the comment in tdx_early_init(). +*/ +if (!enc) +return tdx_enc_status_changed (vaddr, numpages, enc); +return true; +} +void __init tdx_early_init (void ) {u64 cc_mask;@@ -867,9 +891,35 @@ void __init tdx_early_init(void)*/ physical_mask &= cc_mask -1 ;Quoted 4 lines
arch/x86/coco/tdx/tdx.c void __init tdx_early_init(void) · line 871
Quoted 2 lines
arch/x86/coco/tdx/tdx.c +0
-2- x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; - x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;I think you don't need to change the order here.
I wanted to emphasise that the comment is for _prepare/_finish callbacks and I hoped re-order would help with this.
Quoted 37 lines
Quoted 33 lines
arch/x86/coco/tdx/tdx.c +29
-1- x86_platform.guest.enc_status_change_finish = tdx_enc_status_changed; +/* +* Touching privately mapped GPA that is not properly converted to +* private with MapGPA and accepted leads to unrecoverable exit +* to VMM. +* +* load_unaligned_zeropad() can touch memory that is not owned by +* the caller, but just happened to next after the owned memory. +* This load_unaligned_zeropad() behaviour makes it important when +* kernel asks VMM to convert a GPA from shared to private or back. +* Kernel must never have a page mapped into direct mapping (and +* aliases) as private when the GPA is already converted to shared or +* when GPA is not yet converted to private. +* +* guest.enc_status_change_prepare() called before adjusting direct +* mapping and therefore it is responsible for converting the memory +* to private. +* +* guest.enc_status_change_finish() called after adjusting direct +* mapping and it converts the memory to shared. +* +* It is okay to have a shared mapping of memory that is not converted +* properly. handle_mmio() knows how to deal with load_unaligned_zeropad() +* stepping on it. +*/ + x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; + + x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; + x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;pr_info ("Guest detected\n" ); }Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer
Sathyanarayanan Kuppuswamy
Reviewed-byRe: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Hi,
Quoted 26 lines · Tom Lendacky
Quoted 25 lines · Kirill A. Shutemov
Quoted 10 lines · Sathyanarayanan Kuppuswamy
Quoted 5 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 10 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.
Quoted 6 lines
Quoted 4 lines
I don't think AMD affected by
load_unaligned_zeropad()the same way as Intel does. But I'm not sure.Tom, do you have any comments?
Right, shouldn't be an issue for SNP.
Thanks for confirming.
Quoted 18 lines
Thanks,
TomQuoted 14 lines
Quoted 14 lines
Quoted 14 lines
guest.enc_status_change_prepare()called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
guest.enc_status_change_finish()called after adjusting direct mapping and it converts the memory to shared.It is okay to have a shared mapping of memory that is not converted properly.
handle_mmio()knows how to deal withload_unaligned_zeropad()stepping on it.
Rest looks good to me.
Quoted 89 lines
Quoted 89 lines
Quoted 40 lines
Quoted 40 lines
Signed-off-by Kirill A. Shutemov <[email protected]>Fixes 7dbde7631629 ("x86/mm/cpa: Add support for TDX shared memory")---
arch/x86/coco/tdx/tdx.c| 56 ++++++++++++++++++++++++++++++++++++++---1 file changed, 53 insertions(+), 3 deletions(-)arch/x86/coco/tdx/tdx.c +23
-0@@ -840,6 +840,30 @@ static bool tdx_enc_status_changed(unsigned long vaddr, int numpages, bool enc) return true; } +static bool tdx_enc_status_change_prepare(unsigned long vaddr, int numpages, + bool enc) +{ + /* + * Only handle shared->private conversion here. + * See the comment in tdx_early_init(). + */ + if (enc) + return tdx_enc_status_changed(vaddr, numpages, enc); + return true; +} + +static bool tdx_enc_status_change_finish (unsigned long vaddr,int numpages, + bool enc) +{ + /* + * Only handle private->shared conversion here. + * See the comment in tdx_early_init(). + */ + if (!enc) + return tdx_enc_status_changed(vaddr, numpages, enc); + return true; +} + void __init tdx_early_init(void) {Quoted 8 lines
arch/x86/coco/tdx/tdx.c void __init tdx_early_init(void) · line 870
Quoted 6 lines
arch/x86/coco/tdx/tdx.c +0
-1u64 cc_mask;@@ -867,9 +891,35 @@ void __init tdx_early_init(void)*/ physical_mask &= cc_mask - 1; - x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; - x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required;I think you don't need to change the order here.
I wanted to emphasise that the comment is for _prepare/_finish callbacks and I hoped re-order would help with this.
Quoted 36 lines
Quoted 22 lines
arch/x86/coco/tdx/tdx.c +21
-1- x86_platform.guest.enc_status_change_finish = tdx_enc_status_changed; + /* +* Touching privately mapped GPA that is not properly converted to +* private with MapGPA and accepted leads to unrecoverable exit +* to VMM. +* +* load_unaligned_zeropad() can touch memory that is not owned by +* the caller, but just happened to next after the owned memory. +* This load_unaligned_zeropad() behaviour makes it important when +* kernel asks VMM to convert a GPA from shared to private or back. +* Kernel must never have a page mapped into direct mapping (and +* aliases) as private when the GPA is already converted to shared or +* when GPA is not yet converted to private. +* +* guest.enc_status_change_prepare() called before adjusting direct +* mapping and therefore it is responsible for converting the memory +* to private. +* +* guest.enc_status_change_finish() called after adjusting direct +* mapping and it converts the memory to shared. +* +* It is okay to have a shared mapping of memory that is not converted arch/x86/coco/tdx/tdx.c void __init tdx_early_init(void) · line 925
Quoted 10 lines
arch/x86/coco/tdx/tdx.c +8
-0+* properly. handle_mmio() knows how to deal with load_unaligned_zeropad() +* stepping on it. +*/ + x86_platform.guest.enc_status_change_prepare = tdx_enc_status_change_prepare; + x86_platform.guest.enc_status_change_finish = tdx_enc_status_change_finish; + + x86_platform.guest.enc_cache_flush_required = tdx_cache_flush_required; + x86_platform.guest.enc_tlb_flush_required = tdx_tlb_flush_required; pr_info("Guest detected\n"); }--
Sathyanarayanan Kuppuswamy
Linux Kernel Developer
Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer
A message that is not in the archive (sent off-list or not yet received): BYAPR21MB1688EF2A57E90FCE02B82F84D748A@BYAPR21MB1688.namprd21.prod.outlook.com
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 42 lines · Michael Kelley (LINUX)
From: Sathyanarayanan Kuppuswamy <[email protected]>
Sent: Tuesday, May 30, 2023 6:22 AMQuoted 30 lines
Hi,
Quoted 26 lines · Tom Lendacky
Quoted 25 lines · Kirill A. Shutemov
Quoted 10 lines · Sathyanarayanan Kuppuswamy
Quoted 5 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 10 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I don't think AMD affected by
load_unaligned_zeropad()the same way as Intel does. But I'm not sure.Tom, do you have any comments?
Right, shouldn't be an issue for SNP.
Thanks for confirming.
Quoted 8 lines
Tom -- For my education, could you elaborate on why this problem can't occur in an SEV-SNP guest? There's still a window where the direct map PTE and the RMP as maintained by the hypervisor are out-of-sync. If
load_unaligned_zeropad()does a read using the direct map PTE during this out-of-sync window, isn't that going to trap to the hypervisor? How is the scenario is handled from there to provide the zeros toload_unaligned_zeropad()? I need to make sure Hyper-V is doing whatever is needed. :-)
Ah, I think I misunderstood this when it was being talked about. The issue SNP would have would be between setting the c-bit but before the PVALIDATE is issued. Prior to the RMP being updated, referencing the page will generate an #NPF and automatically change the RMP over to private (in KVM). However, after the guest is resumed, the page will not have been validated resulting in a #VC with error code 0x404 being generated, causing the guest to terminate itself.
I suppose, when a 0x404 error code is encountered by the #VC handler, it could call search_exception_tables() and call ex_handler_zeropad() for the EX_TYPE_ZEROPAD type (ex_handler_zeropad is currently static, though).
Thanks,
Tom
Quoted 6 lines
Thanks,
Michael
RE: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
From: Tom Lendacky <[email protected]> Sent: Thursday, June 1, 2023 11:19 AM
Quoted 63 lines · Michael Kelley (LINUX)
Quoted 1 line
From: Sathyanarayanan Kuppuswamy
Quoted 7 lines
Sent: Tuesday, May 30, 2023 6:22 AM
Quoted 6 lines
Hi,
Quoted 2 lines · Tom Lendacky
Quoted 1 line · Kirill A. Shutemov
On Fri, May 26, 2023 at 03:10:56PM -0700, Sathyanarayanan Kuppuswamy
Quoted 34 lines · wrote:
Quoted 24 lines
Quoted 24 lines
Quoted 24 lines
Quoted 10 lines · Kirill A. Shutemov
Quoted 5 lines
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 10 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I don't think AMD affected by
load_unaligned_zeropad()the same way as Intel does. But I'm not sure.Tom, do you have any comments?
Right, shouldn't be an issue for SNP.
Thanks for confirming.
Quoted 8 lines
Tom -- For my education, could you elaborate on why this problem can't occur in an SEV-SNP guest? There's still a window where the direct map PTE and the RMP as maintained by the hypervisor are out-of-sync. If
load_unaligned_zeropad()does a read using the direct map PTE during this out-of-sync window, isn't that going to trap to the hypervisor? How is the scenario is handled from there to provide the zeros toload_unaligned_zeropad()? I need to make sure Hyper-V is doing whatever is needed. :-)Ah, I think I misunderstood this when it was being talked about. The issue SNP would have would be between setting the c-bit but before the PVALIDATE is issued. Prior to the RMP being updated, referencing the page will generate an #NPF and automatically change the RMP over to private (in KVM). However, after the guest is resumed, the page will not have been validated resulting in a #VC with error code
0x404being generated, causing the guest to terminate itself.
Quoted 4 lines
I suppose, when a
0x404error code is encountered by the #VC handler, it could callsearch_exception_tables()and callex_handler_zeropad()for theEX_TYPE_ZEROPADtype (ex_handler_zeropadis currently static, though).
Tom -- Does the above sequence *depend* on the hypervisor doing anything to make it work? I'm not clear on why KVM would automatically change the page over to private. If there's a dependency on the hypervisor doing something, then it seems like we'll need to standardize that "something" across hypervisors, lest we end up with per-hypervisor code in Linux to handle this scenario. And running SEV-SNP with multiple VMPLs probably makes it even more complicated.
Kirill -- Same question about TDX. Does making load_unaligned_zeropad() work in a TDX VM depend on the hypervisor doing anything? Or is the behavior seen by the guest dependent only on architected behavior of the TDX processor?
Looking at this problem from a slightly higher level, and thinking out loud a bit, load_unaligned_zeropad() functionality is provided only for certain
architectures: x86/64, arm, arm64, and PowerPC 64 (little endian). There are fallbacks for architectures that don't support it. With two minor tweaks to Kconfig files, I've built x86 with load_unaligned_zeropad() disabled. Maybe with today's processors the performance benefits are past their prime, and running with it disabled in CoCo VMs is the better solution. Does anyone have a sense of whether the perf impact would be measureable?
If doing the load_unaligned_zeropad() enable/disable at build time is too limiting, maybe it could be runtime based on whether page private/shared state is being enforced. I haven't looked at the details.
Thoughts?
Michael
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 69 lines · Michael Kelley (LINUX)
From: Tom Lendacky <[email protected]> Sent: Thursday, June 1, 2023 11:19 AM
Quoted 63 lines · Michael Kelley (LINUX)
Quoted 1 line
From: Sathyanarayanan Kuppuswamy
Quoted 7 lines
Sent: Tuesday, May 30, 2023 6:22 AM
Quoted 6 lines
Hi,
Quoted 2 lines · Tom Lendacky
Quoted 1 line · Kirill A. Shutemov
On Fri, May 26, 2023 at 03:10:56PM -0700, Sathyanarayanan Kuppuswamy
Quoted 34 lines · wrote:
Quoted 24 lines
Quoted 24 lines
Quoted 24 lines
Quoted 10 lines · Kirill A. Shutemov
Quoted 5 lines
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory./s/to/to be ?
Yep, my bad.
Quoted 10 lines
Quoted 5 lines
This
load_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.I am wondering whether this issue exist in the AMD code?
IMO, you can add some info on the window in
set_memory_encrypted()where this race exists.Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I don't think AMD affected by
load_unaligned_zeropad()the same way as Intel does. But I'm not sure.Tom, do you have any comments?
Right, shouldn't be an issue for SNP.
Thanks for confirming.
Quoted 8 lines
Tom -- For my education, could you elaborate on why this problem can't occur in an SEV-SNP guest? There's still a window where the direct map PTE and the RMP as maintained by the hypervisor are out-of-sync. If
load_unaligned_zeropad()does a read using the direct map PTE during this out-of-sync window, isn't that going to trap to the hypervisor? How is the scenario is handled from there to provide the zeros toload_unaligned_zeropad()? I need to make sure Hyper-V is doing whatever is needed. :-)Ah, I think I misunderstood this when it was being talked about. The issue SNP would have would be between setting the c-bit but before the PVALIDATE is issued. Prior to the RMP being updated, referencing the page will generate an #NPF and automatically change the RMP over to private (in KVM). However, after the guest is resumed, the page will not have been validated resulting in a #VC with error code
0x404being generated, causing the guest to terminate itself.Quoted 4 lines
I suppose, when a
0x404error code is encountered by the #VC handler, it could callsearch_exception_tables()and callex_handler_zeropad()for theEX_TYPE_ZEROPADtype (ex_handler_zeropadis currently static, though).
Quoted 7 lines
Tom -- Does the above sequence *depend* on the hypervisor doing anything to make it work? I'm not clear on why KVM would automatically change the page over to private. If there's a dependency on the hypervisor doing something, then it seems like we'll need to standardize that "something" across hypervisors, lest we end up with per-hypervisor code in Linux to handle this scenario. And running SEV-SNP with multiple VMPLs probably makes it even more complicated.
No, it doesn't depend on the hypervisor doing anything. If the RMP hasn't been updated, then a #VMEXIT(NPF) will be triggered (see APM vol 2, Table 15-39). The hypervisor is free to do what it wants with that, e.g. just resume the guest (and immediately take another #VMEXIT(NPF), possibly). Since it is a different thread/vCPU trying to access the memory than is changing the state of the memory, eventually the guest kernel thread that is changing the state of the memory will issue the page state change to the hypervisor and the other thread can continue.
Thanks,
Tom
Quoted 22 lines
Kirill -- Same question about TDX. Does making
load_unaligned_zeropad()work in a TDX VM depend on the hypervisor doing anything? Or is the behavior seen by the guest dependent only on architected behavior of the TDX processor?Looking at this problem from a slightly higher level, and thinking out loud a bit,
load_unaligned_zeropad()functionality is provided only for certain
architectures: x86/64, arm, arm64, and PowerPC 64 (little endian). There are fallbacks for architectures that don't support it. With two minor tweaks toKconfigfiles, I've built x86 withload_unaligned_zeropad()disabled. Maybe with today's processors the performance benefits are past their prime, and running with it disabled in CoCo VMs is the better solution. Does anyone have a sense of whether the perf impact would be measureable?If doing the
load_unaligned_zeropad()enable/disable at build time is too limiting, maybe it could be runtime based on whether page private/shared state is being enforced. I haven't looked at the details.Thoughts?
Michael
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
In reply to Michael Kelley (LINUX)Quoted 8 lines · Michael Kelley (LINUX)
Tom -- Does the above sequence *depend* on the hypervisor doing anything to make it work? I'm not clear on why KVM would automatically change the page over to private. If there's a dependency on the hypervisor doing something, then it seems like we'll need to standardize that "something" across hypervisors, lest we end up with per-hypervisor code in Linux to handle this scenario. And running SEV-SNP with multiple VMPLs probably makes it even more complicated.
Quoted 4 lines
Kirill -- Same question about TDX. Does making
load_unaligned_zeropad()work in a TDX VM depend on the hypervisor doing anything? Or is the behavior seen by the guest dependent only on architected behavior of the TDX processor?
No, there's no active help from the hypervisor here.
Also, fwiw, the "architected behavior" here is really just the TDX module policy and _arguably_ the hardware Secure-EPT controlled by the TDX module.
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
Quoted 14 lines · Dave Hansen
Quoted 8 lines · Michael Kelley (LINUX)
Tom -- Does the above sequence *depend* on the hypervisor doing anything to make it work? I'm not clear on why KVM would automatically change the page over to private. If there's a dependency on the hypervisor doing something, then it seems like we'll need to standardize that "something" across hypervisors, lest we end up with per-hypervisor code in Linux to handle this scenario. And running SEV-SNP with multiple VMPLs probably makes it even more complicated.
Quoted 4 lines
Kirill -- Same question about TDX. Does making
load_unaligned_zeropad()work in a TDX VM depend on the hypervisor doing anything? Or is the behavior seen by the guest dependent only on architected behavior of the TDX processor?
Quoted 5 lines
No, there's no active help from the hypervisor here.
Also, fwiw, the "architected behavior" here is really just the TDX module policy and _arguably_ the hardware Secure-EPT controlled by the TDX module.
Right. There's nothing VMM can do to change behaviour here. VMM can remove private page, but it will lead to VM termination on access to the page, but VMM controls VM lifecycle anyway.
Signature · 1 line
Kiryl Shutsemau / Kirill A. Shutemov
Re: [PATCHv2 2/3] x86/tdx: Fix race between set_memory_encrypted() and load_unaligned_zeropad()
In reply to Kirill A. ShutemovQuoted 14 lines · Kirill A. Shutemov
Touching privately mapped GPA that is not properly converted to private with MapGPA and accepted leads to unrecoverable exit to VMM.
load_unaligned_zeropad()can touch memory that is not owned by the caller, but just happened to next after the owned memory. Thisload_unaligned_zeropad()behaviour makes it important when kernel asks VMM to convert a GPA from shared to private or back. Kernel must never have a page mapped into direct mapping (and aliases) as private when the GPA is already converted to shared or when GPA is not yet converted to private.
guest.enc_status_change_prepare()called before adjusting direct mapping and therefore it is responsible for converting the memory to private.
Quoted 6 lines
guest.enc_status_change_finish()called after adjusting direct mapping and it converts the memory to shared.It is okay to have a shared mapping of memory that is not converted properly.
handle_mmio()knows how to deal withload_unaligned_zeropad()stepping on it.
Yeah, as other said, the changelog grammar here is a bit funky. Can you fix it up and resend, please?
PATCH v2 3/3 x86/mm: Fix enc_status_change_finish_noop()
In reply to Kirill A. Shutemovenc_status_change_finish_noop() defined as always-fail now which doesn't make sense for noop.
The change doesn't have user-visible effect because it only gets called if the platform has CC_ATTR_MEM_ENCRYPT. All platforms with the attribute override the callback with own implementation.
--- arch/x86/kernel/x86_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/x86_init.c +1 -1
@@ -131,7 +131,7 @@ struct x86_cpuinit_ops x86_cpuinit = {static void default_nmi_init (void ) { };static bool enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) {return true; } -static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return false; } +static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return true; }static bool enc_tlb_flush_required_noop (bool enc) {return false; }static bool enc_cache_flush_required_noop (void ) {return false; }static bool is_private_mmio_noop (u64 addr) {return false; }
Signature · 1 line
2.39.3
Re: [PATCHv2 3/3] x86/mm: Fix enc_status_change_finish_noop()
Quoted 7 lines · Kirill A. Shutemov
enc_status_change_finish_noop()defined as always-fail now which doesn't make sense for noop.The change doesn't have user-visible effect because it only gets called if the platform has
CC_ATTR_MEM_ENCRYPT. All platforms with the attribute override the callback with own implementation.
Quoted 18 lines
Signed-off-by Kirill A. Shutemov <[email protected]>--- arch/x86/kernel/x86_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)arch/x86/kernel/x86_init.c +1
-1@@ -131,7 +131,7 @@ struct x86_cpuinit_ops x86_cpuinit = {static void default_nmi_init (void ) { };static bool enc_status_change_prepare_noop (unsigned long vaddr,int npages,bool enc) {return true; } -static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return false; } +static bool enc_status_change_finish_noop (unsigned long vaddr,int npages,bool enc) {return true; }static bool enc_tlb_flush_required_noop (bool enc) {return false; }static bool enc_cache_flush_required_noop (void ) {return false; }static bool is_private_mmio_noop (u64 addr) {return false; }
Signature · 2 lines
Sathyanarayanan Kuppuswamy Linux Kernel Developer