x86/sev: Make enc_dec_hypercall() accept a size instead of npages
Subsystems (4)
PATCH v2 1/1 x86/sev: Make enc_dec_hypercall() accept a size instead of npages
enc_dec_hypercall() accepted a page count instead of a size, which forced its callers to round up. As a result, non-page aligned vaddrs caused pages to be spuriously marked as decrypted via the encryption status hypercall, which in turn caused consistent corruption of pages during live migration. Live migration requires accurate encryption status information to avoid migrating pages from the wrong perspective.
--- arch/x86/include/asm/mem_encrypt.h | 6 +++--- arch/x86/kernel/kvm.c | 4 +--- arch/x86/mm/mem_encrypt_amd.c | 13 ++++++------- 3 files changed, 10 insertions(+), 13 deletions(-)
arch/x86/include/asm/mem_encrypt.h +3 -3
@@ -50,8 +50,8 @@ void __init sme_enable(struct boot_params *bp);int __init early_set_memory_decrypted (unsigned long vaddr,unsigned long size);int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size); -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages, -bool enc); +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr, +unsigned long size,bool enc);void __init mem_encrypt_free_decrypted_mem (void );
@@ -85,7 +85,7 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;static inline int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size) {return 0 ; }static inline void __init -early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) {} +early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {}static inline void mem_encrypt_free_decrypted_mem (void ) { }
arch/x86/kernel/kvm.c +1 -3
@@ -966,10 +966,8 @@ static void __init kvm_init_platform(void)* Ensure that _bss_decrypted section is marked as decrypted in the * shared pages list. */ - nr_pages =DIV_ROUND_UP (__end_bss_decrypted - __start_bss_decrypted, -PAGE_SIZE );early_set_mem_enc_dec_hypercall ((unsigned long )__start_bss_decrypted, - nr_pages,0 ); + __end_bss_decrypted - __start_bss_decrypted,0 );/* * If not booted using EFI, enable Live migration support.
arch/x86/mm/mem_encrypt_amd.c +6 -7
@@ -288,11 +288,10 @@ static bool amd_enc_cache_flush_required(void)return !cpu_feature_enabled (X86_FEATURE_SME_COHERENT ); } -static void enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +static void enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {#ifdef CONFIG_PARAVIRT -unsigned long sz = npages <<PAGE_SHIFT ; -unsigned long vaddr_end = vaddr + sz; +unsigned long vaddr_end = vaddr + size;while (vaddr < vaddr_end) {int psize, pmask, level;
@@ -342,7 +341,7 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool esnp_set_memory_private (vaddr, npages);if (!cc_platform_has (CC_ATTR_HOST_MEM_ENCRYPT )) -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, npages <<PAGE_SHIFT , enc);return true; }
@@ -466,7 +465,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, ret =0 ; -early_set_mem_enc_dec_hypercall (start,PAGE_ALIGN (size) >>PAGE_SHIFT , enc); +early_set_mem_enc_dec_hypercall (start, size, enc); out:__flush_tlb_all ();return ret;
@@ -482,9 +481,9 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)return early_set_memory_enc_dec (vaddr, size, true); } -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) { -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, size, enc); }void __init sme_early_init (void )
Signature · 1 line
2.42.0.rc1.204.g551eb34607-goog
Quoted 10 lines · Steve Rutherford
enc_dec_hypercall()accepted a page count instead of a size, which forced its callers to round up. As a result, non-page aligned vaddrs caused pages to be spuriously marked as decrypted via the encryption status hypercall, which in turn caused consistent corruption of pages during live migration. Live migration requires accurate encryption status information to avoid migrating pages from the wrong perspective.Fixes 064ce6c550a0 ("mm: x86: Invoke hypercall when page encryption status is changed")Signed-off-by Steve Rutherford <[email protected]>
Quoted 94 lines
--- arch/x86/include/asm/mem_encrypt.h | 6 +++--- arch/x86/kernel/kvm.c | 4 +--- arch/x86/mm/mem_encrypt_amd.c | 13 ++++++------- 3 files changed, 10 insertions(+), 13 deletions(-)arch/x86/include/asm/mem_encrypt.h +3
-3@@ -50,8 +50,8 @@ void __init sme_enable(struct boot_params *bp);int __init early_set_memory_decrypted (unsigned long vaddr,unsigned long size);int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size); -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages, -bool enc); +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr, +unsigned long size,bool enc);void __init mem_encrypt_free_decrypted_mem (void );@@ -85,7 +85,7 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;static inline int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size) {return 0 ; }static inline void __init -early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) {} +early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {}static inline void mem_encrypt_free_decrypted_mem (void ) { }arch/x86/kernel/kvm.c +1
-3@@ -966,10 +966,8 @@ static void __init kvm_init_platform(void)* Ensure that _bss_decrypted section is marked as decrypted in the * shared pages list. */ - nr_pages =DIV_ROUND_UP (__end_bss_decrypted - __start_bss_decrypted, -PAGE_SIZE );early_set_mem_enc_dec_hypercall ((unsigned long )__start_bss_decrypted, - nr_pages,0 ); + __end_bss_decrypted - __start_bss_decrypted,0 );/* * If not booted using EFI, enable Live migration support. arch/x86/mm/mem_encrypt_amd.c +6
-7@@ -288,11 +288,10 @@ static bool amd_enc_cache_flush_required(void)return !cpu_feature_enabled (X86_FEATURE_SME_COHERENT ); } -static void enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +static void enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {#ifdef CONFIG_PARAVIRT -unsigned long sz = npages <<PAGE_SHIFT ; -unsigned long vaddr_end = vaddr + sz; +unsigned long vaddr_end = vaddr + size;while (vaddr < vaddr_end) {int psize, pmask, level;@@ -342,7 +341,7 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool esnp_set_memory_private (vaddr, npages);if (!cc_platform_has (CC_ATTR_HOST_MEM_ENCRYPT )) -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, npages <<PAGE_SHIFT , enc);return true; }@@ -466,7 +465,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, ret =0 ; -early_set_mem_enc_dec_hypercall (start,PAGE_ALIGN (size) >>PAGE_SHIFT , enc); +early_set_mem_enc_dec_hypercall (start, size, enc); out:__flush_tlb_all ();return ret;@@ -482,9 +481,9 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)return early_set_memory_enc_dec (vaddr, size, true); } -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) { -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, size, enc); }void __init sme_early_init (void )
Quoted 94 lines · Steve Rutherford
enc_dec_hypercall()accepted a page count instead of a size, which forced its callers to round up. As a result, non-page aligned vaddrs caused pages to be spuriously marked as decrypted via the encryption status hypercall, which in turn caused consistent corruption of pages during live migration. Live migration requires accurate encryption status information to avoid migrating pages from the wrong perspective.Fixes 064ce6c550a0 ("mm: x86: Invoke hypercall when page encryption status is changed")Signed-off-by Steve Rutherford <[email protected]>--- arch/x86/include/asm/mem_encrypt.h | 6 +++--- arch/x86/kernel/kvm.c | 4 +--- arch/x86/mm/mem_encrypt_amd.c | 13 ++++++------- 3 files changed, 10 insertions(+), 13 deletions(-)arch/x86/include/asm/mem_encrypt.h +3
-3@@ -50,8 +50,8 @@ void __init sme_enable(struct boot_params *bp);int __init early_set_memory_decrypted (unsigned long vaddr,unsigned long size);int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size); -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages, -bool enc); +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr, +unsigned long size,bool enc);void __init mem_encrypt_free_decrypted_mem (void );@@ -85,7 +85,7 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;static inline int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size) {return 0 ; }static inline void __init -early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) {} +early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {}static inline void mem_encrypt_free_decrypted_mem (void ) { }arch/x86/kernel/kvm.c +1
-3@@ -966,10 +966,8 @@ static void __init kvm_init_platform(void)* Ensure that _bss_decrypted section is marked as decrypted in the * shared pages list. */ - nr_pages =DIV_ROUND_UP (__end_bss_decrypted - __start_bss_decrypted, -PAGE_SIZE );early_set_mem_enc_dec_hypercall ((unsigned long )__start_bss_decrypted, - nr_pages,0 ); + __end_bss_decrypted - __start_bss_decrypted,0 );/* * If not booted using EFI, enable Live migration support. arch/x86/mm/mem_encrypt_amd.c +4
-5@@ -288,11 +288,10 @@ static bool amd_enc_cache_flush_required(void)return !cpu_feature_enabled (X86_FEATURE_SME_COHERENT ); } -static void enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +static void enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {#ifdef CONFIG_PARAVIRT -unsigned long sz = npages <<PAGE_SHIFT ; -unsigned long vaddr_end = vaddr + sz; +unsigned long vaddr_end = vaddr + size;while (vaddr < vaddr_end) {int psize, pmask, level;@@ -342,7 +341,7 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool esnp_set_memory_private (vaddr, npages);if (!cc_platform_has (CC_ATTR_HOST_MEM_ENCRYPT )) -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, npages <<PAGE_SHIFT , enc);return true; }@@ -466,7 +465,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, ret =0 ; -early_set_mem_enc_dec_hypercall (start,PAGE_ALIGN (size) >>PAGE_SHIFT , enc); +early_set_mem_enc_dec_hypercall (start, size, enc); out:__flush_tlb_all ();return ret;@@ -482,9 +481,9 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)return early_set_memory_enc_dec (vaddr, size, true);
arch/x86/mm/mem_encrypt_amd.c int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) · line 489
Quoted 10 lines
arch/x86/mm/mem_encrypt_amd.c +2
-2} -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) { -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, size, enc); }void __init sme_early_init (void )
Also had this thought to avoid passing the page boundaries calculation with npages in-place of existing size based, but no strong opinions.
This seems even better. Thanks!
Reuploading v3 with Cc: [email protected].
Quoted 113 lines · Pankaj
Quoted 94 lines · Steve Rutherford
enc_dec_hypercall()accepted a page count instead of a size, which forced its callers to round up. As a result, non-page aligned vaddrs caused pages to be spuriously marked as decrypted via the encryption status hypercall, which in turn caused consistent corruption of pages during live migration. Live migration requires accurate encryption status information to avoid migrating pages from the wrong perspective.Fixes 064ce6c550a0 ("mm: x86: Invoke hypercall when page encryption status is changed")Signed-off-by Steve Rutherford <[email protected]>--- arch/x86/include/asm/mem_encrypt.h | 6 +++--- arch/x86/kernel/kvm.c | 4 +--- arch/x86/mm/mem_encrypt_amd.c | 13 ++++++------- 3 files changed, 10 insertions(+), 13 deletions(-)arch/x86/include/asm/mem_encrypt.h +3
-3@@ -50,8 +50,8 @@ void __init sme_enable(struct boot_params *bp);int __init early_set_memory_decrypted (unsigned long vaddr,unsigned long size);int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size); -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages, -bool enc); +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr, +unsigned long size,bool enc);void __init mem_encrypt_free_decrypted_mem (void );@@ -85,7 +85,7 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;static inline int __init early_set_memory_encrypted (unsigned long vaddr,unsigned long size) {return 0 ; }static inline void __init -early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) {} +early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {}static inline void mem_encrypt_free_decrypted_mem (void ) { }arch/x86/kernel/kvm.c +1
-3@@ -966,10 +966,8 @@ static void __init kvm_init_platform(void)* Ensure that _bss_decrypted section is marked as decrypted in the * shared pages list. */ - nr_pages =DIV_ROUND_UP (__end_bss_decrypted - __start_bss_decrypted, -PAGE_SIZE );early_set_mem_enc_dec_hypercall ((unsigned long )__start_bss_decrypted, - nr_pages,0 ); + __end_bss_decrypted - __start_bss_decrypted,0 );/* * If not booted using EFI, enable Live migration support. arch/x86/mm/mem_encrypt_amd.c +4
-5@@ -288,11 +288,10 @@ static bool amd_enc_cache_flush_required(void)return !cpu_feature_enabled (X86_FEATURE_SME_COHERENT ); } -static void enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +static void enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) {#ifdef CONFIG_PARAVIRT -unsigned long sz = npages <<PAGE_SHIFT ; -unsigned long vaddr_end = vaddr + sz; +unsigned long vaddr_end = vaddr + size;while (vaddr < vaddr_end) {int psize, pmask, level;@@ -342,7 +341,7 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool esnp_set_memory_private (vaddr, npages);if (!cc_platform_has (CC_ATTR_HOST_MEM_ENCRYPT )) -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, npages <<PAGE_SHIFT , enc);return true; }@@ -466,7 +465,7 @@ static int __init early_set_memory_enc_dec(unsigned long vaddr, ret =0 ; -early_set_mem_enc_dec_hypercall (start,PAGE_ALIGN (size) >>PAGE_SHIFT , enc); +early_set_mem_enc_dec_hypercall (start, size, enc); out:__flush_tlb_all ();return ret;@@ -482,9 +481,9 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)return early_set_memory_enc_dec (vaddr, size, true);arch/x86/mm/mem_encrypt_amd.c int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size) · line 489
Quoted 10 lines
arch/x86/mm/mem_encrypt_amd.c +2
-2} -void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,int npages,bool enc) +void __init early_set_mem_enc_dec_hypercall (unsigned long vaddr,unsigned long size,bool enc) { -enc_dec_hypercall (vaddr, npages, enc); +enc_dec_hypercall (vaddr, size, enc); }void __init sme_early_init (void )Also had this thought to avoid passing the page boundaries calculation with npages in-place of existing size based, but no strong opinions.
This seems even better. Thanks!
Reviewed-by Pankaj Gupta <[email protected]>