x86: AMD: Don't flush cache if hardware enforces cache coherency across encryption domains
v3 cover x86: AMD: Don't flush cache if hardware enforces cache coherency across encryption domains
In some hardware implementations, coherency between the encrypted and unencrypted mappings of the same physical page is enforced. In such a system, it is not required for software to flush the page from all CPU caches in the system prior to changing the value of the C-bit for a page. This hardware- enforced cache coherency is indicated by EAX[10] in CPUID leaf 0x8000001f.
Add this as a CPUID feature and skip flushing caches if the feature is present.
v2 -> v3:
Patch# 2: Moves the addition of the CPUID feature from
early_detect_mem_encrypt() to scattered.c.
Patch# 3,4: These two are the split of patch# 3 from v2. Patch# 3
is for non[PATCH 0/4 v3] x86: AMD: Don't flush encrypted pages if hardware enforces cache coherency-SEV encryptions while patch#4 is for SEV encryptions.
[PATCH 1/4 v3] x86: AMD: Replace numeric value for SME CPUID leaf with a [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a [PATCH 3/4 v3] x86: AMD: Don't flush cache if hardware enforces cache [PATCH 4/4 v3] KVM: SVM: Don't flush cache if hardware enforces cache
arch/x86/boot/compressed/mem_encrypt.S | 5 +++-- arch/x86/include/asm/cpufeatures.h | 6 ++++++ arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/scattered.c | 5 +++-- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/svm/sev.c | 3 ++- arch/x86/kvm/svm/svm.c | 4 ++-- arch/x86/mm/mem_encrypt_identity.c | 4 ++-- arch/x86/mm/pat/set_memory.c | 2 +- 9 files changed, 21 insertions(+), 12 deletions(-)
Krish Sadhukhan (4):
x86: AMD: Replace numeric value for SME CPUID leaf with a #define
x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
x86: AMD: Don't flush cache if hardware enforces cache coherency across en
cryption domnains
KVM: SVM: Don't flush cache if hardware enforces cache coherency across en
cryption domains
PATCH v3 1/4 x86: AMD: Replace numeric value for SME CPUID leaf with a #define
--- arch/x86/boot/compressed/mem_encrypt.S | 5 +++-- arch/x86/include/asm/cpufeatures.h | 5 +++++ arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/scattered.c | 4 ++-- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/svm/svm.c | 4 ++-- arch/x86/mm/mem_encrypt_identity.c | 4 ++-- 7 files changed, 16 insertions(+), 10 deletions(-)
arch/x86/boot/compressed/mem_encrypt.S +3 -2
@@ -12,6 +12,7 @@ #include <asm/processor-flags.h> #include <asm/msr.h> #include <asm/asm-offsets.h> +#include <asm/cpufeatures.h> .text .code32
@@ -31,7 +32,7 @@ SYM_FUNC_START(get_sev_encryption_bit) movl $0x80000000, %eax /* CPUID to check the highest leaf */ cpuid - cmpl $0x8000001f, %eax /* See if 0x8000001f is available */ + cmpl $CPUID_AMD_SME, %eax /* See if 0x8000001f is available */ jb .Lno_sev /*
@@ -40,7 +41,7 @@ SYM_FUNC_START(get_sev_encryption_bit) * CPUID Fn8000_001F[EBX] - Bits 5:0 * Pagetable bit position used to indicate encryption */ - movl $0x8000001f, %eax + movl $CPUID_AMD_SME, %eax cpuid bt $1, %eax /* Check if SEV is available */ jnc .Lno_sev
arch/x86/include/asm/cpufeatures.h +5 -0
@@ -10,6 +10,11 @@#include <asm/disabled-features.h> #endif +/* +* AMD CPUID functions +*/ +#define CPUID_AMD_SME 0x8000001f/* Secure Memory Encryption */ +/* * Defines x86 CPU feature bits */
arch/x86/kernel/cpu/amd.c +1 -1
@@ -630,7 +630,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)* will be a value above 32-bits this is still done for * CONFIG_X86_32 so that accurate values are reported. */ - c->x86_phys_bits -= (cpuid_ebx (0x8000001f ) >>6 ) &0x3f ; + c->x86_phys_bits -= (cpuid_ebx (CPUID_AMD_SME ) >>6 ) &0x3f ;if (IS_ENABLED (CONFIG_X86_32 )) goto clear_all;
arch/x86/kernel/cpu/scattered.c +2 -2
@@ -39,8 +39,8 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, - { X86_FEATURE_SME, CPUID_EAX, 0, 0x8000001f, 0 }, - { X86_FEATURE_SEV, CPUID_EAX, 1, 0x8000001f, 0 }, + { X86_FEATURE_SME, CPUID_EAX, 0, CPUID_AMD_SME, 0 }, + { X86_FEATURE_SEV, CPUID_EAX, 1, CPUID_AMD_SME, 0 }, { 0, 0, 0, 0, 0 } };
arch/x86/kvm/cpuid.c +1 -1
@@ -756,7 +756,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->edx =0 ;break ;case 0x80000000 : - entry->eax =min (entry->eax,0x8000001f ); + entry->eax =min (entry->eax,CPUID_AMD_SME );break ;case 0x80000001 :cpuid_entry_override (entry,CPUID_8000_0001_EDX );
arch/x86/kvm/svm/svm.c +2 -2
@@ -749,7 +749,7 @@ static __init void svm_adjust_mmio_mask(void)u64 msr, mask;/* If there is no memory encryption support, use existing mask */ -if (cpuid_eax (0x80000000 ) <0x8000001f ) +if (cpuid_eax (0x80000000 ) <CPUID_AMD_SME )return ;/* If memory encryption is not enabled, use existing mask */
@@ -757,7 +757,7 @@ static __init void svm_adjust_mmio_mask(void)if (!(msr &MSR_K8_SYSCFG_MEM_ENCRYPT ))return ; - enc_bit =cpuid_ebx (0x8000001f ) &0x3f ; + enc_bit =cpuid_ebx (CPUID_AMD_SME ) &0x3f ; mask_bit = boot_cpu_data.x86_phys_bits;/* Increment the mask bit if it is the same as the encryption bit */
arch/x86/mm/mem_encrypt_identity.c +2 -2
@@ -498,7 +498,7 @@ void __init sme_enable(struct boot_params *bp) eax =0x80000000 ; ecx =0 ;native_cpuid (&eax, &ebx, &ecx, &edx); -if (eax <0x8000001f ) +if (eax <CPUID_AMD_SME )return ;#define AMD_SME_BIT BIT(0)
@@ -520,7 +520,7 @@ void __init sme_enable(struct boot_params *bp)* CPUID Fn8000_001F[EBX] * - Bits 5:0 - Pagetable bit position used to indicate encryption */ - eax =0x8000001f ; + eax =CPUID_AMD_SME ; ecx =0 ;native_cpuid (&eax, &ebx, &ecx, &edx);if (!(eax & feature_mask))
Signature · 1 line
2.18.4
Borislav Petkov
maintainerRe: [PATCH 1/4 v3] x86: AMD: Replace numeric value for SME CPUID leaf with a #define
On Fri, Sep 11, 2020 at 07:25:58PM +0000, Krish Sadhukhan wrote:
<-- patches need commit message.
...
arch/x86/kernel/cpu/scattered.c static const struct cpuid_bit cpuid_bits[] = { · line 43
Quoted 12 lines
arch/x86/kernel/cpu/scattered.c +2
-2@@ -39,8 +39,8 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, - { X86_FEATURE_SME, CPUID_EAX, 0, 0x8000001f, 0 }, - { X86_FEATURE_SEV, CPUID_EAX, 1, 0x8000001f, 0 }, + { X86_FEATURE_SME, CPUID_EAX, 0, CPUID_AMD_SME, 0 }, + { X86_FEATURE_SEV, CPUID_EAX, 1, CPUID_AMD_SME, 0 },
So this one gets a name and all the others above don't?
This fact should've given you a hint that there's no need for naming CPUID leafs - it is easier to grep CPU manuals by the values so you can drop this patch.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquettePaolo Bonzini
maintainerRe: [PATCH 1/4 v3] x86: AMD: Replace numeric value for SME CPUID leaf with a #define
Quoted 21 lines · Borislav Petkov
On Fri, Sep 11, 2020 at 07:25:58PM +0000, Krish Sadhukhan wrote:
<-- patches need commit message.
...
arch/x86/kernel/cpu/scattered.c static const struct cpuid_bit cpuid_bits[] = { · line 43
Quoted 12 lines
arch/x86/kernel/cpu/scattered.c +2
-2@@ -39,8 +39,8 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, - { X86_FEATURE_SME, CPUID_EAX, 0, 0x8000001f, 0 }, - { X86_FEATURE_SEV, CPUID_EAX, 1, 0x8000001f, 0 }, + { X86_FEATURE_SME, CPUID_EAX, 0, CPUID_AMD_SME, 0 }, + { X86_FEATURE_SEV, CPUID_EAX, 1, CPUID_AMD_SME, 0 },So this one gets a name and all the others above don't?
Quoted 3 lines
This fact should've given you a hint that there's no need for naming CPUID leafs - it is easier to grep CPU manuals by the values so you can drop this patch.
Also, there'd be confusion between the CPUID_* enum in arch/x86/include/asm/cpufeature.h.
Paolo
PATCH v3 2/4 x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
In reply to Krish SadhukhanIn some hardware implementations, coherency between the encrypted and unencrypted mappings of the same physical page is enforced. In such a system, it is not required for software to flush the page from all CPU caches in the system prior to changing the value of the C-bit for a page. This hardware- enforced cache coherency is indicated by EAX[10] in CPUID leaf 0x8000001f.
--- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/scattered.c | 1 + 2 files changed, 2 insertions(+)
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -293,6 +293,7 @@#define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4)/* "" LFENCE in user entry SWAPGS path */ #define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5)/* "" LFENCE in kernel entry SWAPGS path */ #define X86_FEATURE_SPLIT_LOCK_DETECT (11*32+ 6)/* #AC for split lock */ +#define X86_FEATURE_HW_CACHE_COHERENCY (11*32+ 7)/* AMD hardware-enforced cache coherency */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5)/* AVX512 BFLOAT16 instructions */
arch/x86/kernel/cpu/scattered.c +1 -0
@@ -41,6 +41,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, { X86_FEATURE_SME, CPUID_EAX, 0, CPUID_AMD_SME, 0 }, { X86_FEATURE_SEV, CPUID_EAX, 1, CPUID_AMD_SME, 0 }, + { X86_FEATURE_HW_CACHE_COHERENCY, CPUID_EAX, 10, CPUID_AMD_SME, 0 }, { 0, 0, 0, 0, 0 } };
Signature · 1 line
2.18.4
Re: [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
arch/x86/include/asm/cpufeatures.h line 296
Quoted 10 lines · Krish Sadhukhan
arch/x86/include/asm/cpufeatures.h +1
-0@@ -293,6 +293,7 @@#define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4)/* "" LFENCE in user entry SWAPGS path */ #define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5)/* "" LFENCE in kernel entry SWAPGS path */ #define X86_FEATURE_SPLIT_LOCK_DETECT (11*32+ 6)/* #AC for split lock */ +#define X86_FEATURE_HW_CACHE_COHERENCY (11*32+ 7)/* AMD hardware-enforced cache coherency */
That's an awfully generic name. We generally have "hardware-enforced cache coherency" already everywhere. :)
This probably needs to say something about encryption, or even SEV specifically. I also don't see this bit in the "AMD64 Architecture Programmer’s Manual". Did I look in the wrong spot somehow?
Re: [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
Quoted 13 lines · Dave Hansen
arch/x86/include/asm/cpufeatures.h line 296
Quoted 9 lines · Krish Sadhukhan
arch/x86/include/asm/cpufeatures.h +1
-0@@ -293,6 +293,7 @@#define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4)/* "" LFENCE in user entry SWAPGS path */ #define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5)/* "" LFENCE in kernel entry SWAPGS path */ #define X86_FEATURE_SPLIT_LOCK_DETECT (11*32+ 6)/* #AC for split lock */ +#define X86_FEATURE_HW_CACHE_COHERENCY (11*32+ 7)/* AMD hardware-enforced cache coherency */ That's an awfully generic name. We generally have "hardware-enforced cache coherency" already everywhere. :)
Quoted 2 lines
This probably needs to say something about encryption, or even SEV specifically.
How about X86_FEATURE_ENC_CACHE_COHERENCY ?
Quoted 2 lines
I also don't see this bit in the "AMD64 Architecture Programmer’s Manual". Did I look in the wrong spot somehow?
Section 7.10.6 in APM mentions this.
Re: [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
On 9/11/20 1:10 PM, Krish Sadhukhan wrote:
...
Quoted 9 lines
Quoted 7 lines
Quoted 2 lines
+#define
X86_FEATURE_HW_CACHE_COHERENCY(11*32+ 7) /* AMD hardware-enforced cache coherency */That's an awfully generic name. We generally have "hardware-enforced cache coherency" already everywhere. :)
This probably needs to say something about encryption, or even SEV specifically.
How about
X86_FEATURE_ENC_CACHE_COHERENCY?
I think X86_FEATURE_SME_COHERENT would be the most appropriate name. That bit, as defined, looks totally specific to SME.
Borislav Petkov
maintainerRe: [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
In reply to Krish Sadhukhan+ Tom.
Quoted 1 line · Krish Sadhukhan
+#define
X86_FEATURE_HW_CACHE_COHERENCY(11*32+ 7) /* AMD hardware-enforced cache coherency */
so before you guys paint the bikeshed all kinds of colors :), Tom (CCed) is digging out the official name. (If it is even uglier, we might keep on bikeshedding...).
Once you have that, add the "" after the comment - like X86_FEATURE_FENCE_SWAPGS_USER, for example, so that it doesn't show in /proc/cpuinfo as luserspace doesn't care about hw coherency between enc memory.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH 2/4 v3] x86: AMD: Add hardware-enforced cache coherency as a CPUID feature
Quoted 8 lines · Borislav Petkov
+ Tom.
Quoted 1 line · Krish Sadhukhan
+#define
X86_FEATURE_HW_CACHE_COHERENCY(11*32+ 7) /* AMD hardware-enforced cache coherency */so before you guys paint the bikeshed all kinds of colors :), Tom (CCed) is digging out the official name. (If it is even uglier, we might keep on bikeshedding...).
I believe the official name is something like CoherencyEnforced. Since it's under the 0x8000001f leaf (AMD Secure Encryption), it means that coherency is enforced between the same physical address when referenced with or without the encryption bit (bare metal or in a guest).
So I kind of like the X86_FEATURE_SME_COHERENT suggestion by Dave, even if it also applies to SEV.
Thanks,
Tom
Quoted 8 lines
Once you have that, add the "" after the comment - like
X86_FEATURE_FENCE_SWAPGS_USER, for example, so that it doesn't show in/proc/cpuinfoas luserspace doesn't care about hw coherency between enc memory.Thx.
PATCH v3 3/4 x86: AMD: Don't flush cache if hardware enforces cache coherency across encryption domnains
In reply to Krish SadhukhanIn some hardware implementations, coherency between the encrypted and unencrypted mappings of the same physical page is enforced. In such a system, it is not required for software to flush the page from all CPU caches in the system prior to changing the value of the C-bit for the page.
--- arch/x86/mm/pat/set_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/mm/pat/set_memory.c +1 -1
@@ -1999,7 +1999,7 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)/* * Before changing the encryption attribute, we need to flush caches. */ -cpa_flush (&cpa,1 ); +cpa_flush (&cpa, !this_cpu_has (X86_FEATURE_HW_CACHE_COHERENCY )); ret =__change_page_attr_set_clr (&cpa,1 );
Signature · 1 line
2.18.4
PATCH v3 4/4 KVM: SVM: Don't flush cache if hardware enforces cache coherency across encryption domains
In reply to Krish SadhukhanIn some hardware implementations, coherency between the encrypted and unencrypted mappings of the same physical page in a VM is enforced. In such a system, it is not required for software to flush the VM's page from all CPU caches in the system prior to changing the value of the C-bit for the page.
--- arch/x86/kvm/svm/sev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
arch/x86/kvm/svm/sev.c +2 -1
@@ -384,7 +384,8 @@ static void sev_clflush_pages(struct page *pages[], unsigned long npages)uint8_t *page_virtual;unsigned long i; -if (npages ==0 || pages ==NULL ) +if (this_cpu_has (X86_FEATURE_HW_CACHE_COHERENCY ) || npages ==0 || + pages ==NULL )return ;for (i =0 ; i < npages; i++) {
Signature · 1 line
2.18.4