x86: Add and use support for clflushopt
Subsystems (4)
cover x86: Add and use support for clflushopt
This patch series adds support for the new clflushopt instruction and then uses it in a few appropriate places in x86 specific code. It does this using the alternatives mechanism, so any platforms without support for clflushopt will continue to use clflush instead.
clflushopt was announced in the document "Intel Architecture Instruction Set Extensions Programming Reference" with Ref # 319433-018.
http://download-software.intel.com/sites/default/files/managed/50/1a/319433-018.pdf
clflushopt has the same flushing behavior as clflush, but has more relaxed ordering. clflushopt must be explicitly ordered by sfence or mfence.
The inline assembly for clflushopt was implemented using %P so that the generated addresses will always be absolute instead of sometimes being RIP relative. This is necessary for the alternatives code to behave correctly.
Ross Zwisler (4):
x86: Add support for clflushopt
x86: Use clflushopt in clflush_cache_range
x86: Use clflushopt in drm_clflush_page
x86: Use clflushopt in drm_clflush_virt_range
arch/x86/include/asm/cpufeature.h | 1 + arch/x86/include/asm/special_insns.h | 8 ++++++++ arch/x86/mm/pageattr.c | 8 ++++---- drivers/gpu/drm/drm_cache.c | 10 ++++++++-- 4 files changed, 21 insertions(+), 6 deletions(-)
Signature · 1 line
1.7.10.4
PATCH 1/4 x86: Add support for clflushopt
Add support for the new clflushopt instruction. This instruction was announced in the document "Intel Architecture Instruction Set Extensions Programming Reference" with Ref # 319433-018.
http://download-software.intel.com/sites/default/files/managed/50/1a/319433-018.pdf
--- arch/x86/include/asm/cpufeature.h | 1 + arch/x86/include/asm/special_insns.h | 8 ++++++++ 2 files changed, 9 insertions(+)
arch/x86/include/asm/cpufeature.h +1 -0
@@ -219,6 +219,7 @@#define X86_FEATURE_RDSEED (9*32+18)/* The RDSEED instruction */ #define X86_FEATURE_ADX (9*32+19)/* The ADCX and ADOX instructions */ #define X86_FEATURE_SMAP (9*32+20)/* Supervisor Mode Access Prevention */ +#define X86_FEATURE_CLFLSHOPT (9*32+23)/* "clflushopt" CLFLUSHOPT instruction */ /* * BUG word(s)
arch/x86/include/asm/special_insns.h +8 -0
@@ -191,6 +191,14 @@ static inline void clflush(volatile void *__p) asmvolatile ("clflush %0" :"+m" (*(volatile char __force *)__p)); } +static inline void clflushopt (volatile void *__p) +{ + alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", +".byte 0x66; clflush %P0" , +X86_FEATURE_CLFLSHOPT , +"+m" (*(volatile char __force *)__p)); +} +#define nop () asm volatile ("nop")
Signature · 1 line
1.7.10.4
tip-bot for Ross Zwisler
merge bot[tip:x86/cpufeature] x86: Add support for the clflushopt instruction
x86: Add support for the clflushopt instruction
Commit-ID: 171699f7630c92d0a928f83e5fb3aeabe35398c0 Gitweb: http://git.kernel.org/tip/171699f7630c92d0a928f83e5fb3aeabe35398c0 Author: Ross Zwisler <[email protected]> AuthorDate: Wed, 26 Feb 2014 12:06:49 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 27 Feb 2014 08:23:28 -0800
x86: Add support for the clflushopt instruction
Add support for the new clflushopt instruction. This instruction was announced in the document "Intel Architecture Instruction Set Extensions Programming Reference" with Ref # 319433-018.
http://download-software.intel.com/sites/default/files/managed/50/1a/319433-018.pdf
[ hpa: changed the feature flag to simply X86_FEATURE_CLFLUSHOPT - if
that is what we want to report in /proc/cpuinfo anyway... ]
--- arch/x86/include/asm/cpufeature.h | 1 + arch/x86/include/asm/special_insns.h | 8 ++++++++ 2 files changed, 9 insertions(+)
arch/x86/include/asm/cpufeature.h +1 -0
@@ -221,6 +221,7 @@#define X86_FEATURE_RDSEED (9*32+18)/* The RDSEED instruction */ #define X86_FEATURE_ADX (9*32+19)/* The ADCX and ADOX instructions */ #define X86_FEATURE_SMAP (9*32+20)/* Supervisor Mode Access Prevention */ +#define X86_FEATURE_CLFLUSHOPT (9*32+23)/* CLFLUSHOPT instruction */ #define X86_FEATURE_AVX512PF (9*32+26)/* AVX-512 Prefetch */ #define X86_FEATURE_AVX512ER (9*32+27)/* AVX-512 Exponential and Reciprocal */ #define X86_FEATURE_AVX512CD (9*32+28)/* AVX-512 Conflict Detection */
arch/x86/include/asm/special_insns.h +8 -0
@@ -191,6 +191,14 @@ static inline void clflush(volatile void *__p) asmvolatile ("clflush %0" :"+m" (*(volatile char __force *)__p)); } +static inline void clflushopt (volatile void *__p) +{ + alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", +".byte 0x66; clflush %P0" , +X86_FEATURE_CLFLUSHOPT , +"+m" (*(volatile char __force *)__p)); +} +#define nop () asm volatile ("nop")
PATCH 2/4 x86: Use clflushopt in clflush_cache_range
In reply to Ross ZwislerIf clflushopt is available on the system, use it instead of clflush in clflush_cache_range.
--- arch/x86/mm/pageattr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/x86/mm/pageattr.c +4 -4
@@ -125,8 +125,8 @@ within(unsigned long addr, unsigned long start, unsigned long end)* @vaddr: virtual start address * @size: number of bytes to flush * -* clflush is an unordered instruction which needs fencing with mfence -* to avoid ordering issues. +* clflushopt is an unordered instruction which needs fencing with mfence or +* sfence to avoid ordering issues. */ void clflush_cache_range (void *vaddr,unsigned int size) {
@@ -135,11 +135,11 @@ void clflush_cache_range(void *vaddr, unsigned int size)mb ();for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size) -clflush (vaddr); +clflushopt (vaddr);/* * Flush any possible final partial cacheline: */ -clflush (vend); +clflushopt (vend);mb (); }
Signature · 1 line
1.7.10.4
tip-bot for Ross Zwisler
merge bot[tip:x86/cpu] x86: Use clflushopt in clflush_cache_range
x86: Use clflushopt in clflush_cache_range
Commit-ID: 8b80fd8b45458a7d0b726d454c51c5219e0fb2ee Gitweb: http://git.kernel.org/tip/8b80fd8b45458a7d0b726d454c51c5219e0fb2ee Author: Ross Zwisler <[email protected]> AuthorDate: Wed, 26 Feb 2014 12:06:50 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 27 Feb 2014 08:26:10 -0800
x86: Use clflushopt in clflush_cache_range
If clflushopt is available on the system, use it instead of clflush in clflush_cache_range.
--- arch/x86/mm/pageattr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/x86/mm/pageattr.c +4 -4
@@ -126,8 +126,8 @@ within(unsigned long addr, unsigned long start, unsigned long end)* @vaddr: virtual start address * @size: number of bytes to flush * -* clflush is an unordered instruction which needs fencing with mfence -* to avoid ordering issues. +* clflushopt is an unordered instruction which needs fencing with mfence or +* sfence to avoid ordering issues. */ void clflush_cache_range (void *vaddr,unsigned int size) {
@@ -136,11 +136,11 @@ void clflush_cache_range(void *vaddr, unsigned int size)mb ();for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size) -clflush (vaddr); +clflushopt (vaddr);/* * Flush any possible final partial cacheline: */ -clflush (vend); +clflushopt (vend);mb (); }
PATCH 3/4 x86: Use clflushopt in drm_clflush_page
In reply to Ross ZwislerIf clflushopt is available on the system, use it instead of clflush in drm_clflush_page.
--- drivers/gpu/drm/drm_cache.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/drm_cache.c +7 -1
@@ -32,6 +32,12 @@#include <drm/drmP.h> #if defined(CONFIG_X86 ) + +/* +* clflushopt is an unordered instruction which needs fencing with mfence or +* sfence to avoid ordering issues. For drm_clflush_page this fencing happens +* in the caller. +*/ static void drm_clflush_page (struct page *page) {
@@ -44,7 +50,7 @@ drm_clflush_page(struct page *page) page_virtual =kmap_atomic (page);for (i =0 ; i <PAGE_SIZE ; i += size) -clflush (page_virtual + i); +clflushopt (page_virtual + i);kunmap_atomic (page_virtual); }
Signature · 1 line
1.7.10.4
tip-bot for Ross Zwisler
merge bot[tip:x86/cpu] x86: Use clflushopt in drm_clflush_page
x86: Use clflushopt in drm_clflush_page
Commit-ID: 2a0c772f1967565d5b71a98012e74d5a7bddc1c4 Gitweb: http://git.kernel.org/tip/2a0c772f1967565d5b71a98012e74d5a7bddc1c4 Author: Ross Zwisler <[email protected]> AuthorDate: Wed, 26 Feb 2014 12:06:51 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 27 Feb 2014 08:26:20 -0800
x86: Use clflushopt in drm_clflush_page
If clflushopt is available on the system, use it instead of clflush in drm_clflush_page.
--- drivers/gpu/drm/drm_cache.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/drm_cache.c +7 -1
@@ -32,6 +32,12 @@#include <drm/drmP.h> #if defined(CONFIG_X86 ) + +/* +* clflushopt is an unordered instruction which needs fencing with mfence or +* sfence to avoid ordering issues. For drm_clflush_page this fencing happens +* in the caller. +*/ static void drm_clflush_page (struct page *page) {
@@ -44,7 +50,7 @@ drm_clflush_page(struct page *page) page_virtual =kmap_atomic (page);for (i =0 ; i <PAGE_SIZE ; i += size) -clflush (page_virtual + i); +clflushopt (page_virtual + i);kunmap_atomic (page_virtual); }
PATCH 4/4 x86: Use clflushopt in drm_clflush_virt_range
In reply to Ross ZwislerIf clflushopt is available on the system, use it instead of clflush in drm_clflush_virt_range.
--- drivers/gpu/drm/drm_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/drm_cache.c +1 -1
@@ -139,7 +139,7 @@ drm_clflush_virt_range(char *addr, unsigned long length)mb ();for (; addr < end; addr += boot_cpu_data.x86_clflush_size)clflush (addr); -clflush (end -1 ); +clflushopt (end -1 );mb ();return ; }
Signature · 1 line
1.7.10.4
tip-bot for Ross Zwisler
merge bot[tip:x86/cpu] x86: Use clflushopt in drm_clflush_virt_range
x86: Use clflushopt in drm_clflush_virt_range
Commit-ID: 2a0788dc9bc46789ec98aea0f30c6fb420196b12 Gitweb: http://git.kernel.org/tip/2a0788dc9bc46789ec98aea0f30c6fb420196b12 Author: Ross Zwisler <[email protected]> AuthorDate: Wed, 26 Feb 2014 12:06:52 -0700 Committer: H. Peter Anvin <[email protected]> CommitDate: Thu, 27 Feb 2014 08:26:31 -0800
x86: Use clflushopt in drm_clflush_virt_range
If clflushopt is available on the system, use it instead of clflush in drm_clflush_virt_range.
--- drivers/gpu/drm/drm_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/drm_cache.c +1 -1
@@ -139,7 +139,7 @@ drm_clflush_virt_range(char *addr, unsigned long length)mb ();for (; addr < end; addr += boot_cpu_data.x86_clflush_size)clflush (addr); -clflush (end -1 ); +clflushopt (end -1 );mb ();return ; }