Enable LKGS instruction
Subsystems (3)
v3 cover Enable LKGS instruction
LKGS instruction is introduced with Intel FRED (flexible return and event delivery) specification https://cdrdv2.intel.com/v1/dl/getContent/678938.
LKGS is independent of FRED, so we enable it as a standalone CPU feature.
LKGS behaves like the MOV to GS instruction except that it loads the base address into the IA32_KERNEL_GS_BASE MSR instead of the GS segment’s descriptor cache, which is exactly what Linux kernel does to load user level GS base. Thus, with LKGS, there is no need to SWAPGS away from the kernel GS base.
Changes since V2:
* add "" not to show "lkgs" in /proc/cpuinfo.
* mark DI as input and output (+D) as in V1, since the exception handler modifies it.
Changes since V1:
* use EX_TYPE_ZERO_REG instead of fixup code in the obsolete .fixup code section.
* add a comment that states the LKGS_DI macro will be repalced with "lkgs %di" once the binutils support LKGS instruction.
H. Peter Anvin (Intel) (6):
x86/cpufeature: add cpu feature bit for LKGS
x86/opcode: add LKGS instruction to x86-opcode-map x86/gsseg: make asm_load_gs_index() take an u16 x86/gsseg: move local_irq_save/restore() into asm_load_gs_index() x86/gsseg: move load_gs_index() to its own header file x86/gsseg: use the LKGS instruction if available for load_gs_index()
arch/x86/entry/entry_64.S | 28 +++++++++--- arch/x86/ia32/ia32_signal.c | 1 + arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/gsseg.h | 56 ++++++++++++++++++++++++ arch/x86/include/asm/mmu_context.h | 1 + arch/x86/include/asm/special_insns.h | 21 --------- arch/x86/kernel/paravirt.c | 1 + arch/x86/kernel/tls.c | 1 + arch/x86/lib/x86-opcode-map.txt | 1 + tools/arch/x86/include/asm/cpufeatures.h | 1 + tools/arch/x86/lib/x86-opcode-map.txt | 1 + 11 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 arch/x86/include/asm/gsseg.h
Signature · 1 line
2.34.1
PATCH v3 1/6 x86/cpufeature: add cpu feature bit for LKGS
From: "H. Peter Anvin (Intel)" <[email protected]>
Add the CPU feature bit for LKGS (Load "Kernel" GS).
LKGS instruction is introduced with Intel FRED (flexible return and event delivery) specificaton
https://cdrdv2.intel.com/v1/dl/getContent/678938.
LKGS behaves like the MOV to GS instruction except that it loads the base address into the IA32_KERNEL_GS_BASE MSR instead of the GS segment’s descriptor cache, which is exactly what Linux kernel does to load a user level GS base. Thus, with LKGS, there is no need to SWAPGS away from the kernel GS base.
--- arch/x86/include/asm/cpufeatures.h | 1 + tools/arch/x86/include/asm/cpufeatures.h | 1 + 2 files changed, 2 insertions(+)
arch/x86/include/asm/cpufeatures.h +1 -0
@@ -308,6 +308,7 @@/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4)/* AVX VNNI instructions */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5)/* AVX512 BFLOAT16 instructions */ +#define X86_FEATURE_LKGS (12*32+ 18)/* "" Load "kernel" (userspace) gs */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ #define X86_FEATURE_CLZERO (13*32+ 0)/* CLZERO instruction */
tools/arch/x86/include/asm/cpufeatures.h +1 -0
@@ -308,6 +308,7 @@/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4)/* AVX VNNI instructions */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5)/* AVX512 BFLOAT16 instructions */ +#define X86_FEATURE_LKGS (12*32+ 18)/* "" Load "kernel" (userspace) gs */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ #define X86_FEATURE_CLZERO (13*32+ 0)/* CLZERO instruction */
Signature · 1 line
2.34.1
PATCH v3 2/6 x86/opcode: add LKGS instruction to x86-opcode-map
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
Add the instruction opcode used by LKGS.
Opcode number is per public FRED draft spec v3.0
https://cdrdv2.intel.com/v1/dl/getContent/678938.
--- arch/x86/lib/x86-opcode-map.txt | 1 + tools/arch/x86/lib/x86-opcode-map.txt | 1 + 2 files changed, 2 insertions(+)
arch/x86/lib/x86-opcode-map.txt +1 -0
@@ -1047,6 +1047,7 @@ GrpTable: Grp6 3: LTR Ew 4: VERR Ew 5: VERW Ew +6: LKGS Ew (F2) EndTable GrpTable: Grp7
tools/arch/x86/lib/x86-opcode-map.txt +1 -0
@@ -1047,6 +1047,7 @@ GrpTable: Grp6 3: LTR Ew 4: VERR Ew 5: VERW Ew +6: LKGS Ew (F2) EndTable GrpTable: Grp7
Signature · 1 line
2.34.1
PATCH v3 3/6 x86/gsseg: make asm_load_gs_index() take an u16
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of load_gs_index() argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.
--- arch/x86/entry/entry_64.S | 2 +- arch/x86/include/asm/special_insns.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +1 -1
@@ -779,7 +779,7 @@ _ASM_NOKPROBE(common_interrupt_return) /* * Reload gs selector with exception handling - * edi: new selector + * di: new selector * * Is in entry.text as it shouldn't be instrumented. */
arch/x86/include/asm/special_insns.h +1 -1
@@ -120,7 +120,7 @@ static inline void native_wbinvd(void) asmvolatile ("wbinvd" : : :"memory" ); } -extern asmlinkage void asm_load_gs_index(unsigned int selector); +extern asmlinkage void asm_load_gs_index(u16 selector);static inline void native_load_gs_index (unsigned int selector) {
Signature · 1 line
2.34.1
RE: [PATCH v3 3/6] x86/gsseg: make asm_load_gs_index() take an u16
From: Xin Li
Quoted 8 lines
Sent: 13 October 2022 21:02
From: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of
load_gs_index()argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.
Using u16 will almost always make the code worse. At some point the value has to be masked and/or extended to ensure an out of range value doesn't appear in a register.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
RE: [PATCH v3 3/6] x86/gsseg: make asm_load_gs_index() take an u16
Quoted 11 lines
Quoted 7 lines
From: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of
load_gs_index()argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.Using u16 will almost always make the code worse.
At some point the value has to be masked and/or extended to ensure an out of range value doesn't appear in a register.
Any potential issue with this patch set?
Quoted 6 lines
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
H. Peter Anvin
reviewerRE: [PATCH v3 3/6] x86/gsseg: make asm_load_gs_index() take an u16
In reply to David LaightQuoted 15 lines · David Laight
From: Xin Li
Quoted 8 lines
Sent: 13 October 2022 21:02
From: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of
load_gs_index()argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.Using u16 will almost always make the code worse. At some point the value has to be masked and/or extended to ensure an out of range value doesn't appear in a register.
Quoted 7 lines
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Is that a general statement or are you actually invoking it in this case? This is about it being a narrowing input, *removing* such constraints.
RE: [PATCH v3 3/6] x86/gsseg: make asm_load_gs_index() take an u16
From: H. Peter Anvin
Quoted 18 lines
Sent: 15 October 2022 03:41
Quoted 15 lines · David Laight
From: Xin Li
Quoted 8 lines
Sent: 13 October 2022 21:02
From: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of
load_gs_index()argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.Using u16 will almost always make the code worse. At some point the value has to be masked and/or extended to ensure an out of range value doesn't appear in a register.
Quoted 4 lines
Quoted 1 line
David
Is that a general statement or are you actually invoking it in this case? This is about it being a narrowing input, *removing* such constraints.
It is a general statement.
You suggested you might get better code.
If fact you'll probably get worse code.
It might not matter here, but ...
Most modern calling conventions use cpu register to pass arguments and results.
So the compiler is required to ensure that u16 values are in range in either the caller or called code (or both).
Just because the domain of a value is small doesn't mean that the best type isn't 'int' or 'unsigned int'.
Additionally (except on x86) any arithmetic on sub-32bit values requires additional instructions to mask the result.
Even on x86-64 if you index an array with an 'int' the compiler has to generate code to sign extend the value to 64 bits. You get better code for 'signed long' or unsigned types. This is probably true for all 64bit architectures.
Since (most) cpu have both sign extending an zero extending loads from memory, it can make sense to use u8 and u16 to reduce the size of structures.
But for function arguments and function locals it almost always makes the code worse.
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
H. Peter Anvin
reviewerRE: [PATCH v3 3/6] x86/gsseg: make asm_load_gs_index() take an u16
Quoted 50 lines · David Laight
From: H. Peter Anvin
Quoted 18 lines
Sent: 15 October 2022 03:41
Quoted 15 lines · David Laight
From: Xin Li
Quoted 8 lines
Sent: 13 October 2022 21:02
From: "H. Peter Anvin (Intel)" <[email protected]>
Let gcc know that only the low 16 bits of
load_gs_index()argument actually matter. It might allow it to create slightly better code. However, do not propagate this into the prototypes of functions that end up being paravirtualized, to avoid unnecessary changes.Using u16 will almost always make the code worse. At some point the value has to be masked and/or extended to ensure an out of range value doesn't appear in a register.
Quoted 4 lines
Quoted 1 line
David
Is that a general statement or are you actually invoking it in this case? This is about it being a narrowing input, *removing* such constraints.
It is a general statement.
You suggested you might get better code.
If fact you'll probably get worse code.
It might not matter here, but ...Most modern calling conventions use cpu register to pass arguments and results.
So the compiler is required to ensure that u16 values are in range in either the caller or called code (or both).
Just because the domain of a value is small doesn't mean that the best type isn't 'int' or 'unsigned int'.Additionally (except on x86) any arithmetic on sub-32bit values requires additional instructions to mask the result.
Even on x86-64 if you index an array with an 'int' the compiler has to generate code to sign extend the value to 64 bits. You get better code for 'signed long' or unsigned types. This is probably true for all 64bit architectures.
Since (most) cpu have both sign extending an zero extending loads from memory, it can make sense to use u8 and u16 to reduce the size of structures.
But for function arguments and function locals it almost always makes the code worse.
Quoted 6 lines
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK Registration No: 1397386 (Wales)
Ok. You are plain incorrect in this case for two reasons:
1. The x86-64 calling convention makes it up to the receiver (callee for arguments, caller for returns) to do such masking of values.
2. The consumer of the values here does not need any masking or extensions.
So this is simply telling the compiler what the programmer knows.
PATCH v3 4/6 x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
The need to disable/enable interrupts around asm_load_gs_index() is a consequence of the implementation. Prepare for using the LKGS instruction, which is locally atomic and therefore doesn't need interrupts disabled.
--- arch/x86/entry/entry_64.S | 26 +++++++++++++++++++++----- arch/x86/include/asm/special_insns.h | 4 ---- 2 files changed, 21 insertions(+), 9 deletions(-)
arch/x86/entry/entry_64.S +21 -5
@@ -778,19 +778,35 @@ SYM_CODE_END(common_interrupt_return) _ASM_NOKPROBE(common_interrupt_return) /* - * Reload gs selector with exception handling + * Reload gs selector with exception handling. This is used only on + * native, so using swapgs, pushf, popf, cli, sti, ... directly is fine. + * * di: new selector + * rax: scratch register * * Is in entry.text as it shouldn't be instrumented. + * + * Note: popf is slow, so use pushf to read IF and then execute cli/sti + * if necessary. */ SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN + pushf + pop %rax + andl $X86_EFLAGS_IF, %eax /* Interrupts enabled? */ + jz 1f + cli +1: swapgs .Lgs_change: ANNOTATE_NOENDBR // error_entry movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs + testl %eax, %eax + jz 3f + sti +3: FRAME_END RET
@@ -799,12 +815,12 @@ SYM_FUNC_START(asm_load_gs_index) swapgs /* switch back to user gs */ .macro ZAP_GS /* This can't be a string because the preprocessor needs to see it. */ - movl $__USER_DS, %eax - movl %eax, %gs + movl $__USER_DS, %edi + movl %edi, %gs .endm ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG - xorl %eax, %eax - movl %eax, %gs + xorl %edi, %edi + movl %edi, %gs jmp 2b _ASM_EXTABLE(.Lgs_change, .Lbad_gs)
arch/x86/include/asm/special_insns.h +0 -4
@@ -124,11 +124,7 @@ extern asmlinkage void asm_load_gs_index(u16 selector);static inline void native_load_gs_index (unsigned int selector) { -unsigned long flags; - -local_irq_save (flags);asm_load_gs_index (selector); -local_irq_restore (flags); }static inline unsigned long __read_cr4 (void )
Signature · 1 line
2.34.1
Re: [PATCH v3 4/6] x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()
Quoted 6 lines · Xin Li
From: "H. Peter Anvin (Intel)" <[email protected]>
The need to disable/enable interrupts around
asm_load_gs_index()is a consequence of the implementation. Prepare for using the LKGS instruction, which is locally atomic and therefore doesn't need interrupts disabled.
*Shudder*. We want less ASM not more.
arch/x86/include/asm/special_insns.h extern asmlinkage void asm_load_gs_index(u16 selector); · line 128
Quoted 8 lines
static inline void
native_load_gs_index(unsigned int selector){arch/x86/include/asm/special_insns.h +0
-4-unsigned long flags; - -local_irq_save (flags);asm_load_gs_index (selector); -local_irq_restore (flags); }
static inline void native_load_gs_index(unsigned int selector)
{
unsigned long flags;
if (cpu_feature_enabled(LKGS)) {
native_lkgs(selector);
} else {
local_irq_save(flags);
asm_load_gs_index(selector);
local_irq_restore(flags);
}
}For paravirt enabled kernels we want during feature detection:
if (cpu_feature_enabled(LKGS)))
pv_ops.cpu.load_gs_index = native_lkgs;No?
Thanks,
tglx
RE: [PATCH v3 4/6] x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()
Quoted 20 lines
arch/x86/include/asm/special_insns.h extern asmlinkage void asm_load_gs_index(u16 selector); · line 128
Quoted 7 lines
static inline void
native_load_gs_index(unsigned int selector){arch/x86/include/asm/special_insns.h +0
-4-unsigned long flags; - -local_irq_save (flags);asm_load_gs_index (selector); -local_irq_restore (flags); }static inline void
native_load_gs_index(unsigned int selector){unsigned long flags; if (cpu_feature_enabled(LKGS)) { native_lkgs(selector); } else { local_irq_save(flags); asm_load_gs_index(selector); local_irq_restore(flags); } }
Quoted 4 lines
For paravirt enabled kernels we want during feature detection:
if (cpu_feature_enabled(LKGS))) pv_ops.cpu.load_gs_index = native_lkgs;
If we use static_cpu_has in native_load_gs_index
if (static_cpu_has(X86_FEATURE_LKGS)) {
native_lkgs(selector);
}We don't have to change pv_ops.cpu.load_gs_index, right?
Thanks!
Xin
Quoted 6 lines
No?
Thanks,
tglx
H. Peter Anvin
reviewerRE: [PATCH v3 4/6] x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()
Quoted 26 lines · On October 18, 2022 10:25:31 AM PDT, "Li, Xin3" <[email protected]> wrote:
Quoted 20 lines
arch/x86/include/asm/special_insns.h extern asmlinkage void asm_load_gs_index(u16 selector); · line 128
Quoted 7 lines
static inline void
native_load_gs_index(unsigned int selector){arch/x86/include/asm/special_insns.h +0
-4-unsigned long flags; - -local_irq_save (flags);asm_load_gs_index (selector); -local_irq_restore (flags); }static inline void
native_load_gs_index(unsigned int selector){unsigned long flags; if (cpu_feature_enabled(LKGS)) { native_lkgs(selector); } else { local_irq_save(flags); asm_load_gs_index(selector); local_irq_restore(flags); } }Quoted 4 lines
For paravirt enabled kernels we want during feature detection:
if (cpu_feature_enabled(LKGS))) pv_ops.cpu.load_gs_index = native_lkgs;If we use
static_cpu_hasinnative_load_gs_index
Quoted 16 lines
if (static_cpu_has(X86_FEATURE_LKGS)) { native_lkgs(selector); }We don't have to change pv_ops.cpu.load_gs_index, right?
Thanks!
XinQuoted 6 lines
No?
Thanks,
tglx
You don't *have* to, but it would mean a branch to a branch, so it would be more efficient. It would strictly be an optimization.
RE: [PATCH v3 4/6] x86/gsseg: move local_irq_save/restore() into asm_load_gs_index()
Quoted 39 lines · On October 18, 2022 10:25:31 AM PDT, "Li, Xin3" <[email protected]> wrote:
Quoted 38 lines
Quoted 20 lines
arch/x86/include/asm/special_insns.h extern asmlinkage void asm_load_gs_index(u16 selector); · line 128
Quoted 7 lines
static inline void
native_load_gs_index(unsigned int selector){arch/x86/include/asm/special_insns.h +0
-4-unsigned long flags; - -local_irq_save (flags);asm_load_gs_index (selector); -local_irq_restore (flags); }static inline void
native_load_gs_index(unsigned int selector){unsigned long flags; if (cpu_feature_enabled(LKGS)) { native_lkgs(selector); } else { local_irq_save(flags); asm_load_gs_index(selector); local_irq_restore(flags); } }Quoted 4 lines
For paravirt enabled kernels we want during feature detection:
if (cpu_feature_enabled(LKGS))) pv_ops.cpu.load_gs_index = native_lkgs;If we use
static_cpu_hasinnative_load_gs_indexif (static_cpu_has(X86_FEATURE_LKGS)) { native_lkgs(selector); }We don't have to change pv_ops.cpu.load_gs_index, right?
Thanks!
XinQuoted 3 lines
No?
Quoted 7 lines
Quoted 4 lines
Quoted 3 lines
Thanks,
tglx
You don't *have* to, but it would mean a branch to a branch, so it would be more efficient. It would strictly be an optimization.
Right, the generated object file shows that static_cpu_has is less efficient than ALTERNATIVE.
PATCH v3 5/6 x86/gsseg: move load_gs_index() to its own header file
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
<asm/cpufeature.h> depends on <asm/special_insns.h>, so in order to be able to use alternatives in native_load_gs_index(), factor it out into a separate header file.
--- arch/x86/ia32/ia32_signal.c | 1 + arch/x86/include/asm/gsseg.h | 32 ++++++++++++++++++++++++++++ arch/x86/include/asm/mmu_context.h | 1 + arch/x86/include/asm/special_insns.h | 17 --------------- arch/x86/kernel/paravirt.c | 1 + arch/x86/kernel/tls.c | 1 + 6 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 arch/x86/include/asm/gsseg.h
arch/x86/ia32/ia32_signal.c +1 -0
@@ -34,6 +34,7 @@#include <asm/sigframe.h> #include <asm/sighandling.h> #include <asm/smap.h> +#include <asm/gsseg.h> static inline void reload_segments (struct sigcontext_32 *sc) {
arch/x86/include/asm/gsseg.h new file +32 -0
@@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_X86_GSSEG_H +#define _ASM_X86_GSSEG_H + +#include <linux/types.h> +#include <asm/processor.h> + +#ifdef CONFIG_X86_64 + +extern asmlinkage void asm_load_gs_index(u16 selector); + +static inline void native_load_gs_index (unsigned int selector) +{ +asm_load_gs_index (selector); +} + +#endif /* CONFIG_X86_64 */ + +#ifndef CONFIG_PARAVIRT_XXL + +static inline void load_gs_index (unsigned int selector) +{ +#ifdef CONFIG_X86_64 +native_load_gs_index (selector); +#else +loadsegment (gs, selector); +#endif +} + +#endif /* CONFIG_PARAVIRT_XXL */ + +#endif /* _ASM_X86_GSSEG_H */
arch/x86/include/asm/mmu_context.h +1 -0
@@ -12,6 +12,7 @@#include <asm/tlbflush.h> #include <asm/paravirt.h> #include <asm/debugreg.h> +#include <asm/gsseg.h> extern atomic64_t last_mm_ctx_id;
arch/x86/include/asm/special_insns.h +0 -17
@@ -120,13 +120,6 @@ static inline void native_wbinvd(void) asmvolatile ("wbinvd" : : :"memory" ); } -extern asmlinkage void asm_load_gs_index(u16 selector); - -static inline void native_load_gs_index (unsigned int selector) -{ -asm_load_gs_index (selector); -} -static inline unsigned long __read_cr4 (void ) {return native_read_cr4 ();
@@ -180,16 +173,6 @@ static inline void wbinvd(void) native_wbinvd(); } - -static inline void load_gs_index (unsigned int selector) -{ -#ifdef CONFIG_X86_64 -native_load_gs_index (selector); -#else -loadsegment (gs, selector); -#endif -} -#endif /* CONFIG_PARAVIRT_XXL */ static inline void clflush (volatile void *__p)
arch/x86/kernel/paravirt.c +1 -0
@@ -32,6 +32,7 @@#include <asm/special_insns.h> #include <asm/tlb.h> #include <asm/io_bitmap.h> +#include <asm/gsseg.h> /* * nop stub, which must not clobber anything *including the stack* to
arch/x86/kernel/tls.c +1 -0
@@ -12,6 +12,7 @@#include <asm/ldt.h> #include <asm/processor.h> #include <asm/proto.h> +#include <asm/gsseg.h> #include "tls.h"
Signature · 1 line
2.34.1
PATCH v3 6/6 x86/gsseg: use the LKGS instruction if available for load_gs_index()
In reply to Xin LiFrom: "H. Peter Anvin (Intel)" <[email protected]>
The LKGS instruction atomically loads a segment descriptor into the %gs descriptor registers, *except* that %gs.base is unchanged, and the base is instead loaded into MSR_IA32_KERNEL_GS_BASE, which is exactly what we want this function to do.
--- arch/x86/include/asm/gsseg.h | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)
arch/x86/include/asm/gsseg.h +25 -1
@@ -3,15 +3,39 @@#define _ASM_X86_GSSEG_H#include <linux/types.h> + +#include <asm/asm.h> +#include <asm/cpufeature.h> +#include <asm/alternative.h> #include <asm/processor.h> +#include <asm/nops.h> #ifdef CONFIG_X86_64 extern asmlinkage void asm_load_gs_index(u16 selector); +/* Replace with "lkgs %di" once binutils support LKGS instruction */ +#define LKGS_DI _ASM_BYTES(0xf2,0x0f,0x00,0xf7) +static inline void native_load_gs_index (unsigned int selector) { -asm_load_gs_index (selector); +u16 sel = selector; + +/* +* Note: the fixup is used for the LKGS instruction, but +* it needs to be attached to the primary instruction sequence +* as it isn't something that gets patched. +* +* %rax is provided to the assembly routine as a scratch +* register. +*/ + asm_inline volatile("1:\n" +ALTERNATIVE ("call asm_load_gs_index\n" , + _ASM_BYTES(0x3e) LKGS_DI, + X86_FEATURE_LKGS) + _ASM_EXTABLE_TYPE_REG(1b, 1b, EX_TYPE_ZERO_REG, %k[sel]) + : ASM_OUTPUT2([sel] "+D" (sel), ASM_CALL_CONSTRAINT) + : : _ASM_AX); }#endif /* CONFIG_X86_64 */
Signature · 1 line
2.34.1