x86/alternatives changes for v5.1
Linus,
Please pull the latest x86-alternatives-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86-alternatives-for-linus
# HEAD: 093ae8f9a86a974c920b613860f1f7fd5bbd70ab x86/TSC: Use RDTSCP
Small RDTSCP opimization, enabled by the newly added ALTERNATIVE_3(), and other small improvements.
Thanks,
Ingo
------------------>
Borislav Petkov (4):
x86/alternatives: Add macro comments x86/alternatives: Print containing function x86/alternatives: Add an ALTERNATIVE_3() macro x86/TSC: Use RDTSCP
arch/x86/include/asm/alternative.h | 39 +++++++++++++++++++++++++++++++------- arch/x86/include/asm/msr.h | 16 ++++++++++++++-- arch/x86/kernel/alternative.c | 4 ++-- 3 files changed, 48 insertions(+), 11 deletions(-)
arch/x86/include/asm/alternative.h +32 -7
@@ -94,13 +94,12 @@ static inline int alternatives_text_reserved(void *start, void *end)#define alt_total_slen alt_end_marker"b-661b"#define alt_rlen (num) e_replacement(num)"f-"b_replacement(num)"f" -#define __OLDINSTR (oldinstr, num) \ +#define OLDINSTR (oldinstr, num) \ + "# ALT: oldnstr\n" \ "661:\n\t" oldinstr "\n662:\n" \ + "# ALT: padding\n" \ ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \ - "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" - -#define OLDINSTR (oldinstr, num) \ - __OLDINSTR(oldinstr, num) \ + "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \ alt_end_marker ":\n"/*
@@ -116,11 +115,23 @@ static inline int alternatives_text_reserved(void *start, void *end)* additionally longer than the first replacement alternative. */ #define OLDINSTR_2 (oldinstr, num1, num2) \ + "# ALT: oldinstr2\n" \ "661:\n\t" oldinstr "\n662:\n" \ + "# ALT: padding2\n" \ ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \ "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \ alt_end_marker ":\n" +#define OLDINSTR_3 (oldinsn, n1, n2, n3) \ + "# ALT: oldinstr3\n" \ + "661:\n\t" oldinsn "\n662:\n" \ + "# ALT: padding3\n" \ + ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ + " - (" alt_slen ")) > 0) * " \ + "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \ + " - (" alt_slen ")), 0x90\n" \ + alt_end_marker ":\n" +#define ALTINSTR_ENTRY (feature, num) \ " .long 661b - .\n"/* label */ \ " .long " b_replacement(num)"f - .\n" /* new instruction */ \
@@ -129,8 +140,9 @@ static inline int alternatives_text_reserved(void *start, void *end) " .byte " alt_rlen(num) "\n" /* replacement len */ \ " .byte " alt_pad_len "\n" /* pad len */ -#define ALTINSTR_REPLACEMENT (newinstr, feature, num)/* replacement */ \ - b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t" +#define ALTINSTR_REPLACEMENT (newinstr, feature, num)/* replacement */ \ + "# ALT: replacement " #num "\n" \ + b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"/* alternative assembly primitive: */ #define ALTERNATIVE (oldinstr, newinstr, feature) \
@@ -153,6 +165,19 @@ static inline int alternatives_text_reserved(void *start, void *end) ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ ".popsection\n" +#define ALTERNATIVE_3 (oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \ + OLDINSTR_3(oldinsn, 1, 2, 3) \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feat1, 1) \ + ALTINSTR_ENTRY(feat2, 2) \ + ALTINSTR_ENTRY(feat3, 3) \ + ".popsection\n" \ + ".pushsection .altinstr_replacement, \"ax\"\n" \ + ALTINSTR_REPLACEMENT(newinsn1, feat1, 1) \ + ALTINSTR_REPLACEMENT(newinsn2, feat2, 2) \ + ALTINSTR_REPLACEMENT(newinsn3, feat3, 3) \ + ".popsection\n" +/* * Alternative instructions for different CPU types or capabilities. *
arch/x86/include/asm/msr.h +14 -2
@@ -217,6 +217,8 @@ static __always_inline unsigned long long rdtsc(void)*/ static __always_inline unsigned long long rdtsc_ordered (void ) { +DECLARE_ARGS (val, low, high); +/* * The RDTSC instruction is not ordered relative to memory * access. The Intel SDM and the AMD APM are both vague on this
@@ -227,9 +229,19 @@ static __always_inline unsigned long long rdtsc_ordered(void)* ordering guarantees as reading from a global memory location * that some other imaginary CPU is updating continuously with a * time stamp. +* +* Thus, use the preferred barrier on the respective CPU, aiming for +* RDTSCP as the default. */ -barrier_nospec (); -return rdtsc (); + asm volatile(ALTERNATIVE_3("rdtsc", + "mfence; rdtsc", X86_FEATURE_MFENCE_RDTSC, + "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC, + "rdtscp", X86_FEATURE_RDTSCP) + : EAX_EDX_RET(val, low, high) + /* RDTSCP clobbers ECX with MSR_TSC_AUX. */ + :: "ecx"); + +return EAX_EDX_VAL (val, low, high); }static inline unsigned long long native_read_pmc (int counter)
arch/x86/kernel/alternative.c +2 -2
@@ -393,10 +393,10 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,continue ; } -DPRINTK ("feat: %d*32+%d, old: (%px len: %d), repl: (%px, len: %d), pad: %d" , +DPRINTK ("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d" , a->cpuid >>5 , a->cpuid &0x1f , - instr, a->instrlen, + instr, instr, a->instrlen, replacement, a->replacementlen, a->padlen);DUMP_BYTES (instr, a->instrlen,"%px: old_insn: " , instr);
The pull request you sent on Wed, 6 Mar 2019 10:22:11 +0100:
The pull request you sent on Wed, 6 Mar 2019 10:22:11 +0100:
Quoted 1 line
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.gitx86-alternatives-for-linus
has been merged into torvalds/linux.git:
https://git.kernel.org/torvalds/c/6ea98b4baa1c9089d7a035ebccb993e03d1ac57f
Thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.wiki.kernel.org/userdoc/prtracker