x86,objtool: Optimize !RETPOLINE
Subsystems (3)
Peter Zijlstra
maintainercover x86,objtool: Optimize !RETPOLINE
Hi,
Now that Juergen's paravirt rework, which included the negative alternative stuff, landed in tip, here's a respin of my retpoline patches.
The main feature is replacing the compiler generated (tail) calls to __x86_indirect_thunk_\reg with an ALTERNATIVE that replaces them with regular indirect call instructions, eliding the double jump.
Patches rely on tip/objtool/core and tip/x86/alternatives
Peter Zijlstra
maintainerPATCH 1/9 x86/retpoline: Simplify retpolines
Due to commit c9c324dc22aa ("objtool: Support stack layout changes in alternatives"), it is possible to simplify the retpolines.
Currently our retpolines consist of 2 symbols, __x86_indirect_thunk_\reg, which is the compiler target, and __x86_retpoline_\reg, which is the actual retpoline. Both are consecutive in code and aligned such that for any one register they both live in the same cacheline:
0000000000000000 <__x86_indirect_thunk_rax>:
0: ff e0 jmpq *%rax 2: 90 nop 3: 90 nop 4: 90 nop
0000000000000005 <__x86_retpoline_rax>:
5: e8 07 00 00 00 callq 11 <__x86_retpoline_rax+0xc> a: f3 90 pause c: 0f ae e8 lfence f: eb f9 jmp a <__x86_retpoline_rax+0x5> 11: 48 89 04 24 mov %rax,(%rsp) 15: c3 retq 16: 66 2e 0f 1f 84 00 00 00 00 00 nopw %cs:0x0(%rax,%rax,1)
The thunk is an alternative_2, where one option is a jmp to the retpoline. This was done so that objtool didn't need to deal with alternatives with stack ops. But that problem has been solved, so now it is possible to fold the entire retpoline into the alternative to simplify and consolidate unused bytes:
0000000000000000 <__x86_indirect_thunk_rax>:
0: ff e0 jmpq *%rax 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: 66 66 2e 0f 1f 84 00 00 00 00 00 data16 nopw %cs:0x0(%rax,%rax,1) 1c: 0f 1f 40 00 nopl 0x0(%rax)
Notice that since the longest alternative sequence is now:
0: e8 07 00 00 00 callq c <.altinstr_replacement+0xc> 5: f3 90 pause 7: 0f ae e8 lfence a: eb f9 jmp 5 <.altinstr_replacement+0x5> c: 48 89 04 24 mov %rax,(%rsp) 10: c3 retq
17 bytes, we have 15 bytes NOP at the end of our 32 byte slot.
--- arch/x86/include/asm/asm-prototypes.h | 7 ------- arch/x86/include/asm/nospec-branch.h | 6 +++--- arch/x86/lib/retpoline.S | 34 +++++++++++++++++----------------- tools/objtool/check.c | 3 +-- 4 files changed, 21 insertions(+), 29 deletions(-)
arch/x86/include/asm/asm-prototypes.h +0 -7
@@ -22,15 +22,8 @@ extern void cmpxchg8b_emu(void);#define DECL_INDIRECT_THUNK (reg) \ extern asmlinkage void __x86_indirect_thunk_ ## reg (void); -#define DECL_RETPOLINE (reg) \ - extern asmlinkage void __x86_retpoline_ ## reg (void); -#undef GEN#define GEN (reg) DECL_INDIRECT_THUNK(reg)#include <asm/GEN-for-each-reg.h> -#undef GEN -#define GEN (reg) DECL_RETPOLINE(reg) -#include <asm/GEN-for-each-reg.h> -#endif /* CONFIG_RETPOLINE */
arch/x86/include/asm/nospec-branch.h +3 -3
@@ -81,7 +81,7 @@ .macro JMP_NOSPEC reg:req#ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ - __stringify(jmp __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \ + __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD#else jmp *%\reg
@@ -91,7 +91,7 @@ .macro CALL_NOSPEC reg:req#ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ - __stringify(call __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \ + __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD#else call *%\reg
@@ -129,7 +129,7 @@ALTERNATIVE_2 ( \ANNOTATE_RETPOLINE_SAFE \"call *%[thunk_target]\n" , \ -"call __x86_retpoline_%V[thunk_target]\n" , \ +"call __x86_indirect_thunk_%V[thunk_target]\n" , \X86_FEATURE_RETPOLINE , \"lfence;\n" \ ANNOTATE_RETPOLINE_SAFE \
arch/x86/lib/retpoline.S +17 -17
@@ -10,27 +10,31 @@ #include <asm/unwind_hints.h> #include <asm/frame.h> -.macro THUNK reg - .section .text.__x86.indirect_thunk - - .align 32 -SYM_FUNC_START(__x86_indirect_thunk_\reg) - JMP_NOSPEC \reg -SYM_FUNC_END(__x86_indirect_thunk_\reg) - -SYM_FUNC_START_NOALIGN(__x86_retpoline_\reg) +.macro RETPOLINE reg ANNOTATE_INTRA_FUNCTION_CALL - call .Ldo_rop_\@ + call .Ldo_rop_\@ .Lspec_trap_\@: UNWIND_HINT_EMPTY pause lfence - jmp .Lspec_trap_\@ + jmp .Lspec_trap_\@ .Ldo_rop_\@: - mov %\reg, (%_ASM_SP) + mov %\reg, (%_ASM_SP) UNWIND_HINT_FUNC ret -SYM_FUNC_END(__x86_retpoline_\reg) +.endm + +.macro THUNK reg + .section .text.__x86.indirect_thunk + + .align 32 +SYM_FUNC_START(__x86_indirect_thunk_\reg) + + ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ + __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + +SYM_FUNC_END(__x86_indirect_thunk_\reg) .endm
@@ -48,7 +52,6 @@ SYM_FUNC_END(__x86_retpoline_\reg) #define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) #define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) -#define EXPORT_RETPOLINE(reg) __EXPORT_THUNK(__x86_retpoline_ ## reg) #undef GEN #define GEN(reg) THUNK reg
@@ -58,6 +61,3 @@ SYM_FUNC_END(__x86_retpoline_\reg) #define GEN(reg) EXPORT_THUNK(reg) #include <asm/GEN-for-each-reg.h> -#undef GEN -#define GEN(reg) EXPORT_RETPOLINE(reg) -#include <asm/GEN-for-each-reg.h>
tools/objtool/check.c +1 -2
@@ -800,8 +800,7 @@ static int add_jump_destinations(struct }else if (reloc->sym->type ==STT_SECTION ) { dest_sec = reloc->sym->sec; dest_off =arch_dest_reloc_offset (reloc->addend); - }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 ) || - !strncmp (reloc->sym->name,"__x86_retpoline_" ,16 )) { + }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 )) {/* * Retpoline jumps are really dynamic jumps in * disguise, so convert them accordingly.
Peter Zijlstra
maintainerPATCH 2/9 objtool: Correctly handle retpoline thunk calls
In reply to Peter ZijlstraJust like JMP handling, convert a direct CALL to a retpoline thunk into a retpoline safe indirect CALL.
--- tools/objtool/check.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
tools/objtool/check.c +12 -0
@@ -953,6 +953,18 @@ static int add_call_destinations(struct dest_off);return -1 ; } + + }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 )) { +/* +* Retpoline calls are really dynamic calls in +* disguise, so convert them accodingly. +*/ + insn->type =INSN_CALL_DYNAMIC ; + insn->retpoline_safe = true; + +remove_insn_ops (insn); +continue ; + }else insn->call_dest = reloc->sym;
Re: [PATCH 2/9] objtool: Correctly handle retpoline thunk calls
Quoted 11 lines · Peter Zijlstra
Just like JMP handling, convert a direct CALL to a retpoline thunk into a retpoline safe indirect CALL.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- tools/objtool/check.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
tools/objtool/check.c static int add_call_destinations(struct · line 960
Quoted 8 lines
tools/objtool/check.c +5
-0@@ -953,6 +953,18 @@ static int add_call_destinations(struct dest_off);return -1 ; } + + }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 )) { +/* +* Retpoline calls are really dynamic calls in +* disguise, so convert them accodingly.
s/accodingly/accordingly/
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 3/9 objtool: Per arch retpoline naming
In reply to Peter ZijlstraThe __x86_indirect_ naming is obviously not generic. Shorten to allow matching some additional magic names later.
--- tools/objtool/arch/x86/decode.c | 5 +++++ tools/objtool/check.c | 9 +++++++-- tools/objtool/include/objtool/arch.h | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-)
tools/objtool/arch/x86/decode.c +5 -0
@@ -690,3 +690,8 @@ int arch_decode_hint_reg(struct instructreturn 0 ; } + +bool arch_is_retpoline (struct symbol *sym) +{ +return !strncmp (sym->name,"__x86_indirect_" ,15 ); +}
tools/objtool/check.c +7 -2
@@ -850,6 +850,11 @@ static int add_ignore_alternatives(strucreturn 0 ; } +__weak bool arch_is_retpoline (struct symbol *sym) +{ +return false; +} +/* * Find the destination instructions for all jumps. */
@@ -872,7 +877,7 @@ static int add_jump_destinations(struct }else if (reloc->sym->type ==STT_SECTION ) { dest_sec = reloc->sym->sec; dest_off =arch_dest_reloc_offset (reloc->addend); - }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 )) { + }else if (arch_is_retpoline (reloc->sym)) {/* * Retpoline jumps are really dynamic jumps in * disguise, so convert them accordingly.
@@ -1026,7 +1031,7 @@ static int add_call_destinations(structreturn -1 ; } - }else if (!strncmp (reloc->sym->name,"__x86_indirect_thunk_" ,21 )) { + }else if (arch_is_retpoline (reloc->sym)) {/* * Retpoline calls are really dynamic calls in * disguise, so convert them accodingly.
tools/objtool/include/objtool/arch.h +2 -0
@@ -85,4 +85,6 @@ const char *arch_nop_insn(int len);int arch_decode_hint_reg (struct instruction *insn,u8 sp_reg); +bool arch_is_retpoline (struct symbol *sym); +#endif /* _ARCH_H */
Peter Zijlstra
maintainerPATCH 4/9 objtool: Fix static_call list generation
In reply to Peter ZijlstraCurrently objtool generates tail call entries in add_jump_destination() but waits until validate_branch() to generate the regular call entries, move these to add_call_destination() for consistency.
--- tools/objtool/check.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)
tools/objtool/check.c +12 -5
@@ -974,6 +974,11 @@ static int add_call_destinations(struct }else insn->call_dest = reloc->sym; +if (insn->call_dest && insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + } +/* * Many compilers cannot disable KCOV with a function attribute * so they need a little help, NOP out any KCOV calls from noinstr
@@ -1701,6 +1706,9 @@ static int decode_sections(struct objtooif (ret)return ret; +/* +* Must be before add_{jump_call}_desetination. +*/ ret =read_static_call_tramps (file);if (ret)return ret;
@@ -1717,6 +1725,10 @@ static int decode_sections(struct objtooif (ret)return ret; +/* +* Must be before add_call_destination(); it changes INSN_CALL to +* INSN_JUMP. +*/ ret =read_intra_function_calls (file);if (ret)return ret;
@@ -2659,11 +2671,6 @@ static int validate_branch(struct objtooif (dead_end_function (file, insn->call_dest))return 0 ; -if (insn->type ==INSN_CALL && insn->call_dest->static_call_tramp) { -list_add_tail (&insn->static_call_node, - &file->static_call_list); - } -break ;case INSN_JUMP_CONDITIONAL :
Re: [PATCH 4/9] objtool: Fix static_call list generation
tools/objtool/check.c static int decode_sections(struct objtoo · line 1711
Quoted 7 lines · Peter Zijlstra
tools/objtool/check.c +3
-0@@ -1701,6 +1706,9 @@ static int decode_sections(struct objtooif (ret)return ret; +/* +* Must be before add_{jump_call}_desetination. +*/
s/desetination/destination/
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 5/9 objtool: Rework rebuild_reloc logic
In reply to Peter ZijlstraInstead of manually calling elf_rebuild_reloc_section() on sections we've called elf_add_reloc() on, have elf_write() DTRT.
This makes it easier to add random relocations in places without carefully tracking when we're done and need to flush what section.
--- tools/objtool/check.c | 3 --- tools/objtool/elf.c | 13 ++++++++++++- tools/objtool/include/objtool/elf.h | 2 +- tools/objtool/orc_gen.c | 3 --- 4 files changed, 13 insertions(+), 8 deletions(-)
tools/objtool/check.c +0 -3
@@ -542,9 +542,6 @@ static int create_static_call_sections(s idx++; } -if (elf_rebuild_reloc_section (file->elf, reloc_sec)) -return -1 ; -return 0 ; }
tools/objtool/elf.c +12 -1
@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, strulist_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }static int read_rel_reloc (struct section *sec,int i,struct reloc *reloc,unsigned int *symndx)
@@ -558,7 +560,9 @@ static int read_relocs(struct elf *elf)return -1 ; } -elf_add_reloc (elf, reloc); +list_add_tail (&reloc->list, &sec->reloc_list); +elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + nr_reloc++; } max_reloc =max (max_reloc, nr_reloc);
@@ -890,6 +894,8 @@ int elf_rebuild_reloc_section(struct elfcase SHT_RELA :return elf_rebuild_rela_reloc_section (sec, nr);default :return -1 ; } + + sec->rereloc = false; } int elf_write_insn(struct elf *elf, struct section *sec,
@@ -944,6 +950,11 @@ int elf_write(struct elf *elf)struct section *sec;Elf_Scn *s; +list_for_each_entry (sec, &elf->sections, list) { +if (sec->reloc && sec->reloc->rereloc) +elf_rebuild_reloc_section (elf, sec->reloc); + } +/* Update section headers for changed sections: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) {
tools/objtool/include/objtool/elf.h +1 -1
@@ -39,7 +39,7 @@ struct section {char *name;int idx;unsigned int len; -bool changed, text, rodata, noinstr; +bool changed, text, rodata, noinstr, rereloc; };struct symbol {
tools/objtool/orc_gen.c +0 -3
@@ -254,8 +254,5 @@ int orc_create(struct objtool_file *filereturn -1 ; } -if (elf_rebuild_reloc_section (file->elf, ip_rsec)) -return -1 ; -return 0 ; }
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
tools/objtool/elf.c void elf_add_reloc(struct elf *elf, stru · line 484
Quoted 9 lines · Peter Zijlstra
tools/objtool/elf.c +2
-0@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, strulist_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }
Can we just reuse sec->changed for this? Something like this on top (untested of course):
tools/objtool/elf.c +9 -12
@@ -480,7 +480,7 @@ void elf_add_reloc(struct elf *elf, struct reloc *reloc)list_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); - sec->rereloc = true; + sec->changed = true; }static int read_rel_reloc (struct section *sec,int i,struct reloc *reloc,unsigned int *symndx)
@@ -882,9 +882,6 @@ int elf_rebuild_reloc_section(struct elf *elf, struct section *sec)struct reloc *reloc;int nr; - sec->changed = true; - elf->changed = true; - nr =0 ;list_for_each_entry (reloc, &sec->reloc_list, list) nr++;
@@ -894,8 +891,6 @@ int elf_rebuild_reloc_section(struct elf *elf, struct section *sec)case SHT_RELA :return elf_rebuild_rela_reloc_section (sec, nr);default :return -1 ; } - - sec->rereloc = false; } int elf_write_insn(struct elf *elf, struct section *sec,
@@ -950,14 +945,15 @@ int elf_write(struct elf *elf)struct section *sec;Elf_Scn *s; -list_for_each_entry (sec, &elf->sections, list) { -if (sec->reloc && sec->reloc->rereloc) -elf_rebuild_reloc_section (elf, sec->reloc); - } - -/* Update section headers for changed sections: */ +/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) { +if (sec->reloc && +elf_rebuild_reloc_section (elf, sec->reloc)) { +WARN_ELF ("elf_rebuild_reloc_section" ); +return -1 ; + } + s =elf_getscn (elf->elf, sec->idx);if (!s) {WARN_ELF ("elf_getscn" );
@@ -969,6 +965,7 @@ int elf_write(struct elf *elf) } sec->changed = false; + elf->changed = true; } }
tools/objtool/include/objtool/elf.h +1 -1
@@ -39,7 +39,7 @@ struct section {char *name;int idx;unsigned int len; -bool changed, text, rodata, noinstr, rereloc; +bool changed, text, rodata, noinstr; };struct symbol {
Peter Zijlstra
maintainerRe: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 5 lines · Josh Poimboeuf
Quoted 4 lines · Peter Zijlstra
tools/objtool/elf.c +0
-0@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, stru
Quoted 8 lines
tools/objtool/elf.c void elf_add_reloc(struct elf *elf, struct reloc *reloc) · line 484
Quoted 5 lines
tools/objtool/elf.c +2
-0list_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }Can we just reuse
sec->changedfor this? Something like this on top (untested of course):
I think my worry was that we'd dirty too much and slow down the write, but I haven't done any actual performance measurements on this.
Let me do a few runs and see if it matters at all.
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 15 lines · Peter Zijlstra
Quoted 5 lines · Josh Poimboeuf
Quoted 4 lines · Peter Zijlstra
tools/objtool/elf.c +0
-0@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, struQuoted 8 lines
tools/objtool/elf.c void elf_add_reloc(struct elf *elf, struct reloc *reloc) · line 484
Quoted 5 lines
tools/objtool/elf.c +2
-0list_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }Can we just reuse
sec->changedfor this? Something like this on top (untested of course):
Quoted 2 lines
I think my worry was that we'd dirty too much and slow down the write, but I haven't done any actual performance measurements on this.
Really? I thought my proposal was purely aesthetic, no functional change, but my brain is toasty this week due to other distractions so who knows.
Quoted 1 line
Let me do a few runs and see if it matters at all.
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 16 lines · Josh Poimboeuf
Quoted 15 lines · Peter Zijlstra
Quoted 13 lines · Josh Poimboeuf
tools/objtool/elf.c void elf_add_reloc(struct elf *elf, stru · line 484
Quoted 9 lines · Peter Zijlstra
tools/objtool/elf.c +2
-0@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, strulist_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }Can we just reuse
sec->changedfor this? Something like this on top (untested of course):
Quoted 6 lines
Quoted 2 lines
I think my worry was that we'd dirty too much and slow down the write, but I haven't done any actual performance measurements on this.
Really? I thought my proposal was purely aesthetic, no functional change, but my brain is toasty this week due to other distractions so who knows.
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Because if entries have relocations, then adding an entry will also add relocations etc..
pre: 79.269 +- 0.104 seconds time elapsed ( +- 0.13% )
post: 79.0604 +- 0.0441 seconds time elapsed ( +- 0.06% )
fini: 79.2995 +- 0.0448 seconds time elapsed ( +- 0.06% )
is what I get for kbuild x86_64-defconfig-ish build times with the various patches applied. Which is all noise afaict. I'll fold your thing. Less is more etc..
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 20 lines · Peter Zijlstra
Quoted 19 lines · Josh Poimboeuf
Quoted 17 lines · Peter Zijlstra
Quoted 5 lines · Josh Poimboeuf
Quoted 4 lines · Peter Zijlstra
tools/objtool/elf.c +0
-0@@ -479,6 +479,8 @@ void elf_add_reloc(struct elf *elf, struQuoted 8 lines
tools/objtool/elf.c void elf_add_reloc(struct elf *elf, struct reloc *reloc) · line 484
Quoted 5 lines
tools/objtool/elf.c +2
-0list_add_tail (&reloc->list, &sec->reloc_list);elf_hash_add (elf->reloc_hash, &reloc->hash,reloc_hash (reloc)); + + sec->rereloc = true; }Can we just reuse
sec->changedfor this? Something like this on top (untested of course):I think my worry was that we'd dirty too much and slow down the write, but I haven't done any actual performance measurements on this.
Quoted 7 lines
Quoted 3 lines
Really? I thought my proposal was purely aesthetic, no functional change, but my brain is toasty this week due to other distractions so who knows.
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Hm? This is a *relocation* section, not a normal one. So by definition, it only changes when its relocations change.
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 6 lines · Josh Poimboeuf
Quoted 3 lines
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Hm? This is a *relocation* section, not a normal one. So by definition, it only changes when its relocations change.
The way I read this code:
list_for_each_entry(sec, &elf->sections, list) {
if (sec->changed) {
+ if (sec->reloc &&
+ elf_rebuild_reloc_section(elf, sec->reloc)) {
+ WARN_ELF("elf_rebuild_reloc_section");
+ return -1;
+ }
is that we iterate the regular sections (which could be dirtied because we changed some data), and if that section has a relocation section, we rebuild that for good measure (even though it might not have altered relocations).
Or am I just totally confused ?
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 18 lines · Peter Zijlstra
Quoted 6 lines · Josh Poimboeuf
Quoted 3 lines
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Hm? This is a *relocation* section, not a normal one. So by definition, it only changes when its relocations change.
The way I read this code:
tools/objtool/elf.c +5
-0list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) { +if (sec->reloc && +elf_rebuild_reloc_section (elf, sec->reloc)) { +WARN_ELF ("elf_rebuild_reloc_section" ); +return -1 ; + }
Quoted 6 lines
is that we iterate the regular sections (which could be dirtied because we changed some data), and if that section has a relocation section, we rebuild that for good measure (even though it might not have altered relocations).
Or am I just totally confused ?
Ah, you're right. I'm the one that's confused. I guess I was also confused when I wrote that hunk, but it just happens to work anyway.
It would be cleaner to do something like
if ((is_reloc_sec(sec) &&
elf_rebuild_reloc_section(elf, sec)) {so we process the changed reloc section directly, instead of relying on the (most likely) fact that the corresponding text section also changed.
Signature · 1 line
Josh
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
Quoted 29 lines · Josh Poimboeuf
Quoted 18 lines · Peter Zijlstra
Quoted 6 lines · Josh Poimboeuf
Quoted 3 lines
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Hm? This is a *relocation* section, not a normal one. So by definition, it only changes when its relocations change.
The way I read this code:
(quoted diff) +5
-0list_for_each_entry(sec, &elf->sections, list) { if (sec->changed) { + if (sec->reloc && + elf_rebuild_reloc_section(elf, sec->reloc)) { + WARN_ELF("elf_rebuild_reloc_section"); + return -1; + }Quoted 6 lines
is that we iterate the regular sections (which could be dirtied because we changed some data), and if that section has a relocation section, we rebuild that for good measure (even though it might not have altered relocations).
Or am I just totally confused ?
Ah, you're right. I'm the one that's confused. I guess I was also confused when I wrote that hunk, but it just happens to work anyway.
Quoted 7 lines
It would be cleaner to do something like
if ((is_reloc_sec(sec) && elf_rebuild_reloc_section(elf, sec)) {so we process the changed reloc section directly, instead of relying on the (most likely) fact that the corresponding text section also changed.
i.e., in actual code:
tools/objtool/elf.c +1 -2
@@ -948,8 +948,7 @@ int elf_write(struct elf *elf)/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) { -if (sec->reloc && -elf_rebuild_reloc_section (elf, sec->reloc)) { +if (sec->base &&elf_rebuild_reloc_section (elf, sec)) {WARN_ELF ("elf_rebuild_reloc_section" );return -1 ; }
Peter Zijlstra
maintainerRe: [PATCH 5/9] objtool: Rework rebuild_reloc logic
In reply to Josh PoimboeufQuoted 29 lines · Josh Poimboeuf
Quoted 18 lines · Peter Zijlstra
Quoted 6 lines · Josh Poimboeuf
Quoted 3 lines
I was thinking you could get a section changed without touching relocations, but while that is theoretically possible, it is exceedingly unlikely (and objtool doesn't do that).
Hm? This is a *relocation* section, not a normal one. So by definition, it only changes when its relocations change.
The way I read this code:
(quoted diff) +5
-0list_for_each_entry(sec, &elf->sections, list) { if (sec->changed) { + if (sec->reloc && + elf_rebuild_reloc_section(elf, sec->reloc)) { + WARN_ELF("elf_rebuild_reloc_section"); + return -1; + }Quoted 6 lines
is that we iterate the regular sections (which could be dirtied because we changed some data), and if that section has a relocation section, we rebuild that for good measure (even though it might not have altered relocations).
Or am I just totally confused ?
Ah, you're right. I'm the one that's confused. I guess I was also confused when I wrote that hunk, but it just happens to work anyway.
Quoted 7 lines
It would be cleaner to do something like
if ((is_reloc_sec(sec) && elf_rebuild_reloc_section(elf, sec)) {so we process the changed reloc section directly, instead of relying on the (most likely) fact that the corresponding text section also changed.
Indeed. Done.
tools/objtool/elf.c +6 -1
@@ -814,6 +814,11 @@ struct section *elf_create_reloc_section } } +static inline bool is_reloc_section (struct section *reloc) +{ +return reloc->base && reloc->base->reloc == reloc; +} +static int elf_rebuild_rel_reloc_section (struct section *sec,int nr) {struct reloc *reloc;
@@ -948,7 +953,7 @@ int elf_write(struct elf *elf)/* Update changed relocation sections and section headers: */ list_for_each_entry (sec, &elf->sections, list) {if (sec->changed) { -if (sec->reloc && +if (is_reloc_section (sec) &&elf_rebuild_reloc_section (elf, sec->reloc)) {WARN_ELF ("elf_rebuild_reloc_section" );return -1 ;
Re: [PATCH 5/9] objtool: Rework rebuild_reloc logic
tools/objtool/elf.c struct section *elf_create_reloc_section · line 820
Quoted 8 lines · Peter Zijlstra
tools/objtool/elf.c +4
-0@@ -814,6 +814,11 @@ struct section *elf_create_reloc_section } } +static inline bool is_reloc_section (struct section *reloc) +{ +return reloc->base && reloc->base->reloc == reloc; +}
I believe the 2nd condition will always be true, so it can just be 'return reloc->base'.
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 6/9 objtool: Add elf_create_undef_symbol()
In reply to Peter ZijlstraAllow objtool to create undefined symbols; this allows creating relocations to symbols not currently in the symbol table.
--- tools/objtool/elf.c | 180 +++++++++++++++++++++++++++--------- tools/objtool/include/objtool/elf.h | 1 2 files changed, 139 insertions(+), 42 deletions(-)
tools/objtool/elf.c +138 -42
@@ -316,12 +316,60 @@ static int read_sections(struct elf *elfreturn 0 ; } +static bool elf_symbol_add (struct elf *elf,struct symbol *sym,Elf32_Word shndx) +{ +struct list_head *entry; +struct rb_node *pnode; + + sym->type =GELF_ST_TYPE (sym->sym.st_info); + sym->bind =GELF_ST_BIND (sym->sym.st_info); + +if ((sym->sym.st_shndx >SHN_UNDEF && + sym->sym.st_shndx <SHN_LORESERVE ) || + (shndx !=SHN_XINDEX && sym->sym.st_shndx ==SHN_XINDEX )) { +if (sym->sym.st_shndx !=SHN_XINDEX ) + shndx = sym->sym.st_shndx; + + sym->sec =find_section_by_index (elf, shndx); +if (!sym->sec) { +WARN ("couldn't find section for symbol %s" , + sym->name); +return false; + } +if (sym->type ==STT_SECTION ) { + sym->name = sym->sec->name; + sym->sec->sym = sym; + } + }else + sym->sec =find_section_by_index (elf,0 ); + + sym->offset = sym->sym.st_value; + sym->len = sym->sym.st_size; + +rb_add (&sym->node, &sym->sec->symbol_tree, symbol_to_offset); + pnode =rb_prev (&sym->node); +if (pnode) + entry = &rb_entry(pnode, struct symbol, node)->list; +else + entry = &sym->sec->symbol_list; +list_add (&sym->list, entry); +elf_hash_add (elf->symbol_hash, &sym->hash, sym->idx); +elf_hash_add (elf->symbol_name_hash, &sym->name_hash,str_hash (sym->name)); + +/* +* Don't store empty STT_NOTYPE symbols in the rbtree. They +* can exist within a function, confusing the sorting. +*/ +if (!sym->len) +rb_erase (&sym->node, &sym->sec->symbol_tree); + +return true; +} +static int read_symbols (struct elf *elf) {struct section *symtab, *symtab_shndx, *sec;struct symbol *sym, *pfunc; -struct list_head *entry; -struct rb_node *pnode;int symbols_nr, i;char *coldstr;Elf_Data *shndx_data =NULL ;
@@ -366,47 +414,11 @@ static int read_symbols(struct elf *elf) goto err; } - sym->type =GELF_ST_TYPE (sym->sym.st_info); - sym->bind =GELF_ST_BIND (sym->sym.st_info); +if (!shndx_data) + shndx =SHN_XINDEX ; -if ((sym->sym.st_shndx >SHN_UNDEF && - sym->sym.st_shndx <SHN_LORESERVE ) || - (shndx_data && sym->sym.st_shndx ==SHN_XINDEX )) { -if (sym->sym.st_shndx !=SHN_XINDEX ) - shndx = sym->sym.st_shndx; - - sym->sec =find_section_by_index (elf, shndx); -if (!sym->sec) { -WARN ("couldn't find section for symbol %s" , - sym->name); - goto err; - } -if (sym->type ==STT_SECTION ) { - sym->name = sym->sec->name; - sym->sec->sym = sym; - } - }else - sym->sec =find_section_by_index (elf,0 ); - - sym->offset = sym->sym.st_value; - sym->len = sym->sym.st_size; - -rb_add (&sym->node, &sym->sec->symbol_tree, symbol_to_offset); - pnode =rb_prev (&sym->node); -if (pnode) - entry = &rb_entry(pnode, struct symbol, node)->list; -else - entry = &sym->sec->symbol_list; -list_add (&sym->list, entry); -elf_hash_add (elf->symbol_hash, &sym->hash, sym->idx); -elf_hash_add (elf->symbol_name_hash, &sym->name_hash,str_hash (sym->name)); - -/* -* Don't store empty STT_NOTYPE symbols in the rbtree. They -* can exist within a function, confusing the sorting. -*/ -if (!sym->len) -rb_erase (&sym->node, &sym->sec->symbol_tree); +if (!elf_symbol_add (elf, sym, shndx)) + goto err; }if (stats)
@@ -640,6 +652,90 @@ struct elf *elf_open_read(const char *nareturn NULL ; } +struct symbol *elf_create_undef_symbol (struct elf *elf,const char *name) +{ +struct section *strtab, *symtab; +struct symbol *sym; +Elf_Scn *s; +Elf_Data *data; + + sym =malloc (sizeof (*sym)); +if (!sym) { +perror ("malloc" ); +return NULL ; + } +memset (sym,0 ,sizeof (*sym)); + + sym->name =strdup (name); + + strtab =find_section_by_name (elf,".strtab" ); +if (!strtab) { +WARN ("can't find .strtab" ); +return NULL ; + } + + s =elf_getscn (elf->elf, strtab->idx); +if (!s) { +WARN_ELF ("elf_getscn" ); +return NULL ; + } + + data =elf_newdata (s); +if (!data) { +WARN_ELF ("elf_newdata" ); +return NULL ; + } + + data->d_buf = sym->name; + data->d_size =strlen (sym->name) +1 ; + data->d_align =1 ; + + sym->sym.st_name = strtab->len; + sym->sym.st_info =0x10 ;/* STB_GLOBAL, STT_NOTYPE */ +// st_other 0 +// st_shndx 0 +// st_value 0 +// st_size 0 + + strtab->len += data->d_size; + strtab->changed = true; + + + symtab =find_section_by_name (elf,".symtab" ); +if (!symtab) { +WARN ("can't find .symtab" ); +return NULL ; + } + + s =elf_getscn (elf->elf, symtab->idx); +if (!s) { +WARN_ELF ("elf_getscn" ); +return NULL ; + } + + data =elf_newdata (s); +if (!data) { +WARN_ELF ("elf_newdata" ); +return NULL ; + } + + data->d_buf = &sym->sym; + data->d_size =sizeof (sym->sym); + data->d_align =1 ; + + sym->idx = symtab->len /sizeof (sym->sym); + + symtab->len += data->d_size; + symtab->changed = true; + +if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + } + +return sym; +} +struct section *elf_create_section (struct elf *elf,const char *name,unsigned int sh_flags,size_t entsize,int nr) {
tools/objtool/include/objtool/elf.h +1 -0
@@ -128,6 +128,7 @@ int elf_write_insn(struct elf *elf, stru unsigned long offset, unsigned int len, const char *insn);int elf_write_reloc (struct elf *elf,struct reloc *reloc); +struct symbol *elf_create_undef_symbol (struct elf *elf,const char *name);int elf_write (struct elf *elf);void elf_close (struct elf *elf);
Re: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
[ also correcting my e-mail address ]
On Fri, 12 Mar 2021, Peter Zijlstra wrote:
Just a remark regarding SHN_XINDEX...
Quoted 20 lines
tools/objtool/elf.c +20
-0+static bool elf_symbol_add (struct elf *elf,struct symbol *sym,Elf32_Word shndx) +{ +struct list_head *entry; +struct rb_node *pnode; + + sym->type =GELF_ST_TYPE (sym->sym.st_info); + sym->bind =GELF_ST_BIND (sym->sym.st_info); + +if ((sym->sym.st_shndx >SHN_UNDEF && + sym->sym.st_shndx <SHN_LORESERVE ) || + (shndx !=SHN_XINDEX && sym->sym.st_shndx ==SHN_XINDEX )) { +if (sym->sym.st_shndx !=SHN_XINDEX ) + shndx = sym->sym.st_shndx; + + sym->sec =find_section_by_index (elf, shndx); +if (!sym->sec) { +WARN ("couldn't find section for symbol %s" , + sym->name); +return false; + }
...
Quoted 20 lines
tools/objtool/elf.c +2
-13@@ -366,47 +414,11 @@ static int read_symbols(struct elf *elf) goto err; } - sym->type =GELF_ST_TYPE (sym->sym.st_info); - sym->bind =GELF_ST_BIND (sym->sym.st_info); +if (!shndx_data) + shndx =SHN_XINDEX ; -if ((sym->sym.st_shndx >SHN_UNDEF && - sym->sym.st_shndx <SHN_LORESERVE ) || - (shndx_data && sym->sym.st_shndx ==SHN_XINDEX )) { -if (sym->sym.st_shndx !=SHN_XINDEX ) - shndx = sym->sym.st_shndx; - - sym->sec =find_section_by_index (elf, shndx); -if (!sym->sec) { -WARN ("couldn't find section for symbol %s" , - sym->name); - goto err;
...
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
tools/objtool/elf.c +4
-0+if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEX means that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx when shndx_data is NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please? elf_symbol_add() seems like a good place.
Miroslav
Peter Zijlstra
maintainerRe: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
Quoted 10 lines · Miroslav Benes
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
tools/objtool/elf.c +4
-0+if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEXmeans that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx whenshndx_datais NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please?elf_symbol_add()seems like a good place.
Yes, that was a horrible thing to do :/ And you understood it right.
Looking at it again, I'm not sure it is actually correct tho; shouldn't elf_create_undef_symbol() also look at gelf_getsymshndx() of symtab ?
What toolchain generates these extended sections and how? That is, how do I test this crud..
Re: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
Quoted 13 lines · Peter Zijlstra
Quoted 10 lines · Miroslav Benes
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
tools/objtool/elf.c +4
-0+if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEXmeans that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx whenshndx_datais NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please?elf_symbol_add()seems like a good place.
Quoted 4 lines
Yes, that was a horrible thing to do :/ And you understood it right.
Looking at it again, I'm not sure it is actually correct tho; shouldn't
elf_create_undef_symbol()also look atgelf_getsymshndx()of symtab ?
Probably yes.
Quoted 2 lines
What toolchain generates these extended sections and how? That is, how do I test this crud..
Sami might know.
Miroslav
Re: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
Quoted 21 lines · Miroslav Benes
Quoted 13 lines · Peter Zijlstra
Quoted 10 lines · Miroslav Benes
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
+ if (!elf_symbol_add(elf, sym,
SHN_XINDEX)) {tools/objtool/elf.c +3
-0+WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEXmeans that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx whenshndx_datais NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please?elf_symbol_add()seems like a good place.Quoted 4 lines
Yes, that was a horrible thing to do :/ And you understood it right.
Looking at it again, I'm not sure it is actually correct tho; shouldn't
elf_create_undef_symbol()also look atgelf_getsymshndx()of symtab ?
Quoted 6 lines
Probably yes.
Quoted 2 lines
What toolchain generates these extended sections and how? That is, how do I test this crud..
Sami might know.
Clang generates these with LTO for vmlinux.o, but I'm guessing gcc will do the same with -ffunction-sections -fdata-sections.
Sami
Re: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
In reply to Peter ZijlstraQuoted 13 lines · Peter Zijlstra
Quoted 10 lines · Miroslav Benes
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
tools/objtool/elf.c +4
-0+if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEXmeans that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx whenshndx_datais NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please?elf_symbol_add()seems like a good place.
Quoted 7 lines
Yes, that was a horrible thing to do :/ And you understood it right.
Looking at it again, I'm not sure it is actually correct tho; shouldn't
elf_create_undef_symbol()also look atgelf_getsymshndx()of symtab ?What toolchain generates these extended sections and how? That is, how do I test this crud..
SHN_XINDEX is basically a special-case extension to original ELF for supporting more than 64k sections.
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 6/9] objtool: Add elf_create_undef_symbol()
Quoted 16 lines · Josh Poimboeuf
Quoted 15 lines · Peter Zijlstra
Quoted 10 lines · Miroslav Benes
tools/objtool/elf.c struct elf *elf_open_read(const char *na · line 734
Quoted 4 lines
tools/objtool/elf.c +4
-0+if (!elf_symbol_add (elf, sym,SHN_XINDEX )) { +WARN ("elf_symbol_add" ); +return NULL ; + }
SHN_XINDEXmeans that the extended section index is used. Above you seem to use it in the opposite sense too (assigning to shndx whenshndx_datais NULL). While it makes the code easier to handle, it is a bit confusing (and maybe I am just confused now). Could you add a comment about that, please?elf_symbol_add()seems like a good place.Yes, that was a horrible thing to do :/ And you understood it right.
Quoted 8 lines
Quoted 5 lines
Looking at it again, I'm not sure it is actually correct tho; shouldn't
elf_create_undef_symbol()also look atgelf_getsymshndx()of symtab ?What toolchain generates these extended sections and how? That is, how do I test this crud..
SHN_XINDEXis basically a special-case extension to original ELF for supporting more than 64k sections.
Yeah, I figured it all out again last night; the patch as presented is hideous but correct(ish). But I've rewritten it and it's all much better now. Will post later.
Peter Zijlstra
maintainerPATCH 7/9 objtool: Allow archs to rewrite retpolines
In reply to Peter ZijlstraWhen retpolines are employed, compilers typically emit calls to retpoline thunks. Objtool recognises these calls and marks them as dynamic calls.
Provide infrastructure for architectures to rewrite/augment what the compiler wrote for us.
--- tools/objtool/check.c | 23 +++++++++++++++++++++-- tools/objtool/include/objtool/arch.h | 3 +++ 2 files changed, 24 insertions(+), 2 deletions(-)
tools/objtool/check.c +21 -2
@@ -852,6 +852,14 @@ __weak bool arch_is_retpoline(struct symreturn false; } + +__weak int arch_rewrite_retpoline (struct objtool_file *file, +struct instruction *insn, +struct reloc *reloc) +{ +return 0 ; +} +/* * Find the destination instructions for all jumps. */
@@ -885,6 +893,9 @@ static int add_jump_destinations(struct insn->type =INSN_JUMP_DYNAMIC_CONDITIONAL ; insn->retpoline_safe = true; + +arch_rewrite_retpoline (file, insn, reloc); +continue ; }else if (insn->func) {/* internal or external sibling call (with reloc) */
@@ -1036,6 +1047,8 @@ static int add_call_destinations(struct insn->type =INSN_CALL_DYNAMIC ; insn->retpoline_safe = true; +arch_rewrite_retpoline (file, insn, reloc); +remove_insn_ops (insn);continue ;
@@ -1212,6 +1225,8 @@ static int handle_group_alt(struct objto dest_off =arch_jump_destination (insn);if (dest_off == special_alt->new_off + special_alt->new_len) insn->jump_dest =next_insn_same_sec (file, last_orig_insn); +else + insn->jump_dest =find_insn (file, insn->sec, dest_off);if (!insn->jump_dest) { WARN_FUNC("can't find alternative jump destination",
@@ -1797,11 +1812,15 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =add_jump_destinations (file); +/* +* Must be before add_{jump,call}_destination; for they can add +* magic alternatives. +*/ + ret =add_special_section_alts (file);if (ret)return ret; - ret =add_special_section_alts (file); + ret =add_jump_destinations (file);if (ret)return ret;
tools/objtool/include/objtool/arch.h +3 -0
@@ -82,6 +82,9 @@ unsigned long arch_jump_destination(struunsigned long arch_dest_reloc_offset (int addend);const char *arch_nop_insn (int len); +int arch_rewrite_retpoline (struct objtool_file *file, +struct instruction *insn, +struct reloc *reloc);int arch_decode_hint_reg (struct instruction *insn,u8 sp_reg);
Peter Zijlstra
maintainerPATCH 8/9 objtool: Skip magical retpoline .altinstr_replacement
In reply to Peter ZijlstraWhen the .altinstr_replacement is a retpoline, skip the alternative. We already special case retpolines anyway.
--- tools/objtool/special.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
tools/objtool/special.c +11 -1
@@ -106,6 +106,14 @@ static int get_alt_entry(struct elf *elfreturn -1 ; } +/* +* Skip retpoline .altinstr_replacement... we already rewrite the +* instructions for retpolines anyway, see arch_is_retpoline() +* usage in add_{call,jump}_destinations(). +*/ +if (arch_is_retpoline (new_reloc->sym)) +return 1 ; + alt->new_sec = new_reloc->sym->sec; alt->new_off = (unsigned int )new_reloc->addend;
@@ -154,7 +162,9 @@ int special_get_alts(struct elf *elf, stmemset (alt,0 ,sizeof (*alt)); ret =get_alt_entry (elf, entry, sec, idx, alt); -if (ret) +if (ret >0 ) +continue ; +if (ret <0 )return ret;list_add_tail (&alt->list, alts);
Peter Zijlstra
maintainerPATCH 9/9 objtool,x86: Rewrite retpoline thunk calls
In reply to Peter ZijlstraWhen the compiler emits: "CALL __x86_indirect_thunk_\reg" for an indirect call, have objtool rewrite it to:
ALTERNATIVE "call __x86_indirect_thunk_\reg", "call *%reg", ALT_NOT(X86_FEATURE_RETPOLINE)
Additionally, in order to not emit endless identical .altinst_replacement chunks, use a global symbol for them, see __x86_indirect_alt_*.
--- arch/x86/include/asm/asm-prototypes.h | 12 ++ arch/x86/lib/retpoline.S | 33 +++++++- tools/objtool/arch/x86/decode.c | 139 ++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 3 deletions(-)
arch/x86/include/asm/asm-prototypes.h +10 -2
@@ -19,11 +19,19 @@ extern void cmpxchg8b_emu(void);#ifdef CONFIG_RETPOLINE -#define DECL_INDIRECT_THUNK (reg) \ +#undef GEN +#define GEN (reg) \ extern asmlinkage void __x86_indirect_thunk_ ## reg (void); +#include <asm/GEN-for-each-reg.h> + +#undef GEN +#define GEN (reg) \ + extern asmlinkage void __x86_indirect_alt_call_ ## reg (void); +#include <asm/GEN-for-each-reg.h> #undef GEN -#define GEN (reg) DECL_INDIRECT_THUNK(reg) +#define GEN (reg) \ + extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);#include <asm/GEN-for-each-reg.h> #endif /* CONFIG_RETPOLINE */
arch/x86/lib/retpoline.S +32 -1
@@ -10,6 +10,8 @@ #include <asm/unwind_hints.h> #include <asm/frame.h> + .section .text.__x86.indirect_thunk + .macro RETPOLINE reg ANNOTATE_INTRA_FUNCTION_CALL call .Ldo_rop_\@
@@ -25,7 +27,6 @@ .endm .macro THUNK reg - .section .text.__x86.indirect_thunk .align 32 SYM_FUNC_START(__x86_indirect_thunk_\reg)
@@ -38,6 +39,24 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) .endm +.macro CALL_THUNK reg + + .align 1 + +SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg) + ANNOTATE_RETPOLINE_SAFE +1: call *%\reg +2: .nops 5-(2b-1b) +SYM_FUNC_END(__x86_indirect_alt_call_\reg) + +SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg) + ANNOTATE_RETPOLINE_SAFE +1: jmp *%\reg +2: .nops 5-(2b-1b) +SYM_FUNC_END(__x86_indirect_alt_jmp_\reg) + +.endm + /* * Despite being an assembler file we can't just use .irp here * because __KSYM_DEPS__ only uses the C preprocessor and would
@@ -61,3 +80,15 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg) #define GEN(reg) EXPORT_THUNK(reg) #include <asm/GEN-for-each-reg.h> +#undef GEN +#define GEN(reg) CALL_THUNK reg +#include <asm/GEN-for-each-reg.h> + +#undef GEN +#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg) +#include <asm/GEN-for-each-reg.h> + +#undef GEN +#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg) +#include <asm/GEN-for-each-reg.h> +
tools/objtool/arch/x86/decode.c +139 -0
151 lines not on this page · raw message