x86/kvm/emulate: Avoid RET for FASTOPs
Peter Zijlstra
maintainercover x86/kvm/emulate: Avoid RET for FASTOPs
Hi!
Because I needed a new objtool annotation, and I'd promised Josh I'd clean all that up a while ago, now with a healy sprinking of objtool patches...
Anyway... FASTOP is special in that it relies on RET to preserve FLAGS, while normal C calling convention does not. This has been a problem before, see ba5ca5e5e6a1 ("x86/retpoline: Don't clobber RFLAGS during srso_safe_ret()") but is also a problem for call depth tracking.
Fixing the call-depth tracking return thunk would be significantly harder (and more expensive), so instead change fastops to not use return.
There are two separate instances, test_cc() and fastop(). The first is basically a SETCC wrapper, which seems like a very complicated (and somewhat expensive) way to read FLAGS. Instead use the code we already have to emulate JCC to fully emulate the instruction.
That then leaves fastop(), which when marked noinline is guaranteed to exist only once. As such, CALL+RET isn't needed, because we'll always be RETurning to the same location, as such replace with JMP+JMP.
--- arch/x86/include/asm/alternative.h | 14 +- arch/x86/include/asm/nospec-branch.h | 45 ++++-- arch/x86/include/asm/text-patching.h | 20 ++- arch/x86/kvm/emulate.c | 54 +++---- include/linux/instrumentation.h | 11 +- include/linux/objtool.h | 62 +++++---- include/linux/objtool_types.h | 12 ++ tools/include/linux/objtool_types.h | 12 ++ tools/objtool/check.c | 263 +++++++++++------------------------ 9 files changed, 208 insertions(+), 285 deletions(-)
Peter Zijlstra
maintainerPATCH 1/11 objtool: Generic annotation infrastructure
Avoid endless .discard.foo sections for each annotation, create a single .discard.annotate section that takes an annotation type along with the instruction.
---
include/linux/objtool.h +17 -0
@@ -57,6 +57,13 @@".long 998b\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE (x) \ + "911:\n\t" \ + ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ + ".long 911b - .\n\t" \ + ".long " __stringify(x) "\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /*
@@ -146,6 +153,14 @@ .popsection .endm +.macro ANNOTATE type:req +.Lhere_\@: + .pushsection .discard.annotate,"M",@progbits,8 + .long .Lhere_\@ - . + .long \type + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */
@@ -167,6 +182,8 @@ .endm .macro REACHABLE .endm +.macro ANNOTATE +.endm#endif #endif /* CONFIG_OBJTOOL */
tools/objtool/check.c +37 -0
@@ -2308,6 +2308,41 @@ static int read_unwind_hints(struct objtreturn 0 ; } +static int read_annotate (struct objtool_file *file,void (*func)(int type,struct instruction *insn)) +{ +struct section *rsec, *sec; +struct instruction *insn; +struct reloc *reloc; +int type; + + rsec =find_section_by_name (file->elf,".rela.discard.annotate" ); +if (!rsec) +return 0 ; + + sec =find_section_by_name (file->elf,".discard.annotate" ); +if (!sec) +return 0 ; + +for_each_reloc (rsec, reloc) { + insn =find_insn (file, reloc->sym->sec, + reloc->sym->offset +reloc_addend (reloc)); +if (!insn) { +WARN ("bad .discard.annotate entry: %d" ,reloc_idx (reloc)); +return -1 ; + } + + type = *(u32 *)(sec->data->d_buf + (reloc_idx (reloc) * sec->sh.sh_entsize) +4 ); + +func (type, insn); + } + +return 0 ; +} + +static void __annotate_nop (int type,struct instruction *insn) +{ +} +static int read_noendbr_hints (struct objtool_file *file) {struct instruction *insn;
@@ -2602,6 +2637,8 @@ static int decode_sections(struct objtooif (ret)return ret; + ret =read_annotate (file, __annotate_nop); +/* * Must be before read_unwind_hints() since that needs insn->noendbr. */
Peter Zijlstra
maintainerRe: [PATCH 01/11] objtool: Generic annotation infrastructure
Quoted 72 lines · Peter Zijlstra
Avoid endless .discard.foo sections for each annotation, create a single .discard.annotate section that takes an annotation type along with the instruction.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
include/linux/objtool.h +17
-0@@ -57,6 +57,13 @@".long 998b\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE (x) \ + "911:\n\t" \ + ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ + ".long 911b - .\n\t" \ + ".long " __stringify(x) "\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /* @@ -146,6 +153,14 @@ .popsection .endm +.macro ANNOTATE type:req +.Lhere_\@: + .pushsection .discard.annotate,"M",@progbits,8 + .long .Lhere_\@ - . + .long \type + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ @@ -167,6 +182,8 @@ .endm .macro REACHABLE .endm +.macro ANNOTATE +.endm#endif #endif /* CONFIG_OBJTOOL */ tools/objtool/check.c +20
-0@@ -2308,6 +2308,41 @@ static int read_unwind_hints(struct objtreturn 0 ; } +static int read_annotate (struct objtool_file *file,void (*func)(int type,struct instruction *insn)) +{ +struct section *rsec, *sec; +struct instruction *insn; +struct reloc *reloc; +int type; + + rsec =find_section_by_name (file->elf,".rela.discard.annotate" ); +if (!rsec) +return 0 ; + + sec =find_section_by_name (file->elf,".discard.annotate" ); +if (!sec) +return 0 ; + +for_each_reloc (rsec, reloc) { + insn =find_insn (file, reloc->sym->sec, + reloc->sym->offset +reloc_addend (reloc)); +if (!insn) { +WARN ("bad .discard.annotate entry: %d" ,reloc_idx (reloc));
tools/objtool/check.c static int read_unwind_hints(struct objt · line 2340
Quoted 10 lines
tools/objtool/check.c +10
-0+return -1 ; + } + + type = *(u32 *)(sec->data->d_buf + (reloc_idx (reloc) * sec->sh.sh_entsize) +4 ); + +func (type, insn); + } + +return 0 ; +}
So... ld.lld hates this :-(
From an LLVM=-19 build we can see that:
$ readelf -WS tmp-build/arch/x86/kvm/vmx/vmenter.o | grep annotate [13] .discard.annotate PROGBITS 0000000000000000 00028c 000018 08 M 0 0 1
$ readelf -WS tmp-build/arch/x86/kvm/kvm-intel.o | grep annotate [ 3] .discard.annotate PROGBITS 0000000000000000 069fe0 0089d0 00 M 0 0 1
Which tells us that the translation unit itself has a sh_entsize of 8, while the linked object has sh_entsize of 0.
This then completely messes up the indexing objtool does, which relies on it being a sane number.
GCC/binutils very much does not do this, it retains the 8.
Dear clang folks, help?
Re: [PATCH 01/11] objtool: Generic annotation infrastructure
Quoted 97 lines · Peter Zijlstra
Quoted 72 lines · Peter Zijlstra
Avoid endless .discard.foo sections for each annotation, create a single .discard.annotate section that takes an annotation type along with the instruction.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
include/linux/objtool.h +17
-0@@ -57,6 +57,13 @@".long 998b\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE (x) \ + "911:\n\t" \ + ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ + ".long 911b - .\n\t" \ + ".long " __stringify(x) "\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /* @@ -146,6 +153,14 @@ .popsection .endm +.macro ANNOTATE type:req +.Lhere_\@: + .pushsection .discard.annotate,"M",@progbits,8 + .long .Lhere_\@ - . + .long \type + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ @@ -167,6 +182,8 @@ .endm .macro REACHABLE .endm +.macro ANNOTATE +.endm#endif #endif /* CONFIG_OBJTOOL */ tools/objtool/check.c +20
-0@@ -2308,6 +2308,41 @@ static int read_unwind_hints(struct objtreturn 0 ; } +static int read_annotate (struct objtool_file *file,void (*func)(int type,struct instruction *insn)) +{ +struct section *rsec, *sec; +struct instruction *insn; +struct reloc *reloc; +int type; + + rsec =find_section_by_name (file->elf,".rela.discard.annotate" ); +if (!rsec) +return 0 ; + + sec =find_section_by_name (file->elf,".discard.annotate" ); +if (!sec) +return 0 ; + +for_each_reloc (rsec, reloc) { + insn =find_insn (file, reloc->sym->sec, + reloc->sym->offset +reloc_addend (reloc)); +if (!insn) { +WARN ("bad .discard.annotate entry: %d" ,reloc_idx (reloc));tools/objtool/check.c static int read_unwind_hints(struct objt · line 2340
Quoted 10 lines
tools/objtool/check.c +10
-0+return -1 ; + } + + type = *(u32 *)(sec->data->d_buf + (reloc_idx (reloc) * sec->sh.sh_entsize) +4 ); + +func (type, insn); + } + +return 0 ; +}So... ld.lld hates this :-(
From an LLVM=-19 build we can see that:
$ readelf -WS tmp-build/arch/x86/kvm/vmx/vmenter.o | grep annotate [13] .discard.annotate PROGBITS 0000000000000000 00028c 000018 08 M 0 0 1$ readelf -WS tmp-build/arch/x86/kvm/kvm-intel.o | grep annotate [ 3] .discard.annotate PROGBITS 0000000000000000 069fe0 0089d0 00 M 0 0 1Which tells us that the translation unit itself has a
sh_entsizeof 8, while the linked object hassh_entsizeof 0.
Quoted 6 lines
This then completely messes up the indexing objtool does, which relies on it being a sane number.
GCC/binutils very much does not do this, it retains the 8.
Dear clang folks, help?
Perhaps Fangrui has immediate thoughts, since this appears to be an ld.lld thing? Otherwise, I will see if I can dig into this in the next couple of weeks (I have an LF webinar on Wednesday that I am still prepping for). Is this reproducible with just defconfig or something else?
Cheers,
Nathan
Peter Zijlstra
maintainerRe: [PATCH 01/11] objtool: Generic annotation infrastructure
Quoted 103 lines · Nathan Chancellor
Quoted 102 lines · Peter Zijlstra
Quoted 72 lines · Peter Zijlstra
Avoid endless .discard.foo sections for each annotation, create a single .discard.annotate section that takes an annotation type along with the instruction.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
include/linux/objtool.h +17
-0@@ -57,6 +57,13 @@".long 998b\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE (x) \ + "911:\n\t" \ + ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ + ".long 911b - .\n\t" \ + ".long " __stringify(x) "\n\t" \ + ".popsection\n\t" +#else /* __ASSEMBLY__ */ /* @@ -146,6 +153,14 @@ .popsection .endm +.macro ANNOTATE type:req +.Lhere_\@: + .pushsection .discard.annotate,"M",@progbits,8 + .long .Lhere_\@ - . + .long \type + .popsection +.endm +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */ @@ -167,6 +182,8 @@ .endm .macro REACHABLE .endm +.macro ANNOTATE +.endm#endif #endif /* CONFIG_OBJTOOL */ tools/objtool/check.c +20
-0@@ -2308,6 +2308,41 @@ static int read_unwind_hints(struct objtreturn 0 ; } +static int read_annotate (struct objtool_file *file,void (*func)(int type,struct instruction *insn)) +{ +struct section *rsec, *sec; +struct instruction *insn; +struct reloc *reloc; +int type; + + rsec =find_section_by_name (file->elf,".rela.discard.annotate" ); +if (!rsec) +return 0 ; + + sec =find_section_by_name (file->elf,".discard.annotate" ); +if (!sec) +return 0 ; + +for_each_reloc (rsec, reloc) { + insn =find_insn (file, reloc->sym->sec, + reloc->sym->offset +reloc_addend (reloc)); +if (!insn) { +WARN ("bad .discard.annotate entry: %d" ,reloc_idx (reloc));tools/objtool/check.c static int read_unwind_hints(struct objt · line 2340
Quoted 10 lines
tools/objtool/check.c +10
-0+return -1 ; + } + + type = *(u32 *)(sec->data->d_buf + (reloc_idx (reloc) * sec->sh.sh_entsize) +4 ); + +func (type, insn); + } + +return 0 ; +}So... ld.lld hates this :-(
From an LLVM=-19 build we can see that:
$ readelf -WS tmp-build/arch/x86/kvm/vmx/vmenter.o | grep annotate [13] .discard.annotate PROGBITS 0000000000000000 00028c 000018 08 M 0 0 1$ readelf -WS tmp-build/arch/x86/kvm/kvm-intel.o | grep annotate [ 3] .discard.annotate PROGBITS 0000000000000000 069fe0 0089d0 00 M 0 0 1Which tells us that the translation unit itself has a
sh_entsizeof 8, while the linked object hassh_entsizeof 0.This then completely messes up the indexing objtool does, which relies on it being a sane number.
GCC/binutils very much does not do this, it retains the 8.
Quoted 7 lines
Quoted 1 line
Dear clang folks, help?
Perhaps Fangrui has immediate thoughts, since this appears to be an ld.lld thing? Otherwise, I will see if I can dig into this in the next couple of weeks (I have an LF webinar on Wednesday that I am still prepping for). Is this reproducible with just defconfig or something else?
I took the .config from the report you pointed me at yesterday.
https://lore.kernel.org/[email protected]/
And specifically the kvm build targets from:
$ make O=tmp-build/ LLVM=-19 arch/x86/kvm/
show this problem.
I just ran a defconfig, and that seems to behave properly. Notably, vmlinux.o (definitely a link target) has entsize=8 for the relevant section.
I'm not sure how the kvm targets might be 'special'.
Peter Zijlstra
maintainerRe: [PATCH 01/11] objtool: Generic annotation infrastructure
In reply to Peter ZijlstraQuoted 8 lines · Peter Zijlstra
From an LLVM=-19 build we can see that:
$ readelf -WS tmp-build/arch/x86/kvm/vmx/vmenter.o | grep annotate [13] .discard.annotate PROGBITS 0000000000000000 00028c 000018 08 M 0 0 1$ readelf -WS tmp-build/arch/x86/kvm/kvm-intel.o | grep annotate [ 3] .discard.annotate PROGBITS 0000000000000000 069fe0 0089d0 00 M 0 0 1
Quoted 7 lines
Which tells us that the translation unit itself has a
sh_entsizeof 8, while the linked object hassh_entsizeof 0.This then completely messes up the indexing objtool does, which relies on it being a sane number.
GCC/binutils very much does not do this, it retains the 8.
Anyway, for now I've added:
+ if (sec->sh.sh_entsize != 8) {
+ static bool warn = false;
+ if (!warn) {
+ WARN("%s: dodgy linker, sh_entsize != 8", sec->name);
+ warn = true;
+ }
+ sec->sh.sh_entsize = 8;
+ }
To objtool, this allows it function correctly and prints this reminder to for us to figure out the linker story.
Peter Zijlstra
maintainerPATCH 2/11 objtool: Convert ANNOTATE_NOENDBR to ANNOTATE
In reply to Peter Zijlstra--- include/linux/objtool.h | 17 ++++------------- include/linux/objtool_types.h | 5 +++++ tools/include/linux/objtool_types.h | 5 +++++ tools/objtool/check.c | 32 +++++--------------------------- 4 files changed, 19 insertions(+), 40 deletions(-)
include/linux/objtool.h +4 -13
@@ -45,12 +45,6 @@#define STACK_FRAME_NON_STANDARD_FP (func)#endif -#define ANNOTATE_NOENDBR \ - "986: \n\t" \ - ".pushsection .discard.noendbr\n\t" \ - ".long 986b\n\t" \ - ".popsection\n\t" -#define ASM_REACHABLE \ "998:\n\t" \ ".pushsection .discard.reachable\n\t" \
@@ -64,6 +58,8 @@ ".long " __stringify(x) "\n\t" \ ".popsection\n\t" +#define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR) +#else /* __ASSEMBLY__ */ /*
@@ -122,13 +118,6 @@#endif .endm -.macro ANNOTATE_NOENDBR -.Lhere_\@: - .pushsection .discard.noendbr - .long .Lhere_\@ - .popsection -.endm -/* * Use objtool to validate the entry requirement that all code paths do * VALIDATE_UNRET_END before RET.
@@ -161,6 +150,8 @@ .popsection .endm +#define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */
include/linux/objtool_types.h +5 -0
@@ -54,4 +54,9 @@ struct unwind_hint {#define UNWIND_HINT_TYPE_SAVE 6#define UNWIND_HINT_TYPE_RESTORE 7 +/* +* Annotate types +*/ +#define ANNOTYPE_NOENDBR 1 +#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +5 -0
@@ -54,4 +54,9 @@ struct unwind_hint {#define UNWIND_HINT_TYPE_SAVE 6#define UNWIND_HINT_TYPE_RESTORE 7 +/* +* Annotate types +*/ +#define ANNOTYPE_NOENDBR 1 +#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +5 -27
@@ -2339,32 +2339,12 @@ static int read_annotate(struct objtool_return 0 ; } -static void __annotate_nop (int type,struct instruction *insn) +static void __annotate_noendbr (int type,struct instruction *insn) { -} - -static int read_noendbr_hints (struct objtool_file *file) -{ -struct instruction *insn; -struct section *rsec; -struct reloc *reloc; - - rsec =find_section_by_name (file->elf,".rela.discard.noendbr" ); -if (!rsec) -return 0 ; +if (type !=ANNOTYPE_NOENDBR ) +return ; -for_each_reloc (rsec, reloc) { - insn =find_insn (file, reloc->sym->sec, - reloc->sym->offset +reloc_addend (reloc)); -if (!insn) { -WARN ("bad .discard.noendbr entry" ); -return -1 ; - } - - insn->noendbr =1 ; - } - -return 0 ; + insn->noendbr =1 ; }static int read_retpoline_hints (struct objtool_file *file)
@@ -2637,12 +2617,10 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =read_annotate (file, __annotate_nop); -/* * Must be before read_unwind_hints() since that needs insn->noendbr. */ - ret =read_noendbr_hints (file); + ret =read_annotate (file, __annotate_noendbr);if (ret)return ret;
Peter Zijlstra
maintainerPATCH 3/11 objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE
In reply to Peter Zijlstra--- arch/x86/include/asm/nospec-branch.h | 13 +------- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 52 ++++++++++++----------------------- 4 files changed, 22 insertions(+), 45 deletions(-)
arch/x86/include/asm/nospec-branch.h +2 -11
@@ -193,12 +193,7 @@* objtool the subsequent indirect jump/call is vouched safe for retpoline * builds. */ -.macro ANNOTATE_RETPOLINE_SAFE -.Lhere_\@: - .pushsection .discard.retpoline_safe - .long .Lhere_\@ - .popsection -.endm +#define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE/* * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
@@ -317,11 +312,7 @@#else /* __ASSEMBLY__ */ -#define ANNOTATE_RETPOLINE_SAFE \ - "999:\n\t" \ - ".pushsection .discard.retpoline_safe\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE)typedef u8 retpoline_thunk_t [RETPOLINE_THUNK_SIZE ];extern retpoline_thunk_t __x86_indirect_thunk_array[];
include/linux/objtool_types.h +1 -0
@@ -58,5 +58,6 @@ struct unwind_hint {* Annotate types */ #define ANNOTYPE_NOENDBR 1 +#define ANNOTYPE_RETPOLINE_SAFE 2#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -58,5 +58,6 @@ struct unwind_hint {* Annotate types */ #define ANNOTYPE_NOENDBR 1 +#define ANNOTYPE_RETPOLINE_SAFE 2#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +18 -34
90 lines not on this page · raw message
Sean Christopherson
maintainerRe: [PATCH 03/11] objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE
Quoted 18 lines · Peter Zijlstra
arch/x86/include/asm/nospec-branch.h +1
-6@@ -193,12 +193,7 @@* objtool the subsequent indirect jump/call is vouched safe for retpoline * builds. */ -.macro ANNOTATE_RETPOLINE_SAFE -.Lhere_\@: - .pushsection .discard.retpoline_safe - .long .Lhere_\@ - .popsection -.endm +#define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE/* * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
arch/x86/include/asm/nospec-branch.h line 315
Quoted 9 lines
arch/x86/include/asm/nospec-branch.h +1
-5@@ -317,11 +312,7 @@#else /* __ASSEMBLY__ */ -#define ANNOTATE_RETPOLINE_SAFE \ - "999:\n\t" \ - ".pushsection .discard.retpoline_safe\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE)
This fails for some of my builds that end up with CONFIG_OBJTOOl=n. Adding a stub for ASM_ANNOTATE() gets me past that:
@@ -156,6 +171,7 @@
#define STACK_FRAME_NON_STANDARD(func)
#define STACK_FRAME_NON_STANDARD_FP(func)
#define ANNOTATE_NOENDBR
+#define ASM_ANNOTATE(x)
#define ASM_REACHABLE
#else
#define ANNOTATE_INTRA_FUNCTION_CALL
but then I run into other issues:
arch/x86/kernel/relocate_kernel_32.S: Assembler messages:arch/x86/kernel/relocate_kernel_32.S:96: Error: Parameter named type' does not exist for macro annotate' arch/x86/kernel/relocate_kernel_32.S:166: Error: Parameter named type' does not exist for macro annotate' arch/x86/kernel/relocate_kernel_32.S:174: Error: Parameter named type' does not exist for macro annotate' arch/x86/kernel/relocate_kernel_32.S:200: Error: Parameter named type' does not exist for macro annotate' arch/x86/kernel/relocate_kernel_32.S:220: Error: Parameter named type' does not exist for macro annotate' arch/x86/kernel/relocate_kernel_32.S:285: Error: Parameter named type' does not exist for macro annotate'
Re: [PATCH 03/11] objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE
Hi Peter,
Quoted 43 lines · Sean Christopherson
Quoted 18 lines · Peter Zijlstra
arch/x86/include/asm/nospec-branch.h +1
-6@@ -193,12 +193,7 @@* objtool the subsequent indirect jump/call is vouched safe for retpoline * builds. */ -.macro ANNOTATE_RETPOLINE_SAFE -.Lhere_\@: - .pushsection .discard.retpoline_safe - .long .Lhere_\@ - .popsection -.endm +#define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE/* * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions arch/x86/include/asm/nospec-branch.h line 315
Quoted 9 lines
arch/x86/include/asm/nospec-branch.h +1
-5@@ -317,11 +312,7 @@#else /* __ASSEMBLY__ */ -#define ANNOTATE_RETPOLINE_SAFE \ - "999:\n\t" \ - ".pushsection .discard.retpoline_safe\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE)This fails for some of my builds that end up with CONFIG_OBJTOOl=n. Adding a stub for
ASM_ANNOTATE()gets me past that:(quoted diff) +1
-0@@ -156,6 +171,7 @@ #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR +#define ASM_ANNOTATE(x) #define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALLbut then I run into other issues:
Quoted 7 lines
arch/x86/kernel/relocate_kernel_32.S: Assembler messages:arch/x86/kernel/relocate_kernel_32.S:96: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:166: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:174: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:200: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:220: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:285: Error: Parameter namedtype' does not exist for macroannotate'
Sean pointed me at this series recently. It seems like these compile errors (and some others) go away with the following diff:
arch/x86/include/asm/nospec-branch.h +1 -1
@@ -489,7 +489,7 @@ static inline void call_depth_return_thunk(void) {}" .align 16\n" \"903: lea 4(%%esp), %%esp;\n" \" pushl %[thunk_target];\n" \ -" ret;\n" \ +" ret;\n" , \X86_FEATURE_RETPOLINE , \"lfence;\n" \ANNOTATE_RETPOLINE_SAFE \
include/linux/objtool.h +2 -1
@@ -159,6 +159,7 @@#define STACK_FRAME_NON_STANDARD_FP (func)#define ANNOTATE_NOENDBR #define ASM_REACHABLE +#define ASM_ANNOTATE (x)#else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0
@@ -169,7 +170,7 @@ .endm .macro REACHABLE .endm -.macro ANNOTATE +.macro ANNOTATE type:req .endm#endif
This series applies on top of the latest kvm-x86/next with only a few trivial conflicts, so I hope you are able to send a new version.
I could send one for you, but I have no idea how to properly test it.
Re: [PATCH 03/11] objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE
Quoted 93 lines · James Houghton
Quoted 43 lines · Sean Christopherson
Quoted 18 lines · Peter Zijlstra
arch/x86/include/asm/nospec-branch.h +1
-6@@ -193,12 +193,7 @@* objtool the subsequent indirect jump/call is vouched safe for retpoline * builds. */ -.macro ANNOTATE_RETPOLINE_SAFE -.Lhere_\@: - .pushsection .discard.retpoline_safe - .long .Lhere_\@ - .popsection -.endm +#define ANNOTATE_RETPOLINE_SAFE ANNOTATE type=ANNOTYPE_RETPOLINE_SAFE/* * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions arch/x86/include/asm/nospec-branch.h line 315
Quoted 9 lines
arch/x86/include/asm/nospec-branch.h +1
-5@@ -317,11 +312,7 @@#else /* __ASSEMBLY__ */ -#define ANNOTATE_RETPOLINE_SAFE \ - "999:\n\t" \ - ".pushsection .discard.retpoline_safe\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_RETPOLINE_SAFE ASM_ANNOTATE(ANNOTYPE_RETPOLINE_SAFE)This fails for some of my builds that end up with CONFIG_OBJTOOl=n. Adding a stub for
ASM_ANNOTATE()gets me past that:include/linux/objtool.h +1
-0@@ -156,6 +171,7 @@#define STACK_FRAME_NON_STANDARD (func)#define STACK_FRAME_NON_STANDARD_FP (func)#define ANNOTATE_NOENDBR +#define ASM_ANNOTATE (x)#define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALL but then I run into other issues:
Quoted 7 lines
arch/x86/kernel/relocate_kernel_32.S: Assembler messages:arch/x86/kernel/relocate_kernel_32.S:96: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:166: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:174: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:200: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:220: Error: Parameter namedtype' does not exist for macroannotate'arch/x86/kernel/relocate_kernel_32.S:285: Error: Parameter namedtype' does not exist for macroannotate'Sean pointed me at this series recently. It seems like these compile errors (and some others) go away with the following diff:
arch/x86/include/asm/nospec-branch.h +1
-1@@ -489,7 +489,7 @@ static inline void call_depth_return_thunk(void) {}" .align 16\n" \"903: lea 4(%%esp), %%esp;\n" \" pushl %[thunk_target];\n" \ -" ret;\n" \ +" ret;\n" , \X86_FEATURE_RETPOLINE , \"lfence;\n" \ANNOTATE_RETPOLINE_SAFE \include/linux/objtool.h +2
-1@@ -159,6 +159,7 @@#define STACK_FRAME_NON_STANDARD_FP (func)#define ANNOTATE_NOENDBR #define ASM_REACHABLE +#define ASM_ANNOTATE (x)#else #define ANNOTATE_INTRA_FUNCTION_CALL .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0@@ -169,7 +170,7 @@ .endm .macro REACHABLE .endm -.macro ANNOTATE +.macro ANNOTATE type:req .endm#endif This series applies on top of the latest kvm-x86/next with only a few trivial conflicts, so I hope you are able to send a new version.
Quoted 1 line
I could send one for you, but I have no idea how to properly test it.
Hi Peter,
I'll go ahead and repost this series soon unless you tell me otherwise. :)
Peter Zijlstra
maintainerRe: [PATCH 03/11] objtool: Convert ANNOTATE_RETPOLINE_SAFE to ANNOTATE
Quoted 3 lines · James Houghton
Hi Peter,
I'll go ahead and repost this series soon unless you tell me otherwise. :)
Thanks for your persistent poking. I've refreshed the queue:x86/kvm branch, and provided the robots don't scream, I'll go and post it.
Peter Zijlstra
maintainerPATCH 4/11 objtool: Convert instrumentation_{begin,end}() to ANNOTATE
In reply to Peter Zijlstra--- include/linux/instrumentation.h | 11 +++----- include/linux/objtool.h | 9 ++++-- include/linux/objtool_types.h | 2 + tools/include/linux/objtool_types.h | 2 + tools/objtool/check.c | 49 +++++++----------------------------- 5 files changed, 25 insertions(+), 48 deletions(-)
include/linux/instrumentation.h +5 -6
@@ -4,14 +4,14 @@#ifdef CONFIG_NOINSTR_VALIDATION +#include <linux/objtool.h> #include <linux/stringify.h> /* Begin/end of an instrumentation safe region */ #define __instrumentation_begin (c) ({ \ asm volatile(__stringify(c) ": nop\n\t" \ - ".pushsection .discard.instr_begin\n\t" \ - ".long " __stringify(c) "b - .\n\t" \ - ".popsection\n\t" : : "i" (c)); \ + __ASM_ANNOTATE(c, ANNOTYPE_INSTR_BEGIN) \ + : : "i" (c)); \ })#define instrumentation_begin () __instrumentation_begin(__COUNTER__)
@@ -48,9 +48,8 @@*/ #define __instrumentation_end (c) ({ \ asm volatile(__stringify(c) ": nop\n\t" \ - ".pushsection .discard.instr_end\n\t" \ - ".long " __stringify(c) "b - .\n\t" \ - ".popsection\n\t" : : "i" (c)); \ + __ASM_ANNOTATE(c, ANNOTYPE_INSTR_END) \ + : : "i" (c)); \ })#define instrumentation_end () __instrumentation_end(__COUNTER__)#else /* !CONFIG_NOINSTR_VALIDATION */
include/linux/objtool.h +6 -3
@@ -51,13 +51,16 @@".long 998b\n\t" \ ".popsection\n\t" -#define ASM_ANNOTATE (x) \ - "911:\n\t" \ +#define __ASM_ANNOTATE (s, x) \ ".pushsection .discard.annotate,\"M\",@progbits,8\n\t" \ - ".long 911b - .\n\t" \ + ".long " __stringify(s) "b - .\n\t" \ ".long " __stringify(x) "\n\t" \ ".popsection\n\t" +#define ASM_ANNOTATE (x) \ + "911:\n\t" \ + __ASM_ANNOTATE(911, x) +#define ANNOTATE_NOENDBR ASM_ANNOTATE(ANNOTYPE_NOENDBR)#else /* __ASSEMBLY__ */
include/linux/objtool_types.h +2 -0
@@ -59,5 +59,7 @@ struct unwind_hint {*/ #define ANNOTYPE_NOENDBR 1#define ANNOTYPE_RETPOLINE_SAFE 2 +#define ANNOTYPE_INSTR_BEGIN 3 +#define ANNOTYPE_INSTR_END 4#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +2 -0
@@ -59,5 +59,7 @@ struct unwind_hint {*/ #define ANNOTYPE_NOENDBR 1#define ANNOTYPE_RETPOLINE_SAFE 2 +#define ANNOTYPE_INSTR_BEGIN 3 +#define ANNOTYPE_INSTR_END 4#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +10 -39
65 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 5/11 objtool: Convert VALIDATE_UNRET_BEGIN to ANNOTATE
In reply to Peter Zijlstra--- include/linux/objtool.h | 9 +++------ include/linux/objtool_types.h | 1 + tools/include/linux/objtool_types.h | 1 + tools/objtool/check.c | 28 +++++----------------------- 4 files changed, 10 insertions(+), 29 deletions(-)
include/linux/objtool.h +3 -6
@@ -128,15 +128,12 @@* NOTE: The macro must be used at the beginning of a global symbol, otherwise * it will be ignored. */ -.macro VALIDATE_UNRET_BEGIN#if defined(CONFIG_NOINSTR_VALIDATION ) && \ (defined(CONFIG_CPU_UNRET_ENTRY ) || defined(CONFIG_CPU_SRSO )) -.Lhere_\@: - .pushsection .discard.validate_unret - .long .Lhere_\@ - . - .popsection +#define VALIDATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN +#else +#define VALIDATE_UNRET_BEGIN #endif -.endm .macro REACHABLE .Lhere_\@:
include/linux/objtool_types.h +1 -0
@@ -61,5 +61,6 @@ struct unwind_hint {#define ANNOTYPE_RETPOLINE_SAFE 2#define ANNOTYPE_INSTR_BEGIN 3#define ANNOTYPE_INSTR_END 4 +#define ANNOTYPE_UNRET_BEGIN 5#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -61,5 +61,6 @@ struct unwind_hint {#define ANNOTYPE_RETPOLINE_SAFE 2#define ANNOTYPE_INSTR_BEGIN 3#define ANNOTYPE_INSTR_END 4 +#define ANNOTYPE_UNRET_BEGIN 5#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +5 -23
@@ -2385,33 +2385,15 @@ static int __annotate_instr(int type, streturn 0 ; } -static int read_validate_unret_hints (struct objtool_file *file) +static int __annotate_unret (int type,struct instruction *insn) { -struct section *rsec; -struct instruction *insn; -struct reloc *reloc; - - rsec =find_section_by_name (file->elf,".rela.discard.validate_unret" ); -if (!rsec) +if (type !=ANNOTYPE_UNRET_BEGIN )return 0 ; -for_each_reloc (rsec, reloc) { -if (reloc->sym->type !=STT_SECTION ) { -WARN ("unexpected relocation symbol type in %s" , rsec->name); -return -1 ; - } - - insn =find_insn (file, reloc->sym->sec,reloc_addend (reloc)); -if (!insn) { -WARN ("bad .discard.instr_end entry" ); -return -1 ; - } - insn->unret =1 ; - } - + insn->unret =1 ;return 0 ; -} +}static int read_intra_function_calls (struct objtool_file *file) {
@@ -2629,7 +2611,7 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =read_validate_unret_hints (file); + ret =read_annotate (file, __annotate_unret);if (ret)return ret;
Peter Zijlstra
maintainerPATCH 6/11 objtool: Convert ANNOTATE_IGNORE_ALTERNATIVE to ANNOTATE
In reply to Peter Zijlstra--- arch/x86/include/asm/alternative.h | 14 ++--------- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 45 ++++++++---------------------------- 4 files changed, 15 insertions(+), 46 deletions(-)
arch/x86/include/asm/alternative.h +3 -11
@@ -4,6 +4,7 @@#include <linux/types.h> #include <linux/stringify.h> +#include <linux/objtool.h> #include <asm/asm.h> #define ALT_FLAGS_SHIFT 16
@@ -55,11 +56,7 @@* objtool annotation to ignore the alternatives and only consider the original * instruction(s). */ -#define ANNOTATE_IGNORE_ALTERNATIVE \ - "999:\n\t" \ - ".pushsection .discard.ignore_alts\n\t" \ - ".long 999b\n\t" \ - ".popsection\n\t" +#define ANNOTATE_IGNORE_ALTERNATIVE ASM_ANNOTATE(ANNOTYPE_IGNORE_ALTS)/* * The patching flags are part of the upper bits of the @ft_flags parameter when
@@ -349,12 +346,7 @@ static inline int alternatives_text_rese* objtool annotation to ignore the alternatives and only consider the original * instruction(s). */ -.macro ANNOTATE_IGNORE_ALTERNATIVE - .Lannotate_\@: - .pushsection .discard.ignore_alts - .long .Lannotate_\@ - .popsection -.endm +#define ANNOTATE_IGNORE_ALTERNATIVE ANNOTATE type=ANNOTYPE_IGNORE_ALTS/* * Issue one struct alt_instr descriptor entry (need to put it into
include/linux/objtool_types.h +1 -0
@@ -62,5 +62,6 @@ struct unwind_hint {#define ANNOTYPE_INSTR_BEGIN 3#define ANNOTYPE_INSTR_END 4#define ANNOTYPE_UNRET_BEGIN 5 +#define ANNOTYPE_IGNORE_ALTS 6#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -62,5 +62,6 @@ struct unwind_hint {#define ANNOTYPE_INSTR_BEGIN 3#define ANNOTYPE_INSTR_END 4#define ANNOTYPE_UNRET_BEGIN 5 +#define ANNOTYPE_IGNORE_ALTS 6#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +10 -35
63 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 7/11 objtool: Convert ANNOTATE_INTRA_FUNCTION_CALLS to ANNOTATE
In reply to Peter Zijlstra--- include/linux/objtool.h | 16 ++---- include/linux/objtool_types.h | 1 tools/include/linux/objtool_types.h | 1 tools/objtool/check.c | 96 ++++++++++++++---------------------- 4 files changed, 47 insertions(+), 67 deletions(-)
include/linux/objtool.h +6 -10
@@ -66,16 +66,6 @@#else /* __ASSEMBLY__ */ /* -* This macro indicates that the following intra-function call is valid. -* Any non-annotated intra-function call will cause objtool to issue a warning. -*/ -#define ANNOTATE_INTRA_FUNCTION_CALL \ - 999: \ - .pushsection .discard.intra_function_calls; \ - .long 999b; \ - .popsection; - -/* * In asm, there are two kinds of code: normal C-type callable functions and * the rest. The normal callable functions can be called by other code, and * don't do anything unusual with the stack. Such normal callable functions
@@ -152,6 +142,12 @@#define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR +/* +* This macro indicates that the following intra-function call is valid. +* Any non-annotated intra-function call will cause objtool to issue a warning. +*/ +#define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALLS +#endif /* __ASSEMBLY__ */ #else /* !CONFIG_OBJTOOL */
include/linux/objtool_types.h +1 -0
@@ -63,5 +63,6 @@ struct unwind_hint {#define ANNOTYPE_INSTR_END 4#define ANNOTYPE_UNRET_BEGIN 5#define ANNOTYPE_IGNORE_ALTS 6 +#define ANNOTYPE_INTRA_FUNCTION_CALLS 7#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -63,5 +63,6 @@ struct unwind_hint {#define ANNOTYPE_INSTR_END 4#define ANNOTYPE_UNRET_BEGIN 5#define ANNOTYPE_IGNORE_ALTS 6 +#define ANNOTYPE_INTRA_FUNCTION_CALLS 7#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +39 -57
150 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 8/11 objtool: Collapse annotate sequences
In reply to Peter ZijlstraReduce read_annotate() runs by collapsing subsequent runs into a single call.
--- tools/objtool/check.c | 87 ++++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 55 deletions(-)
tools/objtool/check.c +32 -55
@@ -2308,21 +2308,24 @@ static int read_annotate(struct objtool_return 0 ; } -static int __annotate_ignore_alts (struct objtool_file *file,int type,struct instruction *insn) +static int __annotate_early (struct objtool_file *file,int type,struct instruction *insn) { -if (type !=ANNOTYPE_IGNORE_ALTS ) -return 0 ; +switch (type) { +case ANNOTYPE_IGNORE_ALTS : + insn->ignore_alts = true; +break ; - insn->ignore_alts = true; -return 0 ; -} +/* +* Must be before read_unwind_hints() since that needs insn->noendbr. +*/ +case ANNOTYPE_NOENDBR : + insn->noendbr =1 ; +break ; -static int __annotate_noendbr (struct objtool_file *file,int type,struct instruction *insn) -{ -if (type !=ANNOTYPE_NOENDBR ) -return 0 ; +default : +break ; + } - insn->noendbr =1 ;return 0 ; }
@@ -2356,26 +2359,21 @@ static int __annotate_ifc(struct objtoolreturn 0 ; } -static int __annotate_retpoline_safe (struct objtool_file *file,int type,struct instruction *insn) +static int __annotate_late (struct objtool_file *file,int type,struct instruction *insn) { -if (type !=ANNOTYPE_RETPOLINE_SAFE ) -return 0 ; - -if (insn->type !=INSN_JUMP_DYNAMIC && - insn->type !=INSN_CALL_DYNAMIC && - insn->type !=INSN_RETURN && - insn->type !=INSN_NOP ) { -WARN_INSN (insn,"retpoline_safe hint not an indirect jump/call/ret/nop" ); -return -1 ; - } +switch (type) { +case ANNOTYPE_RETPOLINE_SAFE : +if (insn->type !=INSN_JUMP_DYNAMIC && + insn->type !=INSN_CALL_DYNAMIC && + insn->type !=INSN_RETURN && + insn->type !=INSN_NOP ) { +WARN_INSN (insn,"retpoline_safe hint not an indirect jump/call/ret/nop" ); +return -1 ; + } - insn->retpoline_safe = true; -return 0 ; -} + insn->retpoline_safe = true; +break ; -static int __annotate_instr (struct objtool_file *file,int type,struct instruction *insn) -{ -switch (type) {case ANNOTYPE_INSTR_BEGIN : insn->instr++;break ;
@@ -2384,6 +2382,10 @@ static int __annotate_instr(struct objto insn->instr--;break ; +case ANNOTYPE_UNRET_BEGIN : + insn->unret =1 ; +break ; +default :break ; }
@@ -2391,16 +2393,6 @@ static int __annotate_instr(struct objtoreturn 0 ; } -static int __annotate_unret (struct objtool_file *file,int type,struct instruction *insn) -{ -if (type !=ANNOTYPE_UNRET_BEGIN ) -return 0 ; - - insn->unret =1 ; -return 0 ; - -} -/* * Return true if name matches an instrumentation function, where calls to that * function from noinstr code can safely be removed, but compilers won't do so.
@@ -2507,14 +2499,7 @@ static int decode_sections(struct objtooadd_ignores (file);add_uaccess_safe (file); - ret =read_annotate (file, __annotate_ignore_alts); -if (ret) -return ret; - -/* -* Must be before read_unwind_hints() since that needs insn->noendbr. -*/ - ret =read_annotate (file, __annotate_noendbr); + ret =read_annotate (file, __annotate_early);if (ret)return ret;
@@ -2560,15 +2545,7 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =read_annotate (file, __annotate_retpoline_safe); -if (ret) -return ret; - - ret =read_annotate (file, __annotate_instr); -if (ret) -return ret; - - ret =read_annotate (file, __annotate_unret); + ret =read_annotate (file, __annotate_late);if (ret)return ret;
Peter Zijlstra
maintainerPATCH 9/11 x86/kvm/emulate: Implement test_cc() in C
In reply to Peter ZijlstraCurrent test_cc() uses the fastop infrastructure to test flags using SETcc instructions. However, int3_emulate_jcc() already fully implements the flags->CC mapping, use that.
Removes a pile of gnarly asm.
--- arch/x86/include/asm/text-patching.h | 20 +++++++++++++------- arch/x86/kvm/emulate.c | 34 ++-------------------------------- 2 files changed, 15 insertions(+), 39 deletions(-)
arch/x86/include/asm/text-patching.h +13 -7
@@ -186,9 +186,9 @@ void int3_emulate_ret(struct pt_regs *re }static __always_inline -void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +bool __emulate_cc(unsigned long flags, u8 cc) { -static const unsigned long jcc_mask[6 ] = { +static const unsigned long cc_mask[6 ] = { [0 ] =X86_EFLAGS_OF , [1 ] =X86_EFLAGS_CF , [2 ] =X86_EFLAGS_ZF ,
@@ -201,15 +201,21 @@ void int3_emulate_jcc(struct pt_regs *rebool match;if (cc <0xc ) { - match = regs->flags & jcc_mask[cc >>1 ]; + match = flags & cc_mask[cc >>1 ]; }else { - match = ((regs->flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ - ((regs->flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT ); + match = ((flags &X86_EFLAGS_SF ) >>X86_EFLAGS_SF_BIT ) ^ + ((flags &X86_EFLAGS_OF ) >>X86_EFLAGS_OF_BIT );if (cc >=0xe ) - match = match || (regs->flags &X86_EFLAGS_ZF ); + match = match || (flags &X86_EFLAGS_ZF ); } -if ((match && !invert) || (!match && invert)) +return (match && !invert) || (!match && invert); +} + +static __always_inline +void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp) +{ +if (__emulate_cc (regs->flags, cc)) ip += disp;int3_emulate_jmp (regs, ip);
arch/x86/kvm/emulate.c +2 -32
@@ -26,6 +26,7 @@#include <asm/debugreg.h> #include <asm/nospec-branch.h> #include <asm/ibt.h> +#include <asm/text-patching.h> #include "x86.h" #include "tss.h"
@@ -416,31 +417,6 @@ static int fastop(struct x86_emulate_ctx ON64(FOP3E(op##q, rax, rdx, cl)) \ FOP_END -/* Special case for SETcc - 1 instruction per cc */ -#define FOP_SETCC (op) \ - FOP_FUNC(op) \ - #op " %al \n\t" \ - FOP_RET(op) - -FOP_START(setcc) -FOP_SETCC(seto) -FOP_SETCC(setno) -FOP_SETCC(setc) -FOP_SETCC(setnc) -FOP_SETCC(setz) -FOP_SETCC(setnz) -FOP_SETCC(setbe) -FOP_SETCC(setnbe) -FOP_SETCC(sets) -FOP_SETCC(setns) -FOP_SETCC(setp) -FOP_SETCC(setnp) -FOP_SETCC(setl) -FOP_SETCC(setnl) -FOP_SETCC(setle) -FOP_SETCC(setnle) -FOP_END; - FOP_START(salc) FOP_FUNC(salc)"pushf; sbb %al, %al; popf \n\t"
@@ -1063,13 +1039,7 @@ static int em_bsr_c(struct x86_emulate_c static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) { -u8 rc; -void (*fop)(void ) = (void *)em_setcc +FASTOP_SIZE * (condition &0xf ); - - flags = (flags &EFLAGS_MASK ) |X86_EFLAGS_IF ; - asm("push %[flags]; popf; " CALL_NOSPEC - :"=a" (rc) : [thunk_target]"r" (fop), [flags]"r" (flags)); -return rc; +return __emulate_cc (flags, condition &0xf ); }static void fetch_register_operand (struct operand *op)
Peter Zijlstra
maintainerPATCH 10/11 x86/nospec: JMP_NOSPEC
In reply to Peter Zijlstra--- arch/x86/include/asm/nospec-branch.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)
arch/x86/include/asm/nospec-branch.h +32 -0
@@ -403,6 +403,17 @@ static inline void call_depth_return_thu "call *%[thunk_target]\n", \ X86_FEATURE_RETPOLINE_LFENCE) +# define JMP_NOSPEC \ + ALTERNATIVE_2( \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + "jmp __x86_indirect_thunk_%V[thunk_target]\n", \ + X86_FEATURE_RETPOLINE, \ + "lfence;\n" \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + X86_FEATURE_RETPOLINE_LFENCE) +# define THUNK_TARGET (addr) [thunk_target] "r" (addr)#else /* CONFIG_X86_32 */
@@ -433,10 +444,31 @@ static inline void call_depth_return_thu "call *%[thunk_target]\n", \ X86_FEATURE_RETPOLINE_LFENCE) +# define JMP_NOSPEC \ + ALTERNATIVE_2( \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + " jmp 901f;\n" \ + " .align 16\n" \ + "901: call 903f;\n" \ + "902: pause;\n" \ + " lfence;\n" \ + " jmp 902b;\n" \ + " .align 16\n" \ + "903: lea 4(%%esp), %%esp;\n" \ + " pushl %[thunk_target];\n" \ + " ret;\n" \ + X86_FEATURE_RETPOLINE, \ + "lfence;\n" \ + ANNOTATE_RETPOLINE_SAFE \ + "jmp *%[thunk_target]\n", \ + X86_FEATURE_RETPOLINE_LFENCE) +# define THUNK_TARGET (addr) [thunk_target] "rm" (addr)#endif #else /* No retpoline for C / inline asm */ # define CALL_NOSPEC "call *%[thunk_target]\n" +# define JMP_NOSPEC "jmp *%[thunk_target]\n"# define THUNK_TARGET (addr) [thunk_target] "rm" (addr)#endif
Peter Zijlstra
maintainerPATCH 11/11 x86/kvm/emulate: Avoid RET for fastops
In reply to Peter ZijlstraSince there is only a single fastop() function, convert the FASTOP stuff from CALL_NOSPEC+RET to JMP_NOSPEC+JMP, avoiding the return thunks and all that jazz.
Specifically FASTOPs rely on the return thunk to preserve EFLAGS, which not all of them can trivially do (call depth tracing suffers here).
Objtool strenuously complains about this:
- indirect call without a .rodata, fails to determine JUMP_TABLE, annotate
- fastop functions fall through, exception
- unreachable instruction after fastop_return, save/restore
--- arch/x86/kvm/emulate.c | 20 +++++++++++++++----- include/linux/objtool_types.h | 1 + tools/include/linux/objtool_types.h | 1 + tools/objtool/check.c | 11 ++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-)
arch/x86/kvm/emulate.c +15 -5
@@ -285,8 +285,8 @@ static void invalidate_registers(struct* different operand sizes can be reached by calculation, rather than a jump * table (which would be bigger than the code). * -* The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR -* and 1 for the straight line speculation INT3, leaves 7 bytes for the +* The 16 byte alignment, considering 5 bytes for the JMP, 4 for ENDBR +* and 1 for the straight line speculation INT3, leaves 6 bytes for the * body of the function. Currently none is larger than 4. */ static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop);
@@ -304,7 +304,7 @@ static int fastop(struct x86_emulate_ctx __FOP_FUNC(#name)#define __FOP_RET (name) \ - "11: " ASM_RET \ + "11: jmp fastop_return; int3 \n\t" \ ".size " name ", .-" name "\n\t"#define FOP_RET (name) \
@@ -5071,14 +5071,24 @@ static void fetch_possible_mmx_operand(skvm_read_mmx_reg (op->addr.mm, &op->mm_val); } -static int fastop (struct x86_emulate_ctxt *ctxt,fastop_t fop) +/* +* All the FASTOP magic above relies on there being *one* instance of this +* so it can JMP back, avoiding RET and it's various thunks. +*/ +static noinline int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) {ulong flags = (ctxt->eflags &EFLAGS_MASK ) |X86_EFLAGS_IF ;if (!(ctxt->d & ByteOp)) fop +=__ffs (ctxt->dst.bytes) *FASTOP_SIZE ; - asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n" + asm("push %[flags]; popf \n\t" + UNWIND_HINT(UNWIND_HINT_TYPE_SAVE, 0, 0, 0) +ASM_ANNOTATE (ANNOTYPE_JUMP_TABLE) + JMP_NOSPEC +"fastop_return: \n\t" + UNWIND_HINT(UNWIND_HINT_TYPE_RESTORE, 0, 0, 0) +"pushf; pop %[flags]\n" : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags), [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT : "c"(ctxt->src2.val));
include/linux/objtool_types.h +1 -0
@@ -64,5 +64,6 @@ struct unwind_hint {#define ANNOTYPE_UNRET_BEGIN 5#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALLS 7 +#define ANNOTYPE_JUMP_TABLE 8#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/include/linux/objtool_types.h +1 -0
@@ -64,5 +64,6 @@ struct unwind_hint {#define ANNOTYPE_UNRET_BEGIN 5#define ANNOTYPE_IGNORE_ALTS 6#define ANNOTYPE_INTRA_FUNCTION_CALLS 7 +#define ANNOTYPE_JUMP_TABLE 8#endif /* _LINUX_OBJTOOL_TYPES_H */
tools/objtool/check.c +10 -1
@@ -2386,6 +2386,14 @@ static int __annotate_late(struct objtoo insn->unret =1 ;break ; +/* +* Must be after add_jump_table(); for it doesn't set a sane +* _jump_table value. +*/ +case ANNOTYPE_JUMP_TABLE : + insn->_jump_table = (void *)1 ; +break ; +default :break ; }
@@ -3459,7 +3467,8 @@ static int validate_branch(struct objtooif (func &&insn_func (insn) && func !=insn_func (insn)->pfunc) {/* Ignore KCFI type preambles, which always fall through */ if (!strncmp (func->name,"__cfi_" ,6 ) || - !strncmp (func->name,"__pfx_" ,6 )) + !strncmp (func->name,"__pfx_" ,6 ) || + !strcmp (insn_func (insn)->name,"fastop" ))return 0 ; WARN("%s() falls through to next function %s()",