everything text-poke: ftrace, modules, static_call and jump_label
Subsystems (27)
Peter Zijlstra
maintainerPATCH RESEND 1/1 everything text-poke: ftrace, modules, static_call and jump_label
Because typing on a Monday is hard, here's a resend with the LKML address corrected.
---
Hi All,
Here are 4 series that have accumulated in my queue for a while and I figured I'd 'finish' and send out again.
The first series is rewriting x86/ftrace to use the normal text_poke() interfaces we have, which rids us of the last major W^X violation and a whole second implementation of the INT3 breakpoint code patching infrastructure.
The second series cleans up various module stuff.
The third series is a refresh of the static_call() implementation. I've been meaning to use that to get rid of a number of indirect calls in the x86 pmu implementation (RETPOLINE made them so much more painful), but also other people have been asking for it. This series has a lot of new patches since last time, most all a result of me trying to actually use it.
The fourth series is something that has been languishing in my queue for a long while and I figure I'd send it out to see if anybody has any clever ideas on how to proceed with it.
They're all related, and the 3rd and 4rd series depend on the first two. But instead of sending a giant 30+ patch series, I figured I'd send you 4 smaller series instead ;-)
Enjoy!
Peter Zijlstra
maintainercover Variable size jump_label support
These here patches are something I've been poking at for a while, enabling jump_label to use 2 byte jumps/nops.
It _almost_ works :-/
That is, you can build some kernels with it (x86_64-defconfig for example works just fine).
The problem comes when GCC generates a branch into another section, mostly .text.unlikely. At that point GAS just gives up and throws a fit (more details in the last patch).
Aside from anyone coming up with a really clever GAS trick, I don't see how we can do this other than:
- get binutils/gas 'fixed', for this there's two possible approaches:
* add some extra operators such that something like:
.set disp %[l_yes] ~ 1b ? (%[l_yes] - (1b + 2)) : 128
works, where the new '~' infix operator would indicate left and right hand operands are from the same section, and the ?: conditional operator would be added to make it all work.
* add a 'fake' mnemonic for x86 that directly generates the right NOP:
nojmp{.d8,d32} %target
which would completely mirror how the existing 'jmp{.d8,d32} %target' works, except it would emit single instruction NOP2/NOP5.
- use 'jmp' and get objtool to rewrite the text. Steven has earlier proposed something like that (using recordmcount) and Linus hated that.
Peter Zijlstra
maintainerRFC PATCH 1/9 jump_label, x86: Strip ASM jump_label support
In prepration for variable size jump_label support; remove all ASM bits that are not used.
--- arch/x86/entry/calling.h | 2 +- arch/x86/include/asm/jump_label.h | 28 ++++------------------------ 2 files changed, 5 insertions(+), 25 deletions(-)
arch/x86/entry/calling.h +1 -1
@@ -337,7 +337,7 @@ For 32-bit we have the following convent .macro CALL_enter_from_user_mode#ifdef CONFIG_CONTEXT_TRACKING #ifdef CONFIG_JUMP_LABEL - STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 + STATIC_BRANCH_FALSE_LIKELY .Lafter_call_\@, context_tracking_enabled#endif call enter_from_user_mode .Lafter_call_\@:
arch/x86/include/asm/jump_label.h +4 -24
@@ -55,36 +55,16 @@ static __always_inline bool arch_static_#else /* __ASSEMBLY__ */ -.macro STATIC_JUMP_IF_TRUE target, key, def +.macro STATIC_BRANCH_FALSE_LIKELY target, key .Lstatic_jump_\@: - .if \def/* Equivalent to "jmp.d32 \target" */ .byte 0xe9 - .long \target - .Lstatic_jump_after_\@ -.Lstatic_jump_after_\@: - .else - .byte STATIC_KEY_INIT_NOP - .endif - .pushsection __jump_table, "aw" - _ASM_ALIGN - .long .Lstatic_jump_\@ - ., \target - . - _ASM_PTR \key - . - .popsection -.endm + .long \target - (. + 4) -.macro STATIC_JUMP_IF_FALSE target, key, def -.Lstatic_jump_\@: - .if \def - .byte STATIC_KEY_INIT_NOP - .else -/* Equivalent to "jmp.d32 \target" */ - .byte 0xe9 - .long \target - .Lstatic_jump_after_\@ -.Lstatic_jump_after_\@: - .endif .pushsection __jump_table, "aw" _ASM_ALIGN - .long .Lstatic_jump_\@ - ., \target - . + .long .Lstatic_jump_\@ - . + .long \target - . _ASM_PTR \key + 1 - . .popsection .endm
Peter Zijlstra
maintainerRFC PATCH 2/9 jump_label, x86: Factor out the __jump_table generation
In reply to Peter ZijlstraBoth arch_static_branch() and arch_static_branch_jump() have the same blurb to generate the __jump_table entry, share it.
--- arch/x86/include/asm/jump_label.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-)
arch/x86/include/asm/jump_label.h +12 -12
@@ -20,15 +20,19 @@#include <linux/stringify.h> #include <linux/types.h> +#define JUMP_TABLE_ENTRY \ + ".pushsection __jump_table, \"aw\" \n\t" \ + _ASM_ALIGN "\n\t" \ + ".long 1b - . \n\t" \ + ".long %l[l_yes] - . \n\t" \ + _ASM_PTR "%c0 + %c1 - .\n\t" \ + ".popsection \n\t" + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" - ".pushsection __jump_table, \"aw\" \n\t" - _ASM_ALIGN "\n\t" - ".long 1b - ., %l[l_yes] - . \n\t" - _ASM_PTR "%c0 + %c1 - .\n\t" - ".popsection \n\t" + JUMP_TABLE_ENTRY : : "i" (key), "i" (branch) : : l_yes);return false;
@@ -39,13 +43,9 @@ static __always_inline bool arch_static_ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { asm_volatile_goto("1:" -".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" -"2:\n\t" -".pushsection __jump_table, \"aw\" \n\t" - _ASM_ALIGN"\n\t" -".long 1b - ., %l[l_yes] - . \n\t" - _ASM_PTR"%c0 + %c1 - .\n\t" -".popsection \n\t" + ".byte 0xe9 \n\t" + ".long %l[l_yes] - (. + 4) \n\t" + JUMP_TABLE_ENTRY : : "i" (key), "i" (branch) : : l_yes);return false;
Peter Zijlstra
maintainerRFC PATCH 3/9 jump_label, x86: Remove init NOP optimization
In reply to Peter ZijlstraInstead of checking if the emitted (default) NOP is the ideal NOP, and conditionally rewrite the NOP, just rewrite the NOP.
This shouldn't be a problem because init / module_load uses text_poke_early() which is cheap and this saves us from having to go figure out which NOP to compare against when we go variable size.
--- arch/x86/kernel/jump_label.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-)
arch/x86/kernel/jump_label.c +4 -15
@@ -137,21 +137,10 @@ __init_or_module void arch_jump_label_tr enum jump_label_type type) {/* -* This function is called at boot up and when modules are -* first loaded. Check if the default nop, the one that is -* inserted at compile time, is the ideal nop. If it is, then -* we do not need to update the nop, and we can leave it as is. -* If it is not, then we need to update the nop to the ideal nop. +* Rewrite the NOP on init / module-load to ensure we got the ideal +* nop. Don't bother with trying to figure out what size and what nop +* it should be for now, simply do an unconditional rewrite. */ -if (jlstate ==JL_STATE_START ) { -const unsigned char default_nop[] = {STATIC_KEY_INIT_NOP }; -const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; - -if (memcmp (ideal_nop, default_nop,5 ) !=0 ) - jlstate =JL_STATE_UPDATE ; -else - jlstate =JL_STATE_NO_UPDATE ; - } -if (jlstate ==JL_STATE_UPDATE ) +if (jlstate ==JL_STATE_UPDATE || jlstate ==JL_STATE_START )jump_label_transform (entry, type,1 ); }
Peter Zijlstra
maintainerRFC PATCH 4/9 jump_label, x86: Improve error when we fail expected text
In reply to Peter ZijlstraThere is only a single usage site left, remove the function and extend the print to include more information, like the expected text and the patch type.
--- arch/x86/kernel/jump_label.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-)
arch/x86/kernel/jump_label.c +10 -13
@@ -16,17 +16,6 @@#include <asm/alternative.h> #include <asm/text-patching.h> -static void bug_at (const void *ip,int line) -{ -/* -* The location is not an op that we were expecting. -* Something went wrong. Crash the box, as something could be -* corrupting the kernel. -*/ -pr_crit ("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n" , ip, ip, ip, line); -BUG (); -} -static const void *__jump_label_set_jump_code (struct jump_entry *entry,enum jump_label_type type,int init) {
@@ -49,8 +38,16 @@ __jump_label_set_jump_code(struct jump_e expect = code; line = __LINE__; } -if (memcmp (addr, expect,JUMP_LABEL_NOP_SIZE )) -bug_at (addr, line); +if (memcmp (addr, expect,JUMP_LABEL_NOP_SIZE )) { +/* +* The location is not an op that we were expecting. +* Something went wrong. Crash the box, as something could be +* corrupting the kernel. +*/ +pr_crit ("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) line:%d init:%d type:%d\n" , + addr, addr, addr, expect, line, init, type); +BUG (); + }if (type ==JUMP_LABEL_NOP ) code = ideal_nop;
Peter Zijlstra
maintainerRFC PATCH 5/9 jump_label, x86: Introduce jump_entry_size()
In reply to Peter ZijlstraThis allows architectures to have variable sized jumps.
--- arch/x86/include/asm/jump_label.h | 4 ++-- arch/x86/kernel/jump_label.c | 5 +++++ include/linux/jump_label.h | 9 +++++++++ kernel/jump_label.c | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-)
arch/x86/include/asm/jump_label.h +2 -2
@@ -4,8 +4,6 @@#define HAVE_JUMP_LABEL_BATCH -#define JUMP_LABEL_NOP_SIZE 5 -#ifdef CONFIG_X86_64 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC#else
@@ -53,6 +51,8 @@ static __always_inline bool arch_static_ return true; } +extern int arch_jump_entry_size (struct jump_entry *entry); +#else /* __ASSEMBLY__ */ .macro STATIC_BRANCH_FALSE_LIKELY target, key
arch/x86/kernel/jump_label.c +5 -0
@@ -16,6 +16,11 @@#include <asm/alternative.h> #include <asm/text-patching.h> +int arch_jump_entry_size (struct jump_entry *entry) +{ +return JMP32_INSN_SIZE ; +} +static const void *__jump_label_set_jump_code (struct jump_entry *entry,enum jump_label_type type,int init) {
include/linux/jump_label.h +9 -0
@@ -176,6 +176,15 @@ static inline void jump_entry_set_init(s entry->key |=2 ; } +static inline int jump_entry_size (struct jump_entry *entry) +{ +#ifdef JUMP_LABEL_NOP_SIZE +return JUMP_LABEL_NOP_SIZE ; +#else +return arch_jump_entry_size (entry); +#endif +} +#endif #endif
kernel/jump_label.c +1 -1
@@ -309,7 +309,7 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit)static int addr_conflict (struct jump_entry *entry,void *start,void *end) {if (jump_entry_code (entry) <= (unsigned long )end && -jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE > (unsigned long )start) +jump_entry_code (entry) +jump_entry_size (entry) > (unsigned long )start)return 1 ;return 0 ;
Peter Zijlstra
maintainerRFC PATCH 6/9 jump_label, x86: Add variable length patching support
In reply to Peter ZijlstraThis allows the patching to to emit 2 byte JMP/NOP instruction in addition to the 5 byte JMP/NOP we already did. This allows for more compact code.
This code is not yet used, as we don't emit shorter code at compile time yet.
--- arch/x86/include/asm/jump_label.h | 8 +++-- arch/x86/include/asm/nops.h | 1 arch/x86/kernel/jump_label.c | 60 ++++++++++++++++++++++++-------------- 3 files changed, 45 insertions(+), 24 deletions(-)
arch/x86/include/asm/jump_label.h +5 -3
@@ -5,9 +5,11 @@#define HAVE_JUMP_LABEL_BATCH #ifdef CONFIG_X86_64 -# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC +# define STATIC_KEY_NOP2 P6_NOP2 +# define STATIC_KEY_NOP5 P6_NOP5_ATOMIC#else -# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC +# define STATIC_KEY_NOP2 GENERIC_NOP2 +# define STATIC_KEY_NOP5 GENERIC_NOP5_ATOMIC#endif #include <asm/asm.h>
@@ -29,7 +31,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" - ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" + ".byte " __stringify(STATIC_KEY_NOP5) "\n\t" JUMP_TABLE_ENTRY : : "i" (key), "i" (branch) : : l_yes);
arch/x86/include/asm/nops.h +1 -0
@@ -5,6 +5,7 @@/* * Define nops for use with alternative() and for tracing. * +* *_NOP2 must be a single instruction * *_NOP5_ATOMIC must be a single instruction. */
arch/x86/kernel/jump_label.c +39 -21
117 lines not on this page · raw message
Peter Zijlstra
maintainerRFC PATCH 7/9 jump_label,objtool: Validate variable size jump labels
In reply to Peter ZijlstraSince variable sized jump_label support is somewhat 'inspired', ensure objtool validates the tricky bits. Specifically it is important that the displacement for the 2 byte jump is limited to a single byte -- since this is the hardest part of the whole scheme and relies on somewhat dodgy GAS tricks.
--- tools/objtool/check.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)
tools/objtool/check.c +25 -1
@@ -817,8 +817,32 @@ static int handle_jump_alt(struct objtoo struct instruction *orig_insn, struct instruction **new_insn) { -if (orig_insn->type ==INSN_NOP ) +if ((orig_insn->len !=2 ) && (orig_insn->len !=5 )) { +WARN_FUNC ("jump_label: unsupported INSN length: %d" , + orig_insn->sec, orig_insn->offset, orig_insn->len); +return -1 ; + } + +if (orig_insn->type ==INSN_NOP ) { +long disp; + +if (orig_insn->len ==2 ) { +if (special_alt->orig_sec != special_alt->new_sec) { +WARN_FUNC ("jump_label: JMP8 crossing sections" , + orig_insn->sec, orig_insn->offset); +return -1 ; + } + + disp = special_alt->new_off - (special_alt->orig_off +2 ); +if ((disp >>31 ) != (disp >>7 )) { +WARN_FUNC ("jump_label: JMP8 displacement not a byte" , + orig_insn->sec, orig_insn->offset); +return -1 ; + } + } +return 0 ; + }if (orig_insn->type !=INSN_JUMP_UNCONDITIONAL ) { WARN_FUNC("unsupported instruction at jump label",
Peter Zijlstra
maintainerRFC PATCH 8/9 jump_label,objtool: Generate possible statistics
In reply to Peter Zijlstrax86_64-defconfig x86_64-allmodconfig-UBSAN-KASAN
NOP2 1641 21796
JMP8 48 114
NOP5 1010 31042
JMP32 29 91
Which results in a possible 3*(1641+48) ~ 5k saving for defconfig and 3*(21796+114) ~ 64k saving for allmodconfig.
--- tools/objtool/check.c | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-)
tools/objtool/check.c +41 -2
@@ -817,6 +817,8 @@ static int handle_jump_alt(struct objtoostruct instruction *orig_insn, struct instruction **new_insn) { + long disp; +if ((orig_insn->len !=2 ) && (orig_insn->len !=5 )) {WARN_FUNC ("jump_label: unsupported INSN length: %d" , orig_insn->sec, orig_insn->offset, orig_insn->len);
@@ -824,9 +826,12 @@ static int handle_jump_alt(struct objtoo }if (orig_insn->type ==INSN_NOP ) { -long disp; -if (orig_insn->len ==2 ) { +switch (orig_insn->len) { +case 2 : +WARN_FUNC ("jump_label: NOP2 present" , + orig_insn->sec, orig_insn->offset); +if (special_alt->orig_sec != special_alt->new_sec) {WARN_FUNC ("jump_label: JMP8 crossing sections" , orig_insn->sec, orig_insn->offset);
@@ -839,6 +844,20 @@ static int handle_jump_alt(struct objtoo orig_insn->sec, orig_insn->offset);return -1 ; } +break ; + +case 5 : +if (special_alt->orig_sec == special_alt->new_sec) { + disp = special_alt->new_off - (special_alt->orig_off +2 ); +if ((disp>>31 ) == (disp>>7 )) { +WARN_FUNC ("jump_label: NOP2 possible" , + orig_insn->sec, orig_insn->offset); +break ; + } + } +WARN_FUNC ("jump_label: NOP5 present" , + orig_insn->sec, orig_insn->offset); +break ; }return 0 ;
@@ -850,6 +869,26 @@ static int handle_jump_alt(struct objtooreturn -1 ; } +switch (orig_insn->len) { +case 2 : +WARN_FUNC ("jump_label: JMP8 present" , + orig_insn->sec, orig_insn->offset); +break ; + +case 5 : +if (special_alt->orig_sec == special_alt->new_sec) { + disp = special_alt->new_off - (special_alt->orig_off +2 ); +if ((disp>>31 ) == (disp>>7 )) { +WARN_FUNC ("jump_label: JMP8 possible" , + orig_insn->sec, orig_insn->offset); +break ; + } + } +WARN_FUNC ("jump_label: JMP32 present" , + orig_insn->sec, orig_insn->offset); +break ; + } + *new_insn =list_next_entry (orig_insn, list);return 0 ; }
Peter Zijlstra
maintainerRFC PATCH 9/9 jump_label, x86: Enable JMP8/NOP2 support
In reply to Peter ZijlstraEnable and emit short JMP/NOP jump_label entries.
A lot of the jumps are in fact short, like around tracepoints:
0000 0000000000000920 <native_read_msr>: | 0000 0000000000000920 <native_read_msr>: 0000 920: 53 push %rbx | 0000 920: 53 push %rbx 0001 921: 89 f9 mov %edi,%ecx | 0001 921: 89 f9 mov %edi,%ecx 0003 923: 0f 32 rdmsr | 0003 923: 0f 32 rdmsr 0005 925: 48 c1 e2 20 shl $0x20,%rdx | 0005 925: 48 c1 e2 20 shl $0x20,%rdx 0009 929: 48 89 d3 mov %rdx,%rbx | 0009 929: 48 89 d3 mov %rdx,%rbx 000c 92c: 48 09 c3 or %rax,%rbx | 000c 92c: 48 09 c3 or %rax,%rbx 000f 92f: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) \ 000f 92f: 66 90 xchg %ax,%ax 0014 934: 48 89 d8 mov %rbx,%rax \ 0011 931: 48 89 d8 mov %rbx,%rax 0017 937: 5b pop %rbx \ 0014 934: 5b pop %rbx 0018 938: c3 retq \ 0015 935: c3 retq 0019 939: 48 89 de mov %rbx,%rsi \ 0016 936: 48 89 de mov %rbx,%rsi 001c 93c: 31 d2 xor %edx,%edx \ 0019 939: 31 d2 xor %edx,%edx 001e 93e: e8 00 00 00 00 callq 943 <native_read_msr+0x23> \ 001b 93b: e8 00 00 00 00 callq 940 <native_read_msr+0x20> 001f 93f: R_X86_64_PLT32 do_trace_read_msr-0x4 \ 001c 93c: R_X86_64_PLT32 do_trace_read_msr-0x4 0023 943: 48 89 d8 mov %rbx,%rax \ 0020 940: 48 89 d8 mov %rbx,%rax 0026 946: 5b pop %rbx \ 0023 943: 5b pop %rbx 0027 947: c3 retq \ 0024 944: c3 retq
.rela__jump_table
000000000010 000200000002 R_X86_64_PC32 0000000000000000 .text + 92f 000000000014 000200000002 R_X86_64_PC32 0000000000000000 .text + 939 (or 936) 000000000018 014500000018 R_X86_64_PC64 0000000000000000 __tracepoint_read_msr + 8
The below patch works as long as the jump doesn't cross sections; the moment GCC generates a branch crossing sections and feeds it into our asm-goto things come apart like:
/tmp/ccM70dCh.s: Assembler messages:
/tmp/ccM70dCh.s: Error: invalid operands (.text.unlikely and .text sections) for -' when setting disp' ../arch/x86/include/asm/jump_label.h:39: Error: invalid operands (.text.unlikely and *ABS* sections) for >>' ../arch/x86/include/asm/jump_label.h:39: Error: invalid operands (.text.unlikely and *ABS* sections) for >>'
Which is really unfortunate since it is a completely sane thing to happen. We really need a GAS extention to handle this :-/
All we really need is to detect the two offsets are from different sections and punt to the 5 byte nop. But AFAICT there is nothing that can do that.
--- arch/x86/Kconfig | 10 ++++++++++ arch/x86/include/asm/jump_label.h | 36 +++++++++++++++++++++++++++++++++++- arch/x86/kernel/jump_label.c | 17 +++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-)
arch/x86/Kconfig +10 -0
@@ -230,6 +230,16 @@ config X86 select X86_FEATURE_NAMES if PROC_FS select PROC_PID_ARCH_STATUS if PROC_FS +# +# This mostly depends on the asm ".nops 5" directive existing and emitting a +# single instruction nop, this is true for x86_64, but not for i386, which +# violates the single instruction constraint. +# +config CC_HAS_ASM_NOPS + def_bool y + depends on X86_64 + depends on $(success,echo 'void foo(void) { asm inline (".nops 5"); }' | $(CC) -x c - -c -o /dev/null) + config INSTRUCTION_DECODER def_bool y depends on KPROBES || PERF_EVENTS || UPROBES
arch/x86/include/asm/jump_label.h +35 -1
@@ -4,6 +4,10 @@#define HAVE_JUMP_LABEL_BATCH +#ifdef CONFIG_CC_HAS_ASM_NOPS +#define HAVE_JUMP_LABEL_VARIABLE +#endif +#ifdef CONFIG_X86_64 # define STATIC_KEY_NOP2 P6_NOP2# define STATIC_KEY_NOP5 P6_NOP5_ATOMIC
@@ -31,7 +35,29 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool branch) {asm_volatile_goto ("1:" +#ifdef HAVE_JUMP_LABEL_VARIABLE +/* +* This comes apart mightily when %[l_yes] and 1b are in +* different sections; like for instance .text and +* .text.unlikely. Sadly there is nothing to actually detect +* and handle this case explicitly. +* +* GAS sucks!! +*/ +".set disp, (%l[l_yes]) - (1b + 2) \n\t" +".set res, (disp >> 31) == (disp >> 7) \n\t" +".set is_byte, -res \n\t" +".set is_long, -(~res) \n\t" + +/* +* This relies on .nops: +* - matching the above STATIC_KEY_NOP* bytes +* - emitting a single instruction nop for 2 and 5 bytes. +*/ +".nops (2*is_byte) + (5*is_long)\n\t" +#else ".byte " __stringify(STATIC_KEY_NOP5) "\n\t" +#endif JUMP_TABLE_ENTRY : : "i" (key), "i" (branch) : : l_yes);
@@ -43,8 +69,13 @@ static __always_inline bool arch_static_ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { asm_volatile_goto("1:" +#ifdef HAVE_JUMP_LABEL_VARIABLE + "jmp %l[l_yes] \n\t" +#else +/* Equivalent to "jmp.d32 \target" */ ".byte 0xe9 \n\t" ".long %l[l_yes] - (. + 4) \n\t" +#endif JUMP_TABLE_ENTRY : : "i" (key), "i" (branch) : : l_yes);
@@ -59,9 +90,12 @@ extern int arch_jump_entry_size(struct j .macro STATIC_BRANCH_FALSE_LIKELY target, key .Lstatic_jump_\@: -/* Equivalent to "jmp.d32 \target" */ +#ifdef HAVE_JUMP_LABEL_VARIABLE + jmp \target +#else .byte 0xe9 .long \target - (. + 4) +#endif .pushsection __jump_table, "aw" _ASM_ALIGN
arch/x86/kernel/jump_label.c +17 -0
@@ -18,7 +18,24 @@int arch_jump_entry_size (struct jump_entry *entry) { +#ifdef HAVE_JUMP_LABEL_VARIABLE +struct insn insn; + +/* +* Because the instruction size heuristic doesn't purely rely on +* displacement, but also on section, and we're hindered by GNU as UB +* to emit the assemble time choice, we have to discover the size at +* runtime. +*/ +kernel_insn_init (&insn, (void *)jump_entry_code (entry),MAX_INSN_SIZE ); +insn_get_length (&insn); +BUG_ON (!insn_complete (&insn)); +BUG_ON (insn.length !=2 && insn.length !=5 ); + +return insn.length; +#else return JMP32_INSN_SIZE ; +#endif }struct jump_label_patch {
Peter Zijlstra
maintainerRe: [RFC][PATCH 0/9] Variable size jump_label support
In reply to Peter ZijlstraIn the failed thread Ingo posted:
Quoted 21 lines · Ingo Molnar
Quoted 12 lines · Peter Zijlstra
These here patches are something I've been poking at for a while, enabling
jump_labelto use 2 byte jumps/nops.It _almost_ works :-/
That is, you can build some kernels with it (x86_64-defconfig for example works just fine).
The problem comes when GCC generates a branch into another section, mostly .text.unlikely. At that point GAS just gives up and throws a fit (more details in the last patch).
Quoted 5 lines
Aside from anyone coming up with a really clever GAS trick, I don't see how we can do this other than:
- use 'jmp' and get objtool to rewrite the text. Steven has earlier proposed something like that (using recordmcount) and Linus hated that.
Quoted 8 lines
As long as GCC+GAS correctly generates a 2-byte or 5-byte JMP depending on the target distance, the objtool solution should work fine, shouldn't it?
I can see the recordmcount solution sucking, it would depend on early kernel patchery. But build time patchery is something we already depend on, so assuming some objtool catastrophy it's a more robust solution, isn't it?
IIRC the recordmcount variant from Steve was also rewriting JMP8 to NOP2 at build time.
I dug this here link out of my IRC logs:
https://lore.kernel.org/lkml/[email protected]/
Looking at that, part of the reason might've been running yet another tool, instead of having one tool do everything.
Re: [RFC][PATCH 0/9] Variable size jump_label support
Quoted 6 lines · Peter Zijlstra
IIRC the recordmcount variant from Steve was also rewriting JMP8 to NOP2 at build time.
I dug this here link out of my IRC logs:
Ancient indeed ...
Quoted 2 lines
Looking at that, part of the reason might've been running yet another tool, instead of having one tool do everything.
Yeah - that too wouldn't be a problem with objtool, as we are running it anyway, right?
So I can see about 2 valid technical reasons why Linus would have objected to that old approach from Steve while finding the objtool approach more acceptable.
Basically the main assumption is that we better never run out of competent objtool experts... :-)
Thanks,
Ingo
Steven Rostedt
maintainerRe: [RFC][PATCH 0/9] Variable size jump_label support
Quoted 17 lines · On Mon, 7 Oct 2019 14:55:19 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 6 lines · Peter Zijlstra
IIRC the recordmcount variant from Steve was also rewriting JMP8 to NOP2 at build time.
I dug this here link out of my IRC logs:
Ancient indeed ...
Quoted 2 lines
Looking at that, part of the reason might've been running yet another tool, instead of having one tool do everything.
Yeah - that too wouldn't be a problem with objtool, as we are running it anyway, right?
Quoted 6 lines
So I can see about 2 valid technical reasons why Linus would have objected to that old approach from Steve while finding the objtool approach more acceptable.
Basically the main assumption is that we better never run out of competent objtool experts... :-)
Actually, even back then I said that it would be best to merge all the tools into one (I just didn't have the time to implement it), and then we could pull this off. I have one of my developers working to merge record-mcount into objtool now (there's been some patches floating around).
Then with a single tool, it shouldn't be controversial.
-- Steve
Peter Zijlstra
maintainerv2 cover Add static_call()
In reply to Peter ZijlstraThis series, which depends on the previous two, introduces static_call().
static_call(), is the idea of static_branch() applied to indirect function calls. Remove a data load (indirection) by modifying the text.
These patches are still based on the work Josh did earlier, but incorporated feedback from the last posting and have a lot of extra patches which resulted from me trying to actually use static_call().
This still relies on objtool to generate the .static_call_sites section, mostly because this is a natural place for x86_64 and works for both GCC and LLVM. Other architectures can pick other means if/when they implement the inline patching. The out-of-line (aka. trampoline) variant doesn't require this.
Peter Zijlstra
maintainerPATCH v2 1/13 compiler.h: Make __ADDRESSABLE() symbol truly unique
From: Josh Poimboeuf <[email protected]>
The __ADDRESSABLE() macro uses the __LINE__ macro to create a temporary symbol which has a unique name. However, if the macro is used multiple times from within another macro, the line number will always be the same, resulting in duplicate symbols.
Make the temporary symbols truly unique by using __UNIQUE_ID instead of __LINE__.
--- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/compiler.h +1 -1
@@ -299,7 +299,7 @@ unsigned long read_word_at_a_time(const*/ #define __ADDRESSABLE (sym) \ static void * __section(.discard.addressable) __used \ - __PASTE(__addressable_##sym, __LINE__) = (void *)&sym; + __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;/** * offset_to_ptr - convert a relative memory offset to an absolute pointer
Peter Zijlstra
maintainerPATCH v2 2/13 static_call: Add basic static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Static calls are a replacement for global function pointers. They use code patching to allow direct calls to be used instead of indirect calls. They give the flexibility of function pointers, but with improved performance. This is especially important for cases where retpolines would otherwise be used, as retpolines can significantly impact performance.
The concept and code are an extension of previous work done by Ard Biesheuvel and Steven Rostedt:
https://lkml.kernel.org/r/[email protected], [email protected]
https://lkml.kernel.org/r/[email protected]
There are two implementations, depending on arch support:
1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL)
2) basic function pointers
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 3 include/linux/static_call.h | 134 ++++++++++++++++++++++++++++++++++++++ include/linux/static_call_types.h | 15 ++++ 3 files changed, 152 insertions(+) create mode 100644 include/linux/static_call.h create mode 100644 include/linux/static_call_types.h
arch/Kconfig +3 -0
@@ -960,6 +960,9 @@ config RELR config ARCH_HAS_MEM_ENCRYPT bool +config HAVE_STATIC_CALL + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/linux/static_call.h new file +134 -0
134 lines not on this page · raw message
include/linux/static_call_types.h new file +15 -0
@@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_PREFIX ____static_call_ +#define STATIC_CALL_PREFIX_STR __stringify(STATIC_CALL_PREFIX) + +#define STATIC_CALL_NAME (name) __PASTE(STATIC_CALL_PREFIX, name) + +#define STATIC_CALL_TRAMP (name) STATIC_CALL_NAME(name##_tramp) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +#endif /* _STATIC_CALL_TYPES_H */
Peter Zijlstra
maintainerRe: [PATCH v2 02/13] static_call: Add basic static call infrastructure
Quoted 1 line · Peter Zijlstra
+#define
STATIC_CALL_PREFIX____static_call_
Yesterday I got an allmodconfig build complaining about symbols being too long, in part due to this prefix. Should we change it to something
like: "__SC__" ?
Peter Zijlstra
maintainerPATCH v2 3/13 static_call: Add inline static call infrastructure
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add infrastructure for an arch-specific CONFIG_HAVE_STATIC_CALL_INLINE option, which is a faster version of CONFIG_HAVE_STATIC_CALL. At runtime, the static call sites are patched directly, rather than using the out-of-line trampolines.
Compared to out-of-line static calls, the performance benefits are more modest, but still measurable. Steven Rostedt did some tracepoint measurements:
https://lkml.kernel.org/r/[email protected]
This code is heavily inspired by the jump label code (aka "static jumps"), as some of the concepts are very similar.
For more details, see the comments in include/linux/static_call.h.
--- arch/Kconfig | 4 include/asm-generic/vmlinux.lds.h | 7 include/linux/module.h | 10 + include/linux/static_call.h | 37 ++++ include/linux/static_call_types.h | 9 + kernel/Makefile | 1 kernel/module.c | 5 kernel/static_call.c | 302 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 kernel/static_call.c
arch/Kconfig +4 -0
@@ -963,6 +963,10 @@ config ARCH_HAS_MEM_ENCRYPT config HAVE_STATIC_CALL bool +config HAVE_STATIC_CALL_INLINE + bool + depends on HAVE_STATIC_CALL + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig"
include/asm-generic/vmlinux.lds.h +7 -0
@@ -333,6 +333,12 @@ KEEP(*(__jump_table)) \ __stop___jump_table = .; +#define STATIC_CALL_DATA \ + . = ALIGN(8); \ + __start_static_call_sites = .; \ + KEEP(*(.static_call_sites)) \ + __stop_static_call_sites = .; +/* * Allow architectures to handle ro_after_init data on their * own by defining an empty RO_AFTER_INIT_DATA.
@@ -342,6 +348,7 @@ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ + STATIC_CALL_DATA \ __end_ro_after_init = .;#endif
include/linux/module.h +10 -0
@@ -22,6 +22,7 @@#include <linux/error-injection.h> #include <linux/tracepoint-defs.h> #include <linux/srcu.h> +#include <linux/static_call_types.h> #include <linux/percpu.h> #include <asm/module.h>
@@ -476,6 +477,10 @@ struct module {unsigned int num_ftrace_callsites;unsigned long *ftrace_callsites;#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE +int num_static_call_sites; +struct static_call_site *static_call_sites; +#endif #ifdef CONFIG_LIVEPATCH bool klp;/* Is this a livepatch module? */
@@ -732,6 +737,11 @@ static inline bool within_module(unsigne {return false; } + +static inline bool within_module_init (unsigned long addr,const struct module *mod) +{ +return false; +}/* Get/put a kernel symbol (calls should be symmetric) */ #define symbol_get (x) ({ extern typeof(x) x __attribute__((weak)); &(x); })
include/linux/static_call.h +36 -1
@@ -78,7 +78,42 @@ extern void arch_static_call_transform(v &STATIC_CALL_TRAMP(name), func); \ }) -#if defined(CONFIG_HAVE_STATIC_CALL) +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + +struct static_call_mod { + struct static_call_mod *next; + struct module *mod; /* for vmlinux, mod == NULL */ + struct static_call_site *sites; +}; + +struct static_call_key { + void *func; + struct static_call_mod *next; +}; + +extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); +extern int static_call_mod_init(struct module *mod); + +#define DEFINE_STATIC_CALL(name, _func) \ + DECLARE_STATIC_CALL(name, _func); \ + struct static_call_key STATIC_CALL_NAME(name) = { \ + .func = _func, \ + .next = NULL, \ + }; \ + __ADDRESSABLE(STATIC_CALL_NAME(name)); \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) + +#define static_call(name) STATIC_CALL_TRAMP(name) + +#define EXPORT_STATIC_CALL(name) \ + EXPORT_SYMBOL(STATIC_CALL_NAME(name)); \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) + +#define EXPORT_STATIC_CALL_GPL(name) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name)); \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) + +#elif defined(CONFIG_HAVE_STATIC_CALL) struct static_call_key { void *func;
include/linux/static_call_types.h +9 -0
@@ -12,4 +12,13 @@#define STATIC_CALL_TRAMP (name) STATIC_CALL_NAME(name##_tramp)#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; +#endif /* _STATIC_CALL_TYPES_H */
kernel/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o obj-$(CONFIG_PERF_EVENTS) += events/
kernel/module.c +5 -0
@@ -3231,6 +3231,11 @@ static int find_module_sections(struct msizeof (*mod->ei_funcs), &mod->num_ei_funcs);#endif +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + mod->static_call_sites =section_objs (info,".static_call_sites" , +sizeof (*mod->static_call_sites), + &mod->num_static_call_sites); +#endif mod->extable =section_objs (info,"__ex_table" ,sizeof (*mod->extable), &mod->num_exentries);
kernel/static_call.c new file +302 -0
302 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v2 4/13 static_call: Avoid kprobes on inline static_call()s
In reply to Peter ZijlstraSimilar to how we disallow kprobes on any other dynamic text (ftrace/jump_label) also disallow kprobes on inline static_call()s.
--- arch/x86/kernel/kprobes/opt.c | 4 +- include/linux/static_call.h | 11 +++++++ kernel/kprobes.c | 2 + kernel/static_call.c | 64 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-)
arch/x86/kernel/kprobes/opt.c +3 -1
@@ -16,6 +16,7 @@#include <linux/kallsyms.h> #include <linux/ftrace.h> #include <linux/frame.h> +#include <linux/static_call.h> #include <asm/text-patching.h> #include <asm/cacheflush.h>
@@ -188,7 +189,8 @@ static int copy_optimized_instructions(u/* Check whether the address range is reserved */ if (ftrace_text_reserved (src, src + len -1 ) ||alternatives_text_reserved (src, src + len -1 ) || -jump_label_text_reserved (src, src + len -1 )) +jump_label_text_reserved (src, src + len -1 ) || +static_call_text_reserved (src, src + len -1 ))return -EBUSY ;return len;
include/linux/static_call.h +11 -0
@@ -93,6 +93,7 @@ struct static_call_key {extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);extern int static_call_mod_init (struct module *mod); +extern int static_call_text_reserved (void *start,void *end);#define DEFINE_STATIC_CALL (name, _func) \ DECLARE_STATIC_CALL(name, _func); \
@@ -137,6 +138,11 @@ void __static_call_update(struct static_cpus_read_unlock (); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))#define EXPORT_STATIC_CALL_GPL (name) EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
@@ -161,6 +167,11 @@ void __static_call_update(struct static_WRITE_ONCE (key->func, func); } +static inline int static_call_text_reserved (void *start,void *end) +{ +return 0 ; +} +#define EXPORT_STATIC_CALL (name) EXPORT_SYMBOL(STATIC_CALL_NAME(name))#define EXPORT_STATIC_CALL_GPL (key) EXPORT_SYMBOL_GPL(STATIC_CALL_NAME(name))
kernel/kprobes.c +2 -0
@@ -35,6 +35,7 @@#include <linux/ftrace.h> #include <linux/cpu.h> #include <linux/jump_label.h> +#include <linux/static_call.h> #include <asm/sections.h> #include <asm/cacheflush.h>
@@ -1539,6 +1540,7 @@ static int check_kprobe_address_safe(strif (!kernel_text_address ((unsigned long ) p->addr) ||within_kprobe_blacklist ((unsigned long ) p->addr) ||jump_label_text_reserved (p->addr, p->addr) || +static_call_text_reserved (p->addr, p->addr) ||find_bug ((unsigned long )p->addr)) { ret = -EINVAL ; goto out;
kernel/static_call.c +64 -0
78 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v2 5/13 x86/static_call: Add out-of-line static call implementation
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the x86 out-of-line static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes.
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/static_call.h | 22 ++++++++++++++++++++++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/static_call.c | 31 +++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 arch/x86/include/asm/static_call.h create mode 100644 arch/x86/kernel/static_call.c
arch/x86/Kconfig +1 -0
@@ -205,6 +205,7 @@ config X86 select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 + select HAVE_STATIC_CALL select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
arch/x86/include/asm/static_call.h new file +22 -0
@@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H + +#include <asm/text-patching.h> + +/* +* For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which +* does a direct jump to the function. The direct jump gets patched by +* static_call_update(). +*/ +#define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ + asm(".pushsection .text, \"ax\" \n" \ + ".align 4 \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + " jmp.d32 " #func " \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */
arch/x86/kernel/Makefile +1 -0
@@ -63,6 +63,7 @@ obj-y += tsc.o tsc_msr.o io_delay.o rt obj-y += pci-iommu_table.o obj-y += resource.o obj-y += irqflags.o +obj-y += static_call.o obj-y += process.o obj-y += fpu/
arch/x86/kernel/static_call.c new file +31 -0
@@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/text-patching.h> + +static void __static_call_transform (void *insn,u8 opcode,void *func) +{ +const void *code =text_gen_insn (opcode, (long )insn, (long )func); + +if (WARN_ONCE (*(u8 *)insn != opcode, +"unexpected static call insn opcode 0x%x at %pS\n" , + opcode, insn)) +return ; + +if (memcmp (insn, code,CALL_INSN_SIZE ) ==0 ) +return ; + +text_poke_bp (insn, code,CALL_INSN_SIZE ,NULL ); +} + +void arch_static_call_transform (void *site,void *tramp,void *func) +{ +mutex_lock (&text_mutex); + +if (tramp) +__static_call_transform (tramp,JMP32_INSN_OPCODE , func); + +mutex_unlock (&text_mutex); +} +EXPORT_SYMBOL_GPL (arch_static_call_transform);
Peter Zijlstra
maintainerPATCH v2 6/13 x86/static_call: Add inline static call implementation for x86-64
In reply to Peter ZijlstraFrom: Josh Poimboeuf <[email protected]>
Add the inline static call implementation for x86-64. For each key, a temporary trampoline is created, named __static_call_tramp_<key>. The trampoline has an indirect jump to the destination function.
Objtool uses the trampoline naming convention to detect all the call sites. It then annotates those call sites in the .static_call_sites section.
During boot (and module init), the call sites are patched to call directly into the destination function. The temporary trampoline is then no longer used.
--- arch/x86/Kconfig | 3 arch/x86/include/asm/static_call.h | 15 +++ arch/x86/kernel/static_call.c | 3 arch/x86/kernel/vmlinux.lds.S | 1 include/asm-generic/vmlinux.lds.h | 6 + tools/include/linux/static_call_types.h | 24 ++++++ tools/objtool/check.c | 127 +++++++++++++++++++++++++++++++- tools/objtool/check.h | 2 tools/objtool/elf.h | 1 tools/objtool/sync-check.sh | 1 10 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 tools/objtool/include/linux/static_call_types.h
arch/x86/Kconfig +2 -1
@@ -206,6 +206,7 @@ config X86 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL + select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK
@@ -221,6 +222,7 @@ config X86 select RTC_MC146818_LIB select SPARSE_IRQ select SRCU + select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE) select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT
@@ -445,7 +447,6 @@ config GOLDFISH config RETPOLINE bool "Avoid speculative indirect branches in kernel" default y - select STACK_VALIDATION if HAVE_STACK_VALIDATION help Compile kernel with the retpoline compiler options to guard against kernel-to-user data leaks by avoiding speculative indirect
arch/x86/include/asm/static_call.h +14 -1
@@ -5,12 +5,25 @@#include <asm/text-patching.h> /* +* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which +* uses the current value of the key->func pointer to do an indirect jump to +* the function. This trampoline is only used during boot, before the call +* sites get patched by static_call_update(). The name of this trampoline has +* a magical aspect: objtool uses it to find static call sites so it can create +* the .static_call_sites section. +* * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which * does a direct jump to the function. The direct jump gets patched by * static_call_update(). +* +* Having the trampoline in a special section has two benefits: +* - it makes it 'easy' for objtool to find all the call-sites; +* - it forces GCC to emit a JMP.d32 when it does tail-call optimization on +* the call; since you cannot compute the relative displacement across +* sections. */ #define ARCH_DEFINE_STATIC_CALL_TRAMP (name, func) \ - asm(".pushsection .text, \"ax\" \n" \ + asm(".pushsection .static_call.text, \"ax\" \n" \ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \
arch/x86/kernel/static_call.c +3 -0
@@ -26,6 +26,9 @@ void arch_static_call_transform(void *siif (tramp)__static_call_transform (tramp,JMP32_INSN_OPCODE , func); +if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) +__static_call_transform (site,CALL_INSN_OPCODE , func); +mutex_unlock (&text_mutex); }EXPORT_SYMBOL_GPL (arch_static_call_transform);
arch/x86/kernel/vmlinux.lds.S +1 -0
@@ -133,6 +133,7 @@ SECTIONS IRQENTRY_TEXT ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT + STATIC_CALL_TEXT *(.fixup) *(.gnu.warning)
include/asm-generic/vmlinux.lds.h +6 -0
@@ -576,6 +576,12 @@ *(.softirqentry.text) \ __softirqentry_text_end = .; +#define STATIC_CALL_TEXT \ + ALIGN_FUNCTION(); \ + __static_call_text_start = .; \ + *(.static_call.text) \ + __static_call_text_end = .; +/* Section used for early init (in .S files) */ #define HEAD_TEXT KEEP(*(.head.text))
tools/include/linux/static_call_types.h new file +24 -0
@@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _STATIC_CALL_TYPES_H +#define _STATIC_CALL_TYPES_H + +#include <linux/stringify.h> + +#define STATIC_CALL_PREFIX ____static_call_ +#define STATIC_CALL_PREFIX_STR __stringify(STATIC_CALL_PREFIX) + +#define STATIC_CALL_NAME (name) __PASTE(STATIC_CALL_PREFIX, name) + +#define STATIC_CALL_TRAMP (name) STATIC_CALL_NAME(name##_tramp) +#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name)) + +/* +* The static call site table needs to be created by external tooling (objtool +* or a compiler plugin). +*/ +struct static_call_site { +s32 addr; +s32 key; +}; + +#endif /* _STATIC_CALL_TYPES_H */
tools/objtool/check.c +126 -1
181 lines not on this page · raw message
tools/objtool/check.h +2 -0
@@ -28,6 +28,7 @@ struct insn_state {struct instruction {struct list_head list;struct hlist_node hash; +struct list_head static_call_node;struct section *sec;unsigned long offset;unsigned int len;
@@ -51,6 +52,7 @@ struct objtool_file {struct elf *elf;struct list_head insn_list;DECLARE_HASHTABLE (insn_hash,16 ); +struct list_head static_call_list;bool ignore_unreachables, c_file, hints, rodata; };
tools/objtool/elf.h +1 -0
@@ -51,6 +51,7 @@ struct symbol {unsigned int len;struct symbol *pfunc, *cfunc, *alias;bool uaccess_safe; +bool static_call_tramp; };struct rela {
tools/objtool/sync-check.sh +1 -0
@@ -6,6 +6,7 @@ arch/x86/include/asm/inat_types.h arch/x86/include/asm/orc_types.h arch/x86/lib/x86-opcode-map.txt arch/x86/tools/gen-insn-attr-x86.awk +include/linux/static_call_types.h ' check_2 () {
Peter Zijlstra
maintainerPATCH v2 7/13 static_call: Simple self-test
In reply to Peter Zijlstra--- arch/Kconfig | 6 ++++++ kernel/static_call.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+)
arch/Kconfig +6 -0
@@ -103,6 +103,12 @@ config STATIC_KEYS_SELFTEST help Boot time self-test of the branch patching code. +config STATIC_CALL_SELFTEST + bool "Static call selftest" + depends on HAVE_STATIC_CALL + help + Boot time self-test of the call patching code. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES
kernel/static_call.c +28 -0
@@ -364,3 +364,31 @@ static void __init static_call_init(void#endif }early_initcall (static_call_init); + +#ifdef CONFIG_STATIC_CALL_SELFTEST + +static int func_a (int x) +{ +return x+1 ; +} + +static int func_b (int x) +{ +return x+2 ; +} + +DEFINE_STATIC_CALL (sc_selftest, func_a); + +static int __init test_static_call_init (void ) +{ +WARN_ON (static_call (sc_selftest)(2 ) !=3 ); +static_call_update (sc_selftest, &func_b); +WARN_ON (static_call (sc_selftest)(2 ) !=4 ); +static_call_update (sc_selftest, &func_a); +WARN_ON (static_call (sc_selftest)(2 ) !=3 ); + +return 0 ; +} +early_initcall (test_static_call_init); + +#endif /* CONFIG_STATIC_CALL_SELFTEST */
Peter Zijlstra
maintainerPATCH v2 8/13 tracepoints: Use static_call
In reply to Peter ZijlstraFrom: Steven Rostedt <[email protected]>
--- include/linux/tracepoint-defs.h | 5 ++ include/linux/tracepoint.h | 75 ++++++++++++++++++++++++++-------------- include/trace/define_trace.h | 14 +++---- kernel/tracepoint.c | 19 ++++++++-- 4 files changed, 77 insertions(+), 36 deletions(-)
include/linux/tracepoint-defs.h +5 -0
@@ -11,6 +11,8 @@#include <linux/atomic.h> #include <linux/static_key.h> +struct static_call_key ; +struct trace_print_flags {unsigned long mask;const char *name;
@@ -30,6 +32,9 @@ struct tracepoint_func {struct tracepoint {const char *name;/* Tracepoint name */ struct static_key key; +struct static_call_key *static_call_key; +void *static_call_tramp; +void *iterator;int (*regfunc)(void );void (*unregfunc)(void );struct tracepoint_func __rcu *funcs;
include/linux/tracepoint.h +50 -25
136 lines not on this page · raw message
include/trace/define_trace.h +7 -7
@@ -25,7 +25,7 @@#undef TRACE_EVENT#define TRACE_EVENT (name, proto, args, tstruct, assign, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_CONDITION#define TRACE_EVENT_CONDITION (name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@#undef TRACE_EVENT_FN#define TRACE_EVENT_FN (name, proto, args, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_FN_COND#define TRACE_EVENT_FN_COND (name, proto, args, cond, tstruct, \ assign, print, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef TRACE_EVENT_NOP#define TRACE_EVENT_NOP (name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@#undef DEFINE_EVENT#define DEFINE_EVENT (template, name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_FN#define DEFINE_EVENT_FN (template, name, proto, args, reg, unreg) \ - DEFINE_TRACE_FN(name, reg, unreg) + DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_PRINT#define DEFINE_EVENT_PRINT (template, name, proto, args, print) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef DEFINE_EVENT_CONDITION#define DEFINE_EVENT_CONDITION (template, name, proto, args, cond) \
@@ -70,7 +70,7 @@#undef DECLARE_TRACE#define DECLARE_TRACE (name, proto, args) \ - DEFINE_TRACE(name) + DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))#undef TRACE_INCLUDE#undef __TRACE_INCLUDE
kernel/tracepoint.c +15 -4
@@ -127,7 +127,7 @@ static void debug_print_probes(struct trstatic struct tracepoint_func *func_add (struct tracepoint_func **funcs,struct tracepoint_func *tp_func, -int prio) +int prio,int *tot_probes) {struct tracepoint_func *old, *new;int nr_probes =0 ;
@@ -170,11 +170,12 @@ func_add(struct tracepoint_func **funcs, new[nr_probes +1 ].func =NULL ; *funcs = new;debug_print_probes (*funcs); + *tot_probes = nr_probes +1 ;return old; }static void *func_remove (struct tracepoint_func **funcs, -struct tracepoint_func *tp_func) +struct tracepoint_func *tp_func,int *left) {int nr_probes =0 , nr_del =0 , i;struct tracepoint_func *old, *new;
@@ -228,6 +229,7 @@ static int tracepoint_add_func(struct tr struct tracepoint_func *func, int prio) {struct tracepoint_func *old, *tp_funcs; +int probes =0 ;int ret;if (tp->regfunc && !static_key_enabled (&tp->key)) {
@@ -238,7 +240,7 @@ static int tracepoint_add_func(struct tr tp_funcs =rcu_dereference_protected (tp->funcs,lockdep_is_held (&tracepoints_mutex)); - old =func_add (&tp_funcs, func, prio); + old =func_add (&tp_funcs, func, prio, &probes);if (IS_ERR (old)) {WARN_ON_ONCE (PTR_ERR (old) != -ENOMEM );return PTR_ERR (old);
@@ -253,6 +255,10 @@ static int tracepoint_add_func(struct trrcu_assign_pointer (tp->funcs, tp_funcs);if (!static_key_enabled (&tp->key))static_key_slow_inc (&tp->key); + +__static_call_update (tp->static_call_key, tp->static_call_tramp, + probes ==1 ? tp_funcs->func : tp->iterator); +release_probes (old);return 0 ; }
@@ -267,10 +273,11 @@ static int tracepoint_remove_func(struct struct tracepoint_func *func) {struct tracepoint_func *old, *tp_funcs; +int probes_left =0 ; tp_funcs =rcu_dereference_protected (tp->funcs,lockdep_is_held (&tracepoints_mutex)); - old =func_remove (&tp_funcs, func); + old =func_remove (&tp_funcs, func, &probes_left);if (IS_ERR (old)) {WARN_ON_ONCE (PTR_ERR (old) != -ENOMEM );return PTR_ERR (old);
@@ -284,6 +291,10 @@ static int tracepoint_remove_func(structif (static_key_enabled (&tp->key))static_key_slow_dec (&tp->key); } + +__static_call_update (tp->static_call_key, tp->static_call_tramp, + probes_left ==1 ? tp_funcs->func : tp->iterator); +rcu_assign_pointer (tp->funcs, tp_funcs);release_probes (old);return 0 ;
Peter Zijlstra
maintainerPATCH v2 9/13 x86/alternatives: Teach text_poke_bp() to emulate RET
In reply to Peter ZijlstraFuture patches will need to poke a RET instruction, provide the infrastructure required for this.
--- arch/x86/include/asm/text-patching.h | 16 ++++++++++++++++ arch/x86/kernel/alternative.c | 6 ++++++ 2 files changed, 22 insertions(+)
arch/x86/include/asm/text-patching.h +16 -0
@@ -64,6 +64,9 @@ static inline void int3_emulate_jmp(stru#define INT3_INSN_SIZE 1#define INT3_INSN_OPCODE 0xCC +#define RET_INSN_SIZE 1 +#define RET_INSN_OPCODE 0xC3 +#define CALL_INSN_SIZE 5#define CALL_INSN_OPCODE 0xE8
@@ -88,11 +91,24 @@ static inline void int3_emulate_push(str *(unsigned long *)regs->sp = val; } +static inline unsigned long int3_emulate_pop (struct pt_regs *regs) +{ +unsigned long val = *(unsigned long *)regs->sp; + regs->sp +=sizeof (unsigned long ); +return val; +} +static inline void int3_emulate_call (struct pt_regs *regs,unsigned long func) {int3_emulate_push (regs, regs->ip -INT3_INSN_SIZE +CALL_INSN_SIZE );int3_emulate_jmp (regs, func); } + +static inline void int3_emulate_ret (struct pt_regs *regs) +{ +unsigned long ip =int3_emulate_pop (regs); +int3_emulate_jmp (regs, ip); +}#endif /* !CONFIG_UML_X86 */ #endif /* _ASM_X86_TEXT_PATCHING_H */
arch/x86/kernel/alternative.c +6 -0
@@ -1014,6 +1014,10 @@ int notrace poke_int3_handler(struct pt_*/ return 0 ; +case RET_INSN_OPCODE : +int3_emulate_ret (regs); +break ; +case CALL_INSN_OPCODE :int3_emulate_call (regs, (long )ip + tp->rel32);break ;
@@ -1146,6 +1150,7 @@ void text_poke_loc_init(struct text_pokeswitch (tp->opcode) {case INT3_INSN_OPCODE : +case RET_INSN_OPCODE :break ;case CALL_INSN_OPCODE :
@@ -1260,6 +1265,7 @@ void *text_gen_insn(u8 opcode, unsignedswitch (opcode) {__CASE (INT3 ); +__CASE (RET );__CASE (CALL );__CASE (JMP32 );__CASE (JMP8 );
Peter Zijlstra
maintainerPATCH v2 10/13 static_call: Add static_cond_call()
In reply to Peter ZijlstraExtend the static_call infrastructure to optimize the following common pattern:
if (func_ptr) func_ptr(args...)
For the trampoline (which is in effect a tail-call), we patch the JMP.d32 into a RET, which then directly consumes the trampoline call.
For the in-line sites we replace the CALL with a NOP5.
--- arch/x86/kernel/static_call.c | 42 ++++++++++++++++++++++++++++++++---------- include/linux/static_call.h | 7 +++++++ 2 files changed, 39 insertions(+), 10 deletions(-)
arch/x86/kernel/static_call.c +32 -10
@@ -4,19 +4,41 @@#include <linux/bug.h> #include <asm/text-patching.h> -static void __static_call_transform (void *insn,u8 opcode,void *func) +enum insn_type { + call =0 ,/* site call */ + nop =1 ,/* site cond-call */ + jmp =2 ,/* tramp / site tail-call */ + ret =3 ,/* tramp / site cond-tail-call */ +}; + +static void __static_call_transform (void *insn,enum insn_type type,void *func) { -const void *code =text_gen_insn (opcode, (long )insn, (long )func); +int size =CALL_INSN_SIZE ; +const void *code; -if (WARN_ONCE (*(u8 *)insn != opcode, -"unexpected static call insn opcode 0x%x at %pS\n" , - opcode, insn)) -return ; +switch (type) { +case call: + code =text_gen_insn (CALL_INSN_OPCODE , insn, func); +break ; + +case nop: + code = ideal_nops[NOP_ATOMIC5 ]; +break ; + +case jmp: + code =text_gen_insn (JMP32_INSN_OPCODE , insn, func); +break ; + +case ret: + code =text_gen_insn (RET_INSN_OPCODE , insn, func); + size =RET_INSN_SIZE ; +break ; + } -if (memcmp (insn, code,CALL_INSN_SIZE ) ==0 ) +if (memcmp (insn, code, size) ==0 )return ; -text_poke_bp (insn, code,CALL_INSN_SIZE ,NULL ); +text_poke_bp (insn, code, size,NULL ); }void arch_static_call_transform (void *site,void *tramp,void *func)
@@ -24,10 +46,10 @@ void arch_static_call_transform(void *simutex_lock (&text_mutex);if (tramp) -__static_call_transform (tramp,JMP32_INSN_OPCODE , func); +__static_call_transform (tramp, jmp + !func, func);if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) -__static_call_transform (site,CALL_INSN_OPCODE , func); +__static_call_transform (site, !func, func);mutex_unlock (&text_mutex); }
include/linux/static_call.h +7 -0
@@ -17,6 +17,7 @@* DECLARE_STATIC_CALL(name, func); * DEFINE_STATIC_CALL(name, func); * static_call(name)(args...); +* static_cond_call(name)(args...) * static_call_update(name, func); * * Usage example:
@@ -105,6 +106,7 @@ extern int static_call_text_reserved(voi ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)#define static_call (name) STATIC_CALL_TRAMP(name) +#define static_cond_call (name) STATIC_CALL_TRAMP(name)#define EXPORT_STATIC_CALL (name) \ EXPORT_SYMBOL(STATIC_CALL_NAME(name)); \
@@ -128,6 +130,7 @@ struct static_call_key { ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)#define static_call (name) STATIC_CALL_TRAMP(name) +#define static_cond_call (name) STATIC_CALL_TRAMP(name)static inline void __static_call_update (struct static_call_key *key,void *tramp,void *func)
@@ -161,6 +164,10 @@ struct static_call_key {#define static_call (name) \ ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func)) +#define static_cond_call (name) \ + if (STATIC_CALL_NAME(name).func) \ + ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_NAME(name).func)) +static inline void __static_call_update (struct static_call_key *key,void *tramp,void *func) {
Peter Zijlstra
maintainerPATCH v2 11/13 static_call: Handle tail-calls
In reply to Peter ZijlstraGCC can turn our static_call(name)(args...) into a tail call, in which case we get a JMP.d32 into the trampoline (which then does a further tail-call).
Teach objtool to recognise and mark these in .static_call_sites and adjust the code patching to deal with this.
--- arch/x86/kernel/static_call.c | 4 ++-- include/linux/static_call.h | 4 ++-- include/linux/static_call_types.h | 7 +++++++ kernel/static_call.c | 21 +++++++++++++-------- tools/include/linux/static_call_types.h | 7 +++++++ tools/objtool/check.c | 18 +++++++++++++----- 6 files changed, 44 insertions(+), 17 deletions(-)
arch/x86/kernel/static_call.c +2 -2
@@ -41,7 +41,7 @@ static void __static_call_transform(voidtext_poke_bp (insn, code, size,NULL ); } -void arch_static_call_transform (void *site,void *tramp,void *func) +void arch_static_call_transform (void *site,void *tramp,void *func,bool tail) {mutex_lock (&text_mutex);
@@ -49,7 +49,7 @@ void arch_static_call_transform(void *si__static_call_transform (tramp, jmp + !func, func);if (IS_ENABLED (CONFIG_HAVE_STATIC_CALL_INLINE ) && site) -__static_call_transform (site, !func, func); +__static_call_transform (site,2 *tail + !func, func);mutex_unlock (&text_mutex); }
include/linux/static_call.h +2 -2
@@ -64,7 +64,7 @@/* * Either @site or @tramp can be NULL. */ -extern void arch_static_call_transform (void *site,void *tramp,void *func); +extern void arch_static_call_transform (void *site,void *tramp,void *func,bool tail);#endif
@@ -137,7 +137,7 @@ void __static_call_update(struct static_ {cpus_read_lock ();WRITE_ONCE (key->func, func); -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);cpus_read_unlock (); }
include/linux/static_call_types.h +7 -0
@@ -13,6 +13,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
kernel/static_call.c +13 -8
@@ -15,8 +15,6 @@ extern struct static_call_site __start_sstatic bool static_call_initialized; -#define STATIC_CALL_INIT 1UL -/* mutex to protect key modules/sites */ static DEFINE_MUTEX (static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(strstatic inline struct static_call_key *static_call_key (const struct static_call_site *site) {return (struct static_call_key *) - (((long )site->key + (long )&site->key) & ~STATIC_CALL_INIT ); + (((long )site->key + (long )&site->key) & ~STATIC_CALL_SITE_FLAGS ); }/* These assume the key is word-aligned. */ static inline bool static_call_is_init (struct static_call_site *site) { -return ((long )site->key + (long )&site->key) &STATIC_CALL_INIT ; +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_INIT ; +} + +static inline bool static_call_is_tail (struct static_call_site *site) +{ +return ((long )site->key + (long )&site->key) &STATIC_CALL_SITE_TAIL ; }static inline void static_call_set_init (struct static_call_site *site) { - site->key = ((long )static_call_key (site) |STATIC_CALL_INIT ) - + site->key = ((long )static_call_key (site) |STATIC_CALL_SITE_INIT ) - (long )&site->key; }
@@ -104,7 +107,7 @@ void __static_call_update(struct static_ key->func = func; -arch_static_call_transform (NULL , tramp, func); +arch_static_call_transform (NULL , tramp, func, false);/* * If uninitialized, we'll not update the callsites, but they still
@@ -153,7 +156,8 @@ void __static_call_update(struct static_continue ; } -arch_static_call_transform (site_addr,NULL , func); +arch_static_call_transform (site_addr,NULL , func, +static_call_is_tail (site)); } }
@@ -197,7 +201,8 @@ static int __static_call_init(struct mod key->next = site_mod; } -arch_static_call_transform (site_addr,NULL , key->func); +arch_static_call_transform (site_addr,NULL , key->func, +static_call_is_tail (site)); }return 0 ;
tools/include/linux/static_call_types.h +7 -0
@@ -13,6 +13,13 @@#define STATIC_CALL_TRAMP_STR (name) __stringify(STATIC_CALL_TRAMP(name))/* +* Flags in the low bits of static_call_site::key. +*/ +#define STATIC_CALL_SITE_TAIL 1UL/* tail call */ +#define STATIC_CALL_SITE_INIT 2UL/* init section */ +#define STATIC_CALL_SITE_FLAGS 3UL + +/* * The static call site table needs to be created by external tooling (objtool * or a compiler plugin). */
tools/objtool/check.c +13 -5
@@ -585,6 +585,10 @@ static int add_jump_destinations(struct }else {/* external sibling call */ insn->call_dest = rela->sym; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + }continue ; }
@@ -636,6 +640,10 @@ static int add_jump_destinations(struct/* internal sibling call */ insn->call_dest = insn->jump_dest->func; +if (insn->call_dest->static_call_tramp) { +list_add_tail (&insn->static_call_node, + &file->static_call_list); + } } } }
@@ -1348,6 +1356,10 @@ static int decode_sections(struct objtooif (ret)return ret; + ret =read_static_call_tramps (file); +if (ret) +return ret; + ret =add_jump_destinations (file);if (ret)return ret;
@@ -1372,10 +1384,6 @@ static int decode_sections(struct objtooif (ret)return ret; - ret =read_static_call_tramps (file); -if (ret) -return ret; -return 0 ; }
@@ -2505,7 +2513,7 @@ static int create_static_call_sections(s }memset (rela,0 ,sizeof (*rela)); rela->sym = key_sym; - rela->addend =0 ; + rela->addend =is_sibling_call (insn) ?STATIC_CALL_SITE_TAIL :0 ; rela->type =R_X86_64_PC32 ; rela->offset = idx *sizeof (struct static_call_site ) +4 ;list_add_tail (&rela->list, &rela_sec->rela_list);
Peter Zijlstra
maintainerPATCH v2 12/13 static_call: Allow early init
In reply to Peter ZijlstraIn order to use static_call() to wire up x86_pmu, we need to initialize earlier; copy some of the tricks from jump_label to enable this.
Primarily we overload key->next to store a sites pointer when there are no modules, this avoids having to use kmalloc() to initialize the sites and allows us to run much earlier.
(arguably, this is much much earlier than needed for perf, but it might allow other uses.)
--- arch/x86/kernel/setup.c | 2 + arch/x86/kernel/static_call.c | 3 ++ include/linux/static_call.h | 15 ++++++++++-- kernel/static_call.c | 52 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 67 insertions(+), 5 deletions(-)
arch/x86/kernel/setup.c +2 -0
@@ -73,6 +73,7 @@#include <linux/jiffies.h> #include <linux/mem_encrypt.h> #include <linux/sizes.h> +#include <linux/static_call.h> #include <linux/usb/xhci-dbgp.h> #include <video/edid.h>
@@ -896,6 +897,7 @@ void __init setup_arch(char **cmdline_p)early_cpu_init ();arch_init_ideal_nops ();jump_label_init (); +static_call_init ();early_ioremap_init ();setup_olpc_ofw_pgd ();
arch/x86/kernel/static_call.c +3 -0
@@ -38,6 +38,9 @@ static void __static_call_transform(voidif (memcmp (insn, code, size) ==0 )return ; +if (unlikely (system_state ==SYSTEM_BOOTING )) +return text_poke_early (insn, code, size); +text_poke_bp (insn, code, size,NULL ); }
include/linux/static_call.h +13 -2
@@ -81,6 +81,8 @@ extern void arch_static_call_transform(v #ifdef CONFIG_HAVE_STATIC_CALL_INLINE +extern void __init static_call_init(void); + struct static_call_mod { struct static_call_mod *next; struct module *mod; /* for vmlinux, mod == NULL */
@@ -89,7 +91,12 @@ struct static_call_mod {struct static_call_key {void *func; -struct static_call_mod *next; +union { +/* bit0 => 0 - next, 1 - sites */ +unsigned long type; +struct static_call_mod *next; +struct static_call_site *sites; + }; };extern void __static_call_update (struct static_call_key *key,void *tramp,void *func);
@@ -100,7 +107,7 @@ extern int static_call_text_reserved(voiDECLARE_STATIC_CALL (name, _func); \struct static_call_key STATIC_CALL_NAME (name ) = { \ .func = _func, \ - .next =NULL , \ + .type =1 , \ }; \__ADDRESSABLE (STATIC_CALL_NAME (name)); \ARCH_DEFINE_STATIC_CALL_TRAMP (name, _func)
@@ -118,6 +125,8 @@ extern int static_call_text_reserved(voi#elif defined(CONFIG_HAVE_STATIC_CALL ) +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
@@ -151,6 +160,8 @@ static inline int static_call_text_reser#else /* Generic implementation */ +static inline void static_call_init (void ) { } +struct static_call_key {void *func; };
kernel/static_call.c +49 -3
89 lines not on this page · raw message
Peter Zijlstra
maintainerRFC PATCH v2 13/13 x86/perf, static_call: Optimize x86_pmu methods
In reply to Peter ZijlstraReplace many of the indirect calls with static_call().
XXX run performance numbers
--- arch/x86/events/core.c | 136 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 98 insertions(+), 38 deletions(-)
arch/x86/events/core.c +98 -38
@@ -28,6 +28,7 @@#include <linux/bitops.h> #include <linux/device.h> #include <linux/nospec.h> +#include <linux/static_call.h> #include <asm/apic.h> #include <asm/stacktrace.h>
@@ -51,6 +52,45 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpuDEFINE_STATIC_KEY_FALSE (rdpmc_always_available_key); +static void _x86_pmu_add (struct perf_event *event) { } +static void _x86_pmu_del (struct perf_event *event) { } +static void _x86_pmu_read (struct perf_event *event) {x86_perf_event_update (event); } +static void _x86_pmu_put_event_constraints (struct cpu_hw_events *cpuc,struct perf_event *event) { } +static void _x86_pmu_drain_pebs (struct pt_regs *regs) { } +static void _x86_pmu_pebs_aliases (struct perf_event *event) { } +static void _x86_pmu_start_scheduling (struct cpu_hw_events *cpuc) { } +static void _x86_pmu_commit_scheduling (struct cpu_hw_events *cpuc,int idx,int cntr) { } +static void _x86_pmu_stop_scheduling (struct cpu_hw_events *cpuc) { } +static void _x86_pmu_sched_task (struct perf_event_context *ctx,bool sched_in) { } + +DEFINE_STATIC_CALL (x86_pmu_handle_irq, x86_pmu_handle_irq); +DEFINE_STATIC_CALL (x86_pmu_disable_all, x86_pmu_disable_all); +DEFINE_STATIC_CALL (x86_pmu_enable_all, x86_pmu_enable_all); +DEFINE_STATIC_CALL (x86_pmu_enable, x86_pmu_enable_event); +DEFINE_STATIC_CALL (x86_pmu_disable, x86_pmu_disable_event); + +DEFINE_STATIC_CALL (x86_pmu_add, _x86_pmu_add); +DEFINE_STATIC_CALL (x86_pmu_del, _x86_pmu_del); +DEFINE_STATIC_CALL (x86_pmu_read, _x86_pmu_read); + +DEFINE_STATIC_CALL (x86_pmu_schedule_events, x86_schedule_events); + +// addr_offset +// rdpmc_index +// event_map + +DEFINE_STATIC_CALL (x86_pmu_get_event_constraints, x86_get_event_constraints); +DEFINE_STATIC_CALL (x86_pmu_put_event_constraints, _x86_pmu_put_event_constraints); + +DEFINE_STATIC_CALL (x86_pmu_drain_pebs, _x86_pmu_drain_pebs); +DEFINE_STATIC_CALL (x86_pmu_pebs_aliases, _x86_pmu_pebs_aliases); + +DEFINE_STATIC_CALL (x86_pmu_start_scheduling, _x86_pmu_start_scheduling); +DEFINE_STATIC_CALL (x86_pmu_commit_scheduling, _x86_pmu_commit_scheduling); +DEFINE_STATIC_CALL (x86_pmu_stop_scheduling, _x86_pmu_stop_scheduling); + +DEFINE_STATIC_CALL (x86_pmu_sched_task, _x86_pmu_sched_task); +u64 __read_mostly hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ]
@@ -651,7 +691,7 @@ static void x86_pmu_disable(struct pmu * cpuc->enabled =0 ;barrier (); - x86_pmu.disable_all (); +static_call (x86_pmu_disable_all)(); }void x86_pmu_enable_all (int added)
@@ -884,8 +924,7 @@ int x86_schedule_events(struct cpu_hw_evif (cpuc->txn_flags &PERF_PMU_TXN_ADD ) n0 -= cpuc->n_txn; -if (x86_pmu.start_scheduling) - x86_pmu.start_scheduling (cpuc); +static_cond_call (x86_pmu_start_scheduling)(cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { c = cpuc->event_constraint[i];
@@ -902,7 +941,7 @@ int x86_schedule_events(struct cpu_hw_ev* change due to external factors (sibling state, allow_tfa). */ if (!c || (c->flags &PERF_X86_EVENT_DYNAMIC )) { - c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); + c =static_call (x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); cpuc->event_constraint[i] = c; }
@@ -969,8 +1008,7 @@ int x86_schedule_events(struct cpu_hw_evif (!unsched && assign) {for (i =0 ; i < n; i++) { e = cpuc->event_list[i]; -if (x86_pmu.commit_scheduling) - x86_pmu.commit_scheduling (cpuc, i, assign[i]); +static_cond_call (x86_pmu_commit_scheduling)(cpuc, i, assign[i]); } }else {for (i = n0; i < n; i++) {
@@ -979,15 +1017,13 @@ int x86_schedule_events(struct cpu_hw_ev/* * release events that failed scheduling */ -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, e); +static_cond_call (x86_pmu_put_event_constraints)(cpuc, e); cpuc->event_constraint[i] =NULL ; } } -if (x86_pmu.stop_scheduling) - x86_pmu.stop_scheduling (cpuc); +static_cond_call (x86_pmu_stop_scheduling)(cpuc);return unsched ? -EINVAL :0 ; }
@@ -1174,7 +1210,7 @@ static void x86_pmu_enable(struct pmu *p cpuc->enabled =1 ;barrier (); - x86_pmu.enable_all (added); +static_call (x86_pmu_enable_all)(added); } static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
@@ -1288,7 +1324,7 @@ static int x86_pmu_add(struct perf_eventif (cpuc->txn_flags &PERF_PMU_TXN_ADD ) goto done_collect; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret) goto out;/*
@@ -1306,13 +1342,11 @@ static int x86_pmu_add(struct perf_event cpuc->n_added += n - n0; cpuc->n_txn += n - n0; -if (x86_pmu.add) { -/* -* This is before x86_pmu_enable() will call x86_pmu_start(), -* so we enable LBRs before an event needs them etc.. -*/ - x86_pmu.add (event); - } +/* +* This is before x86_pmu_enable() will call x86_pmu_start(), +* so we enable LBRs before an event needs them etc.. +*/ +static_cond_call (x86_pmu_add)(event); ret =0 ; out:
@@ -1340,7 +1374,7 @@ static void x86_pmu_start(struct perf_ev cpuc->events[idx] = event;__set_bit (idx, cpuc->active_mask);__set_bit (idx, cpuc->running); - x86_pmu.enable (event); +static_call (x86_pmu_enable)(event);perf_event_update_userpage (event); }
@@ -1410,7 +1444,7 @@ void x86_pmu_stop(struct perf_event *evestruct hw_perf_event *hwc = &event->hw;if (test_bit (hwc->idx, cpuc->active_mask)) { - x86_pmu.disable (event); +static_call (x86_pmu_disable)(event);__clear_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] =NULL ;WARN_ON_ONCE (hwc->state &PERF_HES_STOPPED );
@@ -1460,8 +1494,7 @@ static void x86_pmu_del(struct perf_evenif (i >= cpuc->n_events - cpuc->n_added) --cpuc->n_added; -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, event); +static_cond_call (x86_pmu_put_event_constraints)(cpuc, event);/* Delete the array entry. */ while (++i < cpuc->n_events) {
@@ -1474,13 +1507,12 @@ static void x86_pmu_del(struct perf_evenperf_event_update_userpage (event); do_del: -if (x86_pmu.del) { -/* -* This is after x86_pmu_stop(); so we disable LBRs after any -* event can need them etc.. -*/ - x86_pmu.del (event); - } + +/* +* This is after x86_pmu_stop(); so we disable LBRs after any +* event can need them etc.. +*/ +static_cond_call (x86_pmu_del)(event); }int x86_pmu_handle_irq (struct pt_regs *regs)
@@ -1558,7 +1590,7 @@ perf_event_nmi_handler(unsigned int cmd,return NMI_DONE ; start_clock =sched_clock (); - ret = x86_pmu.handle_irq (regs); + ret =static_call (x86_pmu_handle_irq)(regs); finish_clock =sched_clock ();perf_sample_event_took (finish_clock - start_clock);
@@ -1765,6 +1797,32 @@ ssize_t x86_event_sysfs_show(char *page,static struct attribute_group x86_pmu_attr_group;static struct attribute_group x86_pmu_caps_group; +static void x86_pmu_static_call_update (void ) +{ +static_call_update (x86_pmu_handle_irq, x86_pmu.handle_irq); +static_call_update (x86_pmu_disable_all, x86_pmu.disable_all); +static_call_update (x86_pmu_enable_all, x86_pmu.enable_all); +static_call_update (x86_pmu_enable, x86_pmu.enable); +static_call_update (x86_pmu_disable, x86_pmu.disable); + +static_call_update (x86_pmu_add, x86_pmu.add); +static_call_update (x86_pmu_del, x86_pmu.del); +static_call_update (x86_pmu_read, x86_pmu.read); + +static_call_update (x86_pmu_schedule_events, x86_pmu.schedule_events); +static_call_update (x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); +static_call_update (x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); + +static_call_update (x86_pmu_drain_pebs, x86_pmu.drain_pebs); +static_call_update (x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); + +static_call_update (x86_pmu_start_scheduling, x86_pmu.start_scheduling); +static_call_update (x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); +static_call_update (x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); + +static_call_update (x86_pmu_sched_task, x86_pmu.sched_task); +} +static int __init init_hw_perf_events (void ) {struct x86_pmu_quirk *quirk;
@@ -1829,6 +1887,11 @@ static int __init init_hw_perf_events(vopr_info ("... fixed-purpose events: %d\n" , x86_pmu.num_counters_fixed);pr_info ("... event mask: %016Lx\n" , x86_pmu.intel_ctrl); +if (!x86_pmu.read) + x86_pmu.read = _x86_pmu_read; + +x86_pmu_static_call_update (); +/* * Install callbacks. Core will call them for each online * cpu.
@@ -1865,11 +1928,9 @@ static int __init init_hw_perf_events(vo }early_initcall (init_hw_perf_events); -static inline void x86_pmu_read (struct perf_event *event) +static void x86_pmu_read (struct perf_event *event) { -if (x86_pmu.read) -return x86_pmu.read (event); -x86_perf_event_update (event); +static_call (x86_pmu_read)(event); }/*
@@ -1946,7 +2007,7 @@ static int x86_pmu_commit_txn(struct pmuif (!x86_pmu_initialized ())return -EAGAIN ; - ret = x86_pmu.schedule_events (cpuc, n, assign); + ret =static_call (x86_pmu_schedule_events)(cpuc, n, assign);if (ret)return ret;
@@ -2239,8 +2300,7 @@ static const struct attribute_group *x86static void x86_pmu_sched_task (struct perf_event_context *ctx,bool sched_in) { -if (x86_pmu.sched_task) - x86_pmu.sched_task (ctx, sched_in); +static_cond_call (x86_pmu_sched_task)(ctx, sched_in); }void perf_check_microcode (void )
Peter Zijlstra
maintainerRe: [PATCH v2 00/13] Add static_call()
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
This series, which depends on the previous two, introduces
static_call().
static_call(), is the idea ofstatic_branch()applied to indirect function calls. Remove a data load (indirection) by modifying the text.
Quoted 9 lines
These patches are still based on the work Josh did earlier, but incorporated feedback from the last posting and have a lot of extra patches which resulted from me trying to actually use
static_call().This still relies on objtool to generate the .static_call_sites section, mostly because this is a natural place for
x86_64and works for both GCC and LLVM. Other architectures can pick other means if/when they implement the inline patching. The out-of-line (aka. trampoline) variant doesn't require this.
FWIW, Steve, if we were to do something like:
CFLAGS += -mfentry -mfentry_name=____static_call_fentry_tramp
And have:
struct static_call_key ____static_call_fentry = {
.func = NULL,
.next = 1,
};
asm(".pushsection .static_call.text, \"ax\" \n"
".align 4 \n"
".globl ____static_call_fentry_tramp \n"
"____static_call_fentry_tramp: \n"
" ret \n"
".type ____static_call_fentry_tramp, @function \n"
".size ____static_call_fentry_tramp, . - ____static_call_fentry_tramp \n"
".popsection \n");Then the whole function entry thing would automagicaly turn into something like static_cond_call(fentry)(...);
Not saying we should do that, but we could ;-)
Peter Zijlstra
maintainerv2 cover Propagate module notifier errors
In reply to Peter ZijlstraThese patches came from the desire to propagate MODULE_STATE_COMING errors. While looking at that I spotted fail with a number of module notifiers themselves and with the whole notification business.
I will probably add a notifier for MODULE_STATE_UNFORMED, but that is later.
Fixing this error propagation fixes a fairly serious (but really uncommon) bug when memory allocation fails for jump-labels while loading a module. And since we're going to be introducing more of that (static_call), fix it now.
Peter Zijlstra
maintainerPATCH v2 1/4 notifier: Fix broken error handling pattern
The current notifiers have the following error handling pattern all over the place:
int err, nr; err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr); if (err & NOTIFIER_STOP_MASK) __foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL)
And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless.
Fix this by replacing all the __foo_notifier_call_chain() functions with foo_notifier_call_chain_robust() that embeds the above pattern, but ensures it is inside a single lock region.
Note: I switched atomic_notifier_call_chain_robust() to use
the spinlock, since RCU cannot provide the guarantee required for the recovery.
Note: software_resume() error handling was broken afaict.
--- include/linux/notifier.h | 15 +-- kernel/cpu_pm.c | 46 ++++------- kernel/notifier.c | 144 ++++++++++++++++++++++--------------- kernel/power/hibernate.c | 26 +++--- kernel/power/main.c | 8 +- kernel/power/power.h | 3 kernel/power/suspend.c | 14 +-- kernel/power/user.c | 14 +-- tools/power/pm-graph/sleepgraph.py | 2 9 files changed, 139 insertions(+), 133 deletions(-)
include/linux/notifier.h +7 -8
@@ -165,20 +165,19 @@ extern int srcu_notifier_chain_unregisteextern int atomic_notifier_call_chain (struct atomic_notifier_head *nh,unsigned long val,void *v); -extern int __atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int blocking_notifier_call_chain (struct blocking_notifier_head *nh,unsigned long val,void *v); -extern int __blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int raw_notifier_call_chain (struct raw_notifier_head *nh,unsigned long val,void *v); -extern int __raw_notifier_call_chain (struct raw_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int srcu_notifier_call_chain (struct srcu_notifier_head *nh,unsigned long val,void *v); -extern int __srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls); + +extern int atomic_notifier_call_chain_robust (struct atomic_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int blocking_notifier_call_chain_robust (struct blocking_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int raw_notifier_call_chain_robust (struct raw_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v);#define NOTIFY_DONE 0x0000/* Don't care */ #define NOTIFY_OK 0x0001/* Suits me */
kernel/cpu_pm.c +16 -30
89 lines not on this page · raw message
kernel/notifier.c +88 -56
240 lines not on this page · raw message
kernel/power/hibernate.c +12 -14
@@ -693,8 +693,8 @@ static int load_image_and_restore(void)*/ int hibernate (void ) { -int error, nr_calls =0 ;bool snapshot_test = false; +int error;if (!hibernation_available ()) {pm_pr_dbg ("Hibernation not available.\n" );
@@ -710,11 +710,9 @@ int hibernate(void)pr_info ("hibernation entry\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto Restore;ksys_sync_helper ();
@@ -772,7 +770,8 @@ int hibernate(void)/* Don't bother checking whether freezer_test_done is true */ freezer_test_done = false; Exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); + Restore:pm_restore_console ();atomic_inc (&snapshot_device_available); Unlock:
@@ -800,7 +799,7 @@ int hibernate(void)*/ static int software_resume (void ) { -int error, nr_calls =0 ; +int error;/* * If the user said "noresume".. bail out early.
@@ -887,11 +886,9 @@ static int software_resume(void)pr_info ("resume from hibernation\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Close_Finish; - } + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE ); +if (error) + goto Restore;pm_pr_dbg ("Preparing processes for restore.\n" ); error =freeze_processes ();
@@ -900,7 +897,8 @@ static int software_resume(void) error =load_image_and_restore ();thaw_processes (); Finish: -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_RESTORE ); + Restore:pm_restore_console ();pr_info ("resume from hibernation failed (%d)\n" , error);atomic_inc (&snapshot_device_available);
kernel/power/main.c +4 -4
@@ -79,18 +79,18 @@ int unregister_pm_notifier(struct notifi }EXPORT_SYMBOL_GPL (unregister_pm_notifier); -int __pm_notifier_call_chain (unsigned long val,int nr_to_call,int *nr_calls) +int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down) {int ret; - ret =__blocking_notifier_call_chain (&pm_chain_head, val,NULL , - nr_to_call, nr_calls); + ret =blocking_notifier_call_chain_robust (&pm_chain_head, val_up, val_down,NULL );return notifier_to_errno (ret); } +int pm_notifier_call_chain (unsigned long val) { -return __pm_notifier_call_chain (val,-1 ,NULL ); +return blocking_notifier_call_chain (&pm_chain_head, val,NULL ); }/* If set, devices may be suspended and resumed asynchronously. */
kernel/power/power.h +1 -2
@@ -210,8 +210,7 @@ static inline void suspend_test_finish(c#ifdef CONFIG_PM_SLEEP /* kernel/power/main.c */ -extern int __pm_notifier_call_chain (unsigned long val,int nr_to_call, -int *nr_calls); +extern int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down);extern int pm_notifier_call_chain (unsigned long val);#endif
kernel/power/suspend.c +6 -8
@@ -352,18 +352,16 @@ static int suspend_test(int level)*/ static int suspend_prepare (suspend_state_t state) { -int error, nr_calls =0 ; +int error;if (!sleep_state_supported (state))return -EPERM ;pm_prepare_console (); - error =__pm_notifier_call_chain (PM_SUSPEND_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Finish; - } + error =pm_notifier_call_chain_robust (PM_SUSPEND_PREPARE ,PM_POST_SUSPEND ); +if (error) + goto Restore;trace_suspend_resume (TPS ("freeze_processes" ),0 , true); error =suspend_freeze_processes ();
@@ -373,8 +371,8 @@ static int suspend_prepare(suspend_state suspend_stats.failed_freeze++;dpm_save_failed_step (SUSPEND_FREEZE ); - Finish: -__pm_notifier_call_chain (PM_POST_SUSPEND , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_SUSPEND ); + Restore:pm_restore_console ();return error; }
kernel/power/user.c +4 -10
@@ -44,7 +44,7 @@ atomic_t snapshot_device_available = ATOstatic int snapshot_open (struct inode *inode,struct file *filp) {struct snapshot_data *data; -int error, nr_calls =0 ; +int error;if (!hibernation_available ())return -EPERM ;
@@ -71,9 +71,7 @@ static int snapshot_open(struct inode *i swap_type_of(swsusp_resume_device, 0, NULL) : -1; data->mode =O_RDONLY ; data->free_bitmaps = false; - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) -__pm_notifier_call_chain (PM_POST_HIBERNATION , --nr_calls,NULL ); + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); }else {/* * Resuming. We may need to wait for the image device to
@@ -83,15 +81,11 @@ static int snapshot_open(struct inode *i data->swap =-1 ; data->mode =O_WRONLY ; - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE );if (!error) { error =create_basic_memory_bitmaps (); data->free_bitmaps = !error; - }else - nr_calls--; - -if (error) -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); + } }if (error)atomic_inc (&snapshot_device_available);
tools/power/pm-graph/sleepgraph.py +1 -1
@@ -153,7 +153,7 @@ import base64 tracefuncs = { 'sys_sync': {}, 'ksys_sync': {}, - '__pm_notifier_call_chain': {}, + 'pm_notifier_call_chain_robust': {}, 'pm_prepare_console': {}, 'pm_notifier_call_chain': {}, 'freeze_processes': {},
Re: [PATCH v2 1/4] notifier: Fix broken error handling pattern
Quoted 24 lines · CEST Peter Zijlstra
The current notifiers have the following error handling pattern all over the place:
int err, nr; err = __foo_notifier_call_chain(&chain, val_up, v, -1, &nr); if (err & NOTIFIER_STOP_MASK) __foo_notifier_call_chain(&chain, val_down, v, nr-1, NULL)And aside from the endless repetition thereof, it is broken. Consider blocking notifiers; both calls take and drop the rwsem, this means that the notifier list can change in between the two calls, making @nr meaningless.
Fix this by replacing all the
__foo_notifier_call_chain()functions withfoo_notifier_call_chain_robust()that embeds the above pattern, but ensures it is inside a single lock region.Note: I switched
atomic_notifier_call_chain_robust()to use
the spinlock, since RCU cannot provide the guarantee required for the recovery.Note:
software_resume()error handling was broken afaict.
Quoted 12 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Pavel Machek <[email protected]>Cc Alexios Zavras <[email protected]>Cc Allison Randal <[email protected]>Cc Sam Protsenko <[email protected]>Cc "Rafael J. Wysocki" <[email protected]>Cc Andrew Morton <[email protected]>Cc Todd Brandt <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Vasily Averin <[email protected]>Cc Len Brown <[email protected]>Cc Greg Kroah-Hartman <[email protected]>
for the bits I care about.
Quoted 588 lines
--- include/linux/notifier.h | 15 +-- kernel/cpu_pm.c | 46 ++++------- kernel/notifier.c | 144 ++++++++++++++++++++++--------------- kernel/power/hibernate.c | 26 +++--- kernel/power/main.c | 8 +- kernel/power/power.h | 3 kernel/power/suspend.c | 14 +-- kernel/power/user.c | 14 +-- tools/power/pm-graph/sleepgraph.py | 2 9 files changed, 139 insertions(+), 133 deletions(-)include/linux/notifier.h +7
-8@@ -165,20 +165,19 @@ extern int srcu_notifier_chain_unregisteextern int atomic_notifier_call_chain (struct atomic_notifier_head *nh,unsigned long val,void *v); -extern int __atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int blocking_notifier_call_chain (struct blocking_notifier_head *nh,unsigned long val,void *v); -extern int __blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int raw_notifier_call_chain (struct raw_notifier_head *nh,unsigned long val,void *v); -extern int __raw_notifier_call_chain (struct raw_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls);extern int srcu_notifier_call_chain (struct srcu_notifier_head *nh,unsigned long val,void *v); -extern int __srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v,int nr_to_call,int *nr_calls); + +extern int atomic_notifier_call_chain_robust (struct atomic_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int blocking_notifier_call_chain_robust (struct blocking_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v); +extern int raw_notifier_call_chain_robust (struct raw_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v);#define NOTIFY_DONE 0x0000/* Don't care */ #define NOTIFY_OK 0x0001/* Suits me */ kernel/cpu_pm.c +16
-30@@ -15,23 +15,31 @@static ATOMIC_NOTIFIER_HEAD (cpu_pm_notifier_chain); -static int cpu_pm_notify (enum cpu_pm_event event,int nr_to_call,int *nr_calls) +static int cpu_pm_notify (enum cpu_pm_event event) {int ret;/* -* __atomic_notifier_call_chain has a RCU read critical section, which +* atomic_notifier_call_chain has a RCU read critical section, which * could be disfunctional in cpu idle. Copy RCU_NONIDLE code to let * RCU know this. */ rcu_irq_enter_irqson (); - ret =__atomic_notifier_call_chain (&cpu_pm_notifier_chain, event,NULL , - nr_to_call, nr_calls); + ret =atomic_notifier_call_chain (&cpu_pm_notifier_chain, event,NULL );rcu_irq_exit_irqson ();return notifier_to_errno (ret); } +static int cpu_pm_notify_robust (enum cpu_pm_event event_up,enum cpu_pm_event event_down) +{ +int ret; + + ret =atomic_notifier_call_chain_robust (&cpu_pm_notifier_chain, event_up, event_down,NULL ); + +return notifier_to_errno (ret); +} +/** * cpu_pm_register_notifier - register a driver with cpu_pm * @nb: notifier block to register @@ -80,18 +88,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_unregister_noti*/ int cpu_pm_enter (void ) { -int nr_calls; -int ret =0 ; - - ret =cpu_pm_notify (CPU_PM_ENTER ,-1 , &nr_calls); -if (ret) -/* -* Inform listeners (nr_calls - 1) about failure of CPU PM -* PM entry who are notified earlier to prepare for it. -*/ -cpu_pm_notify (CPU_PM_ENTER_FAILED , nr_calls -1 ,NULL ); - -return ret; +return cpu_pm_notify_robust (CPU_PM_ENTER ,CPU_PM_ENTER_FAILED ); }EXPORT_SYMBOL_GPL (cpu_pm_enter);@@ -109,7 +106,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_enter);*/ int cpu_pm_exit (void ) { -return cpu_pm_notify (CPU_PM_EXIT ,-1 ,NULL ); +return cpu_pm_notify (CPU_PM_EXIT ); }EXPORT_SYMBOL_GPL (cpu_pm_exit);@@ -131,18 +128,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_exit);*/ int cpu_cluster_pm_enter (void ) { -int nr_calls; -int ret =0 ; - - ret =cpu_pm_notify (CPU_CLUSTER_PM_ENTER ,-1 , &nr_calls); -if (ret) -/* -* Inform listeners (nr_calls - 1) about failure of CPU cluster -* PM entry who are notified earlier to prepare for it. -*/ -cpu_pm_notify (CPU_CLUSTER_PM_ENTER_FAILED , nr_calls -1 ,NULL ); - -return ret; +return cpu_pm_notify_robust (CPU_CLUSTER_PM_ENTER ,CPU_CLUSTER_PM_ENTER_FAILED ); }EXPORT_SYMBOL_GPL (cpu_cluster_pm_enter);@@ -163,7 +149,7 @@ EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter);*/ int cpu_cluster_pm_exit (void ) { -return cpu_pm_notify (CPU_CLUSTER_PM_EXIT ,-1 ,NULL ); +return cpu_pm_notify (CPU_CLUSTER_PM_EXIT ); }EXPORT_SYMBOL_GPL (cpu_cluster_pm_exit);kernel/notifier.c +88
-56@@ -106,6 +106,34 @@ static int notifier_call_chain(struct no }NOKPROBE_SYMBOL (notifier_call_chain); +/** +* notifier_call_chain_robust - Inform the registered notifiers about an event +* and rollback on error. +* @nl: Pointer to head of the blocking notifier chain +* @val_up: Value passed unmodified to the notifier function +* @val_down: Value passed unmodified to the notifier function when recovering +* from an error on @val_up +* @v Pointer passed unmodified to the notifier function +* +* NOTE: It is important the @nl chain doesn't change between the two +* invocations of notifier_call_chain() such that we visit the +* exact same notifier callbacks; this rules out any RCU usage. +* +* Returns: the return value of the @val_up call. +*/ +static int notifier_call_chain_robust (struct notifier_block **nl, +unsigned long val_up,unsigned long val_down, +void *v) +{ +int ret, nr =0 ; + + ret =notifier_call_chain (nl, val_up, v,-1 , &nr); +if (ret &NOTIFY_STOP_MASK ) +notifier_call_chain (nl, val_down, v, nr-1 ,NULL ); + +return ret; +} +/* * Atomic notifier chain routines. Registration and unregistration * use a spinlock, and call_chain is synchronized by RCU (no locks). @@ -156,13 +184,30 @@ int atomic_notifier_chain_unregister(str }EXPORT_SYMBOL_GPL (atomic_notifier_chain_unregister); +int atomic_notifier_call_chain_robust (struct atomic_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v) +{ +unsigned long flags; +int ret; + +/* +* Musn't use RCU; because then the notifier list can +* change between the up and down traversal. +*/ +spin_lock_irqsave (&nh->lock, flags); + ret =notifier_call_chain_robust (&nh->head, val_up, val_down, v); +spin_unlock_irqrestore (&nh->lock, flags); + +return ret; +} +EXPORT_SYMBOL_GPL (atomic_notifier_call_chain_robust); +NOKPROBE_SYMBOL (atomic_notifier_call_chain_robust); +/** -* __atomic_notifier_call_chain - Call functions in an atomic notifier chain +* atomic_notifier_call_chain - Call functions in an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function -* @nr_to_call: See the comment for notifier_call_chain. -* @nr_calls: See the comment for notifier_call_chain. * * Calls each function in a notifier chain in turn. The functions * run in an atomic context, so they must not block. @@ -175,24 +220,16 @@ EXPORT_SYMBOL_GPL(atomic_notifier_chain_* Otherwise the return value is the return value * of the last notifier function called. */ -int __atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v, -int nr_to_call,int *nr_calls) +int atomic_notifier_call_chain (struct atomic_notifier_head *nh, +unsigned long val,void *v) {int ret;rcu_read_lock (); - ret =notifier_call_chain (&nh->head, val, v, nr_to_call, nr_calls); + ret =notifier_call_chain (&nh->head, val, v,-1 ,NULL );rcu_read_unlock (); -return ret; -} -EXPORT_SYMBOL_GPL (__atomic_notifier_call_chain); -NOKPROBE_SYMBOL (__atomic_notifier_call_chain); -int atomic_notifier_call_chain (struct atomic_notifier_head *nh, -unsigned long val,void *v) -{ -return __atomic_notifier_call_chain (nh, val, v,-1 ,NULL ); +return ret; }EXPORT_SYMBOL_GPL (atomic_notifier_call_chain);NOKPROBE_SYMBOL (atomic_notifier_call_chain);@@ -285,13 +322,30 @@ int blocking_notifier_chain_unregister(s }EXPORT_SYMBOL_GPL (blocking_notifier_chain_unregister); +int blocking_notifier_call_chain_robust (struct blocking_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v) +{ +int ret =NOTIFY_DONE ; + +/* +* We check the head outside the lock, but if this access is +* racy then it does not matter what the result of the test +* is, we re-check the list after having taken the lock anyway: +*/ +if (rcu_access_pointer (nh->head)) { +down_read (&nh->rwsem); + ret =notifier_call_chain_robust (&nh->head, val_up, val_down, v); +up_read (&nh->rwsem); + } +return ret; +} +EXPORT_SYMBOL_GPL (blocking_notifier_call_chain_robust); +/** -* __blocking_notifier_call_chain - Call functions in a blocking notifier chain +* blocking_notifier_call_chain - Call functions in a blocking notifier chain * @nh: Pointer to head of the blocking notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function -* @nr_to_call: See comment for notifier_call_chain. -* @nr_calls: See comment for notifier_call_chain. * * Calls each function in a notifier chain in turn. The functions * run in a process context, so they are allowed to block. @@ -303,9 +357,8 @@ EXPORT_SYMBOL_GPL(blocking_notifier_chai* Otherwise the return value is the return value * of the last notifier function called. */ -int __blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v, -int nr_to_call,int *nr_calls) +int blocking_notifier_call_chain (struct blocking_notifier_head *nh, +unsigned long val,void *v) {int ret =NOTIFY_DONE ;@@ -316,19 +369,11 @@ int __blocking_notifier_call_chain(struc*/ if (rcu_access_pointer (nh->head)) {down_read (&nh->rwsem); - ret =notifier_call_chain (&nh->head, val, v, nr_to_call, - nr_calls); + ret =notifier_call_chain (&nh->head, val, v,-1 ,NULL );up_read (&nh->rwsem); }return ret; } -EXPORT_SYMBOL_GPL (__blocking_notifier_call_chain); - -int blocking_notifier_call_chain (struct blocking_notifier_head *nh, -unsigned long val,void *v) -{ -return __blocking_notifier_call_chain (nh, val, v,-1 ,NULL ); -}EXPORT_SYMBOL_GPL (blocking_notifier_call_chain);/* @@ -370,13 +415,18 @@ int raw_notifier_chain_unregister(struct }EXPORT_SYMBOL_GPL (raw_notifier_chain_unregister); +int raw_notifier_call_chain_robust (struct raw_notifier_head *nh, +unsigned long val_up,unsigned long val_down,void *v) +{ +return notifier_call_chain_robust (&nh->head, val_up, val_down, v); +} +EXPORT_SYMBOL_GPL (raw_notifier_call_chain_robust); +/** -* __raw_notifier_call_chain - Call functions in a raw notifier chain +* raw_notifier_call_chain - Call functions in a raw notifier chain * @nh: Pointer to head of the raw notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function -* @nr_to_call: See comment for notifier_call_chain. -* @nr_calls: See comment for notifier_call_chain * * Calls each function in a notifier chain in turn. The functions * run in an undefined context. @@ -389,18 +439,10 @@ EXPORT_SYMBOL_GPL(raw_notifier_chain_unr* Otherwise the return value is the return value * of the last notifier function called. */ -int __raw_notifier_call_chain (struct raw_notifier_head *nh, -unsigned long val,void *v, -int nr_to_call,int *nr_calls) -{ -return notifier_call_chain (&nh->head, val, v, nr_to_call, nr_calls); -} -EXPORT_SYMBOL_GPL (__raw_notifier_call_chain); -int raw_notifier_call_chain (struct raw_notifier_head *nh,unsigned long val,void *v) { -return __raw_notifier_call_chain (nh, val, v,-1 ,NULL ); +return notifier_call_chain (&nh->head, val, v,-1 ,NULL ); }EXPORT_SYMBOL_GPL (raw_notifier_call_chain);@@ -472,12 +514,10 @@ int srcu_notifier_chain_unregister(strucEXPORT_SYMBOL_GPL (srcu_notifier_chain_unregister);/** -* __srcu_notifier_call_chain - Call functions in an SRCU notifier chain +* srcu_notifier_call_chain - Call functions in an SRCU notifier chain * @nh: Pointer to head of the SRCU notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function -* @nr_to_call: See comment for notifier_call_chain. -* @nr_calls: See comment for notifier_call_chain * * Calls each function in a notifier chain in turn. The functions * run in a process context, so they are allowed to block. @@ -489,25 +529,17 @@ EXPORT_SYMBOL_GPL(srcu_notifier_chain_un* Otherwise the return value is the return value * of the last notifier function called. */ -int __srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v, -int nr_to_call,int *nr_calls) +int srcu_notifier_call_chain (struct srcu_notifier_head *nh, +unsigned long val,void *v) {int ret;int idx; idx =srcu_read_lock (&nh->srcu); - ret =notifier_call_chain (&nh->head, val, v, nr_to_call, nr_calls); + ret =notifier_call_chain (&nh->head, val, v,-1 ,NULL );srcu_read_unlock (&nh->srcu, idx);return ret; } -EXPORT_SYMBOL_GPL (__srcu_notifier_call_chain); - -int srcu_notifier_call_chain (struct srcu_notifier_head *nh, -unsigned long val,void *v) -{ -return __srcu_notifier_call_chain (nh, val, v,-1 ,NULL ); -}EXPORT_SYMBOL_GPL (srcu_notifier_call_chain);/** kernel/power/hibernate.c +12
-14@@ -693,8 +693,8 @@ static int load_image_and_restore(void)*/ int hibernate (void ) { -int error, nr_calls =0 ;bool snapshot_test = false; +int error;if (!hibernation_available ()) {pm_pr_dbg ("Hibernation not available.\n" );@@ -710,11 +710,9 @@ int hibernate(void)pr_info ("hibernation entry\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Exit; - } + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); +if (error) + goto Restore;ksys_sync_helper ();@@ -772,7 +770,8 @@ int hibernate(void)/* Don't bother checking whether freezer_test_done is true */ freezer_test_done = false; Exit: -__pm_notifier_call_chain (PM_POST_HIBERNATION , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_HIBERNATION ); + Restore:pm_restore_console ();atomic_inc (&snapshot_device_available); Unlock:@@ -800,7 +799,7 @@ int hibernate(void)*/ static int software_resume (void ) { -int error, nr_calls =0 ; +int error;/* * If the user said "noresume".. bail out early. @@ -887,11 +886,9 @@ static int software_resume(void)pr_info ("resume from hibernation\n" );pm_prepare_console (); - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Close_Finish; - } + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE ); +if (error) + goto Restore;pm_pr_dbg ("Preparing processes for restore.\n" ); error =freeze_processes ();@@ -900,7 +897,8 @@ static int software_resume(void) error =load_image_and_restore ();thaw_processes (); Finish: -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_RESTORE ); + Restore:pm_restore_console ();pr_info ("resume from hibernation failed (%d)\n" , error);atomic_inc (&snapshot_device_available);kernel/power/main.c +4
-4@@ -79,18 +79,18 @@ int unregister_pm_notifier(struct notifi }EXPORT_SYMBOL_GPL (unregister_pm_notifier); -int __pm_notifier_call_chain (unsigned long val,int nr_to_call,int *nr_calls) +int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down) {int ret; - ret =__blocking_notifier_call_chain (&pm_chain_head, val,NULL , - nr_to_call, nr_calls); + ret =blocking_notifier_call_chain_robust (&pm_chain_head, val_up, val_down,NULL );return notifier_to_errno (ret); } +int pm_notifier_call_chain (unsigned long val) { -return __pm_notifier_call_chain (val,-1 ,NULL ); +return blocking_notifier_call_chain (&pm_chain_head, val,NULL ); }/* If set, devices may be suspended and resumed asynchronously. */ kernel/power/power.h +1
-2@@ -210,8 +210,7 @@ static inline void suspend_test_finish(c#ifdef CONFIG_PM_SLEEP /* kernel/power/main.c */ -extern int __pm_notifier_call_chain (unsigned long val,int nr_to_call, -int *nr_calls); +extern int pm_notifier_call_chain_robust (unsigned long val_up,unsigned long val_down);extern int pm_notifier_call_chain (unsigned long val);#endif kernel/power/suspend.c +6
-8@@ -352,18 +352,16 @@ static int suspend_test(int level)*/ static int suspend_prepare (suspend_state_t state) { -int error, nr_calls =0 ; +int error;if (!sleep_state_supported (state))return -EPERM ;pm_prepare_console (); - error =__pm_notifier_call_chain (PM_SUSPEND_PREPARE ,-1 , &nr_calls); -if (error) { - nr_calls--; - goto Finish; - } + error =pm_notifier_call_chain_robust (PM_SUSPEND_PREPARE ,PM_POST_SUSPEND ); +if (error) + goto Restore;trace_suspend_resume (TPS ("freeze_processes" ),0 , true); error =suspend_freeze_processes ();@@ -373,8 +371,8 @@ static int suspend_prepare(suspend_state suspend_stats.failed_freeze++;dpm_save_failed_step (SUSPEND_FREEZE ); - Finish: -__pm_notifier_call_chain (PM_POST_SUSPEND , nr_calls,NULL ); +pm_notifier_call_chain (PM_POST_SUSPEND ); + Restore:pm_restore_console ();return error; }kernel/power/user.c +4
-10@@ -44,7 +44,7 @@ atomic_t snapshot_device_available = ATOstatic int snapshot_open (struct inode *inode,struct file *filp) {struct snapshot_data *data; -int error, nr_calls =0 ; +int error;if (!hibernation_available ())return -EPERM ;@@ -71,9 +71,7 @@ static int snapshot_open(struct inode *i swap_type_of(swsusp_resume_device, 0, NULL) : -1; data->mode =O_RDONLY ; data->free_bitmaps = false; - error =__pm_notifier_call_chain (PM_HIBERNATION_PREPARE ,-1 , &nr_calls); -if (error) -__pm_notifier_call_chain (PM_POST_HIBERNATION , --nr_calls,NULL ); + error =pm_notifier_call_chain_robust (PM_HIBERNATION_PREPARE ,PM_POST_HIBERNATION ); }else {/* * Resuming. We may need to wait for the image device to @@ -83,15 +81,11 @@ static int snapshot_open(struct inode *i data->swap =-1 ; data->mode =O_WRONLY ; - error =__pm_notifier_call_chain (PM_RESTORE_PREPARE ,-1 , &nr_calls); + error =pm_notifier_call_chain_robust (PM_RESTORE_PREPARE ,PM_POST_RESTORE );if (!error) { error =create_basic_memory_bitmaps (); data->free_bitmaps = !error; - }else - nr_calls--; - -if (error) -__pm_notifier_call_chain (PM_POST_RESTORE , nr_calls,NULL ); + } }if (error)atomic_inc (&snapshot_device_available);tools/power/pm-graph/sleepgraph.py +1
-1@@ -153,7 +153,7 @@ import base64 tracefuncs = { 'sys_sync': {}, 'ksys_sync': {}, - '__pm_notifier_call_chain': {}, + 'pm_notifier_call_chain_robust': {}, 'pm_prepare_console': {}, 'pm_notifier_call_chain': {}, 'freeze_processes': {},
Peter Zijlstra
maintainerPATCH v2 2/4 module: Fix up module_notifier return values.
In reply to Peter ZijlstraWhile auditing all module notifiers I noticed a whole bunch of fail wrt the return value. Notifiers have a 'special' return semantics.
As is; NOTIFY_DONE vs NOTIFY_OK is a bit vague; but notifier_from_errno(0) results in NOTIFY_OK and NOTIFY_DONE has a comment that says "Don't care".
Quoted 1 line
From this I've used
NOTIFY_DONEwhen the function completely ignores
the callback and notifier_to_error() isn't used.
--- drivers/oprofile/buffer_sync.c | 4 ++-- kernel/trace/bpf_trace.c | 8 ++++++-- kernel/trace/trace.c | 2 +- kernel/trace/trace_events.c | 2 +- kernel/trace/trace_printk.c | 4 ++-- kernel/tracepoint.c | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-)
drivers/oprofile/buffer_sync.c +2 -2
@@ -116,7 +116,7 @@ module_load_notify(struct notifier_block {#ifdef CONFIG_MODULES if (val !=MODULE_STATE_COMING ) -return 0 ; +return NOTIFY_DONE ;/* FIXME: should we process all CPU buffers ? */ mutex_lock (&buffer_mutex);
@@ -124,7 +124,7 @@ module_load_notify(struct notifier_blockadd_event_entry (MODULE_LOADED_CODE );mutex_unlock (&buffer_mutex);#endif -return 0 ; +return NOTIFY_OK ; }
kernel/trace/bpf_trace.c +6 -2
@@ -1451,10 +1451,11 @@ static int bpf_event_notify(struct notif {struct bpf_trace_module *btm, *tmp;struct module *mod = module; +int ret =0 ;if (mod->num_bpf_raw_events ==0 || (op !=MODULE_STATE_COMING && op !=MODULE_STATE_GOING )) -return 0 ; + goto out;mutex_lock (&bpf_module_mutex);
@@ -1464,6 +1465,8 @@ static int bpf_event_notify(struct notifif (btm) { btm->module = module;list_add (&btm->list, &bpf_trace_modules); + }else { + ret = -ENOMEM ; }break ;case MODULE_STATE_GOING :
@@ -1479,7 +1482,8 @@ static int bpf_event_notify(struct notifmutex_unlock (&bpf_module_mutex); -return 0 ; +out: +return notifier_from_errno (ret); }static struct notifier_block bpf_module_nb = {
kernel/trace/trace.c +1 -1
@@ -8696,7 +8696,7 @@ static int trace_module_notify(struct nobreak ; } -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_events.c +1 -1
@@ -2442,7 +2442,7 @@ static int trace_module_notify(struct nomutex_unlock (&trace_types_lock);mutex_unlock (&event_mutex); -return 0 ; +return NOTIFY_OK ; }static struct notifier_block trace_module_nb = {
kernel/trace/trace_printk.c +2 -2
@@ -95,7 +95,7 @@ static int module_trace_bprintk_format_nif (val ==MODULE_STATE_COMING )hold_module_trace_bprintk_format (start, end); } -return 0 ; +return NOTIFY_OK ; }/*
@@ -173,7 +173,7 @@ __init static int module_trace_bprintk_format_notify(struct notifier_block *self, unsigned long val, void *data) { - return 0; + return NOTIFY_OK; }static inline const char **find_next_mod_format (int start_index,void *v,const char **fmt,loff_t *pos)
kernel/tracepoint.c +1 -1
@@ -521,7 +521,7 @@ static int tracepoint_module_notify(strucase MODULE_STATE_UNFORMED :break ; } -return ret; +return notifier_from_errno (ret); }static struct notifier_block tracepoint_module_nb = {
Re: [PATCH v2 2/4] module: Fix up module_notifier return values.
Quoted 4 lines · On Mon, 07 Oct 2019 10:25:43 +0200 Peter Zijlstra <[email protected]> wrote:
kernel/trace/trace.c | 2 +- kernel/trace/trace_events.c | 2 +- kernel/trace/trace_printk.c | 4 ++-- kernel/tracepoint.c | 2 +-
-- Steve
Peter Zijlstra
maintainerPATCH v2 3/4 module: Properly propagate MODULE_STATE_COMING failure
In reply to Peter ZijlstraNow that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.
There were already MODULE_STATE_COMING notifiers that failed; notably:
- jump_label_module_notifier()
- tracepoint_module_notify()
- bpf_event_notify()
By propagating this error, we fix those users.
--- kernel/module.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)
kernel/module.c +7 -3
@@ -3751,9 +3751,13 @@ static int prepare_coming_module(structif (err)return err; -blocking_notifier_call_chain (&module_notify_list, -MODULE_STATE_COMING , mod); -return 0 ; + err =blocking_notifier_call_chain_robust (&module_notify_list, +MODULE_STATE_COMING ,MODULE_STATE_GOING , mod); + err =notifier_to_errno (err); +if (err) +klp_module_going (mod); + +return err; } static int unknown_module_param_cb(char *param, char *val, const char *modname,
Miroslav Benes
maintainerRe: [PATCH v2 3/4] module: Properly propagate MODULE_STATE_COMING failure
Quoted 29 lines · Peter Zijlstra
Now that notifiers got unbroken; use the proper interface to handle notifier errors and propagate them.
There were already
MODULE_STATE_COMINGnotifiers that failed; notably:-
jump_label_module_notifier()
-tracepoint_module_notify()
-bpf_event_notify()By propagating this error, we fix those users.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Yonghong Song <[email protected]>Cc Alexei Starovoitov <[email protected]>Cc Daniel Borkmann <[email protected]>Cc Song Liu <[email protected]>Cc Jessica Yu <[email protected]>Cc Martin KaFai Lau <[email protected]>--- kernel/module.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)kernel/module.c +0
-1@@ -3751,9 +3751,13 @@ static int prepare_coming_module(structif (err)return err; - blocking_notifier_call_chain(&module_notify_list,
kernel/module.c static int prepare_coming_module(struct · line 3761
Quoted 10 lines
kernel/module.c +7
-2- MODULE_STATE_COMING, mod); -return 0 ; + err =blocking_notifier_call_chain_robust (&module_notify_list, +MODULE_STATE_COMING ,MODULE_STATE_GOING , mod); + err =notifier_to_errno (err); +if (err) +klp_module_going (mod); + +return err; }
It looks almost ok. At least klp_ error handling is correct now. I see only one possible problem. If there is an error in a MODULE_STATE_COMING notifier, all MODULE_STATE_GOING notifiers will be called with mod->state set to MODULE_STATE_COMING. Not nice. I don't think it is actually a problem, because all notifiers, that I checked, only use the correct value (MODULE_STATE_COMING or MODULE_STATE_GOING, coming from the function parameter) and not mod->state. Better to doublecheck though.
However, mod->state is not set to MODULE_STATE_GOING anywhere under bug_cleanup label in load_module(). That is a bug and it is there regardless of this patch.
Jessica?
Regards
Miroslav
Peter Zijlstra
maintainerPATCH v2 4/4 jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved
In reply to Peter ZijlstraNothing ensures the module exists while we're iterating mod->jump_entries in __jump_label_mod_text_reserved(), take a module reference to ensure the module sticks around.
--- kernel/jump_label.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/jump_label.c +8 -2
@@ -539,19 +539,25 @@ static void static_key_set_mod(struct ststatic int __jump_label_mod_text_reserved (void *start,void *end) {struct module *mod; +int ret;preempt_disable (); mod =__module_text_address ((unsigned long )start);WARN_ON_ONCE (__module_text_address ((unsigned long )end) != mod); +if (!try_module_get (mod)) + mod =NULL ;preempt_enable ();if (!mod)return 0 ; - -return __jump_label_text_reserved (mod->jump_entries, + ret =__jump_label_text_reserved (mod->jump_entries, mod->jump_entries + mod->num_jump_entries, start, end); + +module_put (mod); + +return ret; }static void __jump_label_mod_update (struct static_key *key)
Re: [PATCH v2 4/4] jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved
Quoted 5 lines · On Mon, 07 Oct 2019 10:25:45 +0200 Peter Zijlstra <[email protected]> wrote:
Nothing ensures the module exists while we're iterating
mod->jump_entriesin__jump_label_mod_text_reserved(), take a module reference to ensure the module sticks around.Signed-off-by Peter Zijlstra (Intel) <[email protected]>
-- Steve
Quoted 35 lines
--- kernel/jump_label.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)kernel/jump_label.c +8
-2@@ -539,19 +539,25 @@ static void static_key_set_mod(struct ststatic int __jump_label_mod_text_reserved (void *start,void *end) {struct module *mod; +int ret;preempt_disable (); mod =__module_text_address ((unsigned long )start);WARN_ON_ONCE (__module_text_address ((unsigned long )end) != mod); +if (!try_module_get (mod)) + mod =NULL ;preempt_enable ();if (!mod)return 0 ; - -return __jump_label_text_reserved (mod->jump_entries, + ret =__jump_label_text_reserved (mod->jump_entries, mod->jump_entries + mod->num_jump_entries, start, end); + +module_put (mod); + +return ret; }static void __jump_label_mod_update (struct static_key *key)
Peter Zijlstra
maintainerv3 cover Rewrite x86/ftrace to use text_poke()
In reply to Peter ZijlstraFtrace was one of the last W^X violators; these patches move it over to the generic text_poke() interface and thereby get rid of this oddity.
Peter Zijlstra
maintainerPATCH v3 1/6 x86/alternatives: Teach text_poke_bp() to emulate instructions
In preparation for static_call and variable size jump_label support, teach text_poke_bp() to emulate instructions, namely:
JMP32, JMP8, CALL, NOP2, NOP_ATOMIC5, INT3
The current text_poke_bp() takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach poke_int3_handler() to emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except arch_unoptimize_kprobe() which restores random text, and for that site we have to build an explicit emulate instruction.
--- arch/x86/include/asm/text-patching.h | 24 ++++-- arch/x86/kernel/alternative.c | 132 ++++++++++++++++++++++++++--------- arch/x86/kernel/jump_label.c | 9 -- arch/x86/kernel/kprobes/opt.c | 11 ++ 4 files changed, 130 insertions(+), 46 deletions(-)
arch/x86/include/asm/text-patching.h +18 -6
@@ -26,10 +26,11 @@ static inline void apply_paravirt(struct#define POKE_MAX_OPCODE_SIZE 5struct text_poke_loc { -void *detour;void *addr; -size_t len; -const char opcode[POKE_MAX_OPCODE_SIZE ]; +int len; +s32 rel32; +u8 opcode; +const char text[POKE_MAX_OPCODE_SIZE ]; };extern void text_poke_early (void *addr,const void *opcode,size_t len);
@@ -51,8 +52,10 @@ extern void text_poke_early(void *addr,extern void *text_poke (void *addr,const void *opcode,size_t len);extern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs); -extern void text_poke_bp (void *addr,const void *opcode,size_t len,void *handler); +extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate);extern void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries); +extern void text_poke_loc_init (struct text_poke_loc *tp,void *addr, +const void *opcode,size_t len,const void *emulate);extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1 +#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP32_INSN_SIZE 5 +#define JMP32_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEBstatic inline void int3_emulate_push (struct pt_regs *regs,unsigned long val) {
arch/x86/kernel/alternative.c +101 -31
202 lines not on this page · raw message
arch/x86/kernel/jump_label.c +3 -6
@@ -89,8 +89,7 @@ static void __ref __jump_label_transformreturn ; } -text_poke_bp ((void *)jump_entry_code (entry), &code,JUMP_LABEL_NOP_SIZE , - (void *)jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); +text_poke_bp ((void *)jump_entry_code (entry), &code,JUMP_LABEL_NOP_SIZE ,NULL ); } void arch_jump_label_transform(struct jump_entry *entry,
@@ -147,11 +146,9 @@ bool arch_jump_label_transform_queue(str }__jump_label_set_jump_code (entry, type, - (union jump_code_union *) &tp->opcode,0 ); + (union jump_code_union *)&tp->text,0 ); - tp->addr = entry_code; - tp->detour = entry_code +JUMP_LABEL_NOP_SIZE ; - tp->len =JUMP_LABEL_NOP_SIZE ; +text_poke_loc_init (tp, entry_code,NULL ,JUMP_LABEL_NOP_SIZE ,NULL ); tp_vec_nr++;
arch/x86/kernel/kprobes/opt.c +8 -3
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h insn_buff[0 ] =RELATIVEJUMP_OPCODE ; *(s32 *)(&insn_buff[1 ]) = rel; -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE ,NULL );list_del_init (&op->list); }
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; +u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); + + emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; + *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - + ((long )op->kp.addr +RELATIVEJUMP_SIZE )); +text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - op->optinsn.insn); + emulate_buff); }/*
Borislav Petkov
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 15 lines · Peter Zijlstra
In preparation for
static_calland variable sizejump_labelsupport, teachtext_poke_bp()to emulate instructions, namely:JMP32, JMP8, CALL, NOP2,
NOP_ATOMIC5, INT3The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.
...
Quoted 7 lines
arch/x86/include/asm/text-patching.h +1
-2@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1
arch/x86/include/asm/text-patching.h static inline void int3_emulate_jmp(stru · line 79
Quoted 10 lines
arch/x86/include/asm/text-patching.h +10
-0+#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP32_INSN_SIZE 5 +#define JMP32_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEB
You probably should switch those to have the name prefix come first and make them even shorter:
OPCODE_CALLINSN_SIZE_CALLOPCODE_JMP32INSN_SIZE_JMP32OPCODE_JMP8
...
This way you have the opcodes prefixed with OPCODE_ and the insn sizes with INSN_SIZE_. I.e., what they actually are.
Quoted 2 lines
arch/x86/kernel/alternative.c +0
-0
...
Quoted 34 lines
arch/x86/kernel/alternative.c +6
-6@@ -1027,9 +1046,9 @@ NOKPROBE_SYMBOL(poke_int3_handler);*/ void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries) { -int patched_all_but_first =0 ; -unsigned char int3 =0xcc ; +unsigned char int3 =INT3_INSN_OPCODE ;unsigned int i; +int do_sync;lockdep_assert_held (&text_mutex);@@ -1053,16 +1072,16 @@ void text_poke_bp_batch(struct text_poke/* * Second step: update all but the first byte of the patched range. */ -for (i =0 ; i < nr_entries; i++) { +for (do_sync =0 , i =0 ; i < nr_entries; i++) {if (tp[i].len -sizeof (int3) >0 ) {text_poke ((char *)tp[i].addr +sizeof (int3), - (const char *)tp[i].opcode +sizeof (int3), + (const char *)tp[i].text +sizeof (int3), tp[i].len -sizeof (int3)); - patched_all_but_first++; + do_sync++; } } -if (patched_all_but_first) { +if (do_sync) {/* * According to Intel, this core syncing is very likely * not necessary and we'd be safe even without it. But
arch/x86/kernel/alternative.c void text_poke_bp_batch(struct text_poke · line 1097
Quoted 6 lines
arch/x86/kernel/alternative.c +1
-2@@ -1075,10 +1094,17 @@ void text_poke_bp_batch(struct text_poke* Third step: replace the first byte (int3) by the first byte of * replacing opcode. */ -for (i =0 ; i < nr_entries; i++) -text_poke (tp[i].addr, tp[i].opcode,sizeof (int3)); +for (do_sync =0 , i =0 ; i < nr_entries; i++) {
Can we have the do_sync reset outside of the loop?
arch/x86/kernel/alternative.c void text_poke_bp_batch(struct text_poke · line 1099
Quoted 2 lines
arch/x86/kernel/alternative.c +2
-0+if (tp[i].text[0 ] ==INT3_INSN_OPCODE ) +continue ;
I'm guessing we preset the 0th byte to 0xcc somewhere.... I just can't seem to find it...
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteSteven Rostedt
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 47 lines · On Tue, 8 Oct 2019 16:29:24 +0200 Borislav Petkov <[email protected]> wrote:
Quoted 15 lines · Peter Zijlstra
In preparation for
static_calland variable sizejump_labelsupport, teachtext_poke_bp()to emulate instructions, namely:JMP32, JMP8, CALL, NOP2,
NOP_ATOMIC5, INT3The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction....
Quoted 7 lines
arch/x86/include/asm/text-patching.h +1
-2@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1arch/x86/include/asm/text-patching.h static inline void int3_emulate_jmp(stru · line 79
Quoted 10 lines
arch/x86/include/asm/text-patching.h +10
-0+#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP32_INSN_SIZE 5 +#define JMP32_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEBYou probably should switch those to have the name prefix come first and make them even shorter:
OPCODE_CALLINSN_SIZE_CALLOPCODE_JMP32INSN_SIZE_JMP32OPCODE_JMP8
...
Quoted 2 lines
This way you have the opcodes prefixed with
OPCODE_and the insn sizes withINSN_SIZE_. I.e., what they actually are.
Honestly, I like the original way better.
Seeing OPCODE_JMP32 and INSN_SIZE_JMP32 doesn't look like they are related to me.
-- Steve
Borislav Petkov
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 2 lines · Steven Rostedt
Seeing
OPCODE_JMP32andINSN_SIZE_JMP32doesn't look like they are related to me.
But if it starts with OPCODE_ you know what it is - an opcode. JMP32_INSN_OPCODE can first be shortened to JMP32_OPCODE and then having instruction mnemonic start the macro name doesn't show they belong to the same type of things. Ditto for the insn size.
But I'm way too busy to bikeshed so whatever...
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquettePeter Zijlstra
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Borislav PetkovQuoted 30 lines · Borislav Petkov
Quoted 7 lines · Peter Zijlstra
arch/x86/include/asm/text-patching.h +1
-2@@ -63,8 +66,17 @@ static inline void int3_emulate_jmp(stru regs->ip = ip; } -#define INT3_INSN_SIZE 1 -#define CALL_INSN_SIZE 5 +#define INT3_INSN_SIZE 1arch/x86/include/asm/text-patching.h static inline void int3_emulate_jmp(stru · line 79
Quoted 10 lines
arch/x86/include/asm/text-patching.h +10
-0+#define INT3_INSN_OPCODE 0xCC + +#define CALL_INSN_SIZE 5 +#define CALL_INSN_OPCODE 0xE8 + +#define JMP32_INSN_SIZE 5 +#define JMP32_INSN_OPCODE 0xE9 + +#define JMP8_INSN_SIZE 2 +#define JMP8_INSN_OPCODE 0xEBYou probably should switch those to have the name prefix come first and make them even shorter:
OPCODE_CALLINSN_SIZE_CALLOPCODE_JMP32INSN_SIZE_JMP32OPCODE_JMP8
...
Quoted 2 lines
This way you have the opcodes prefixed with
OPCODE_and the insn sizes withINSN_SIZE_. I.e., what they actually are.
I really don't like that; the important part is which instruction and that really should come first. Also, your variant is horribly inconsistent.
Quoted 33 lines
Quoted 2 lines
arch/x86/kernel/alternative.c +0
-0...
Quoted 28 lines
arch/x86/kernel/alternative.c +5
-5@@ -1027,9 +1046,9 @@ NOKPROBE_SYMBOL(poke_int3_handler);*/ void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries) { -int patched_all_but_first =0 ; -unsigned char int3 =0xcc ; +unsigned char int3 =INT3_INSN_OPCODE ;unsigned int i; +int do_sync;lockdep_assert_held (&text_mutex);@@ -1053,16 +1072,16 @@ void text_poke_bp_batch(struct text_poke/* * Second step: update all but the first byte of the patched range. */ -for (i =0 ; i < nr_entries; i++) { +for (do_sync =0 , i =0 ; i < nr_entries; i++) {if (tp[i].len -sizeof (int3) >0 ) {text_poke ((char *)tp[i].addr +sizeof (int3), - (const char *)tp[i].opcode +sizeof (int3), + (const char *)tp[i].text +sizeof (int3), tp[i].len -sizeof (int3)); - patched_all_but_first++; + do_sync++; } }
Quoted 14 lines
arch/x86/kernel/alternative.c void text_poke_bp_batch(struct text_poke · line 1097
Quoted 12 lines
arch/x86/kernel/alternative.c +2
-3-if (patched_all_but_first) { +if (do_sync) {/* * According to Intel, this core syncing is very likely * not necessary and we'd be safe even without it. But @@ -1075,10 +1094,17 @@ void text_poke_bp_batch(struct text_poke* Third step: replace the first byte (int3) by the first byte of * replacing opcode. */ -for (i =0 ; i < nr_entries; i++) -text_poke (tp[i].addr, tp[i].opcode,sizeof (int3)); +for (do_sync =0 , i =0 ; i < nr_entries; i++) {Can we have the
do_syncreset outside of the loop?
Can, but why? That's more lines for no raisin ;-)
Quoted 5 lines
arch/x86/kernel/alternative.c void text_poke_bp_batch(struct text_poke · line 1099
Quoted 2 lines
arch/x86/kernel/alternative.c +2
-0+if (tp[i].text[0 ] ==INT3_INSN_OPCODE ) +continue ;I'm guessing we preset the 0th byte to
0xccsomewhere.... I just can't seem to find it...
Very first pass, we write INT3's everywhere.
Borislav Petkov
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 1 line · Peter Zijlstra
Can, but why? That's more lines for no raisin ;-)
Here's a raisin: I was looking at this and then all of a sudden went:
"W00t, why is this do_sync part of the loop at all? Do they belong together? Nope."
If we're going to save on lines, we've lost long ago.
Quoted 1 line
Very first pass, we write INT3's everywhere.
Doh.
Thx.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteSteven Rostedt
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 7 lines · On Tue, 8 Oct 2019 16:54:24 +0200 Borislav Petkov <[email protected]> wrote:
Quoted 1 line · Peter Zijlstra
Can, but why? That's more lines for no raisin ;-)
Here's a raisin: I was looking at this and then all of a sudden went:
"W00t, why is thisdo_syncpart of the loop at all? Do they belong together? Nope."
But it is part of the loop...
+ for (do_sync = 0, i = 0; i < nr_entries; i++) {
+ if (tp[i].text[0] == INT3_INSN_OPCODE)
+ continue;
+
+ text_poke(tp[i].addr, tp[i].text, sizeof(int3));
+ do_sync++;
+ }
+
The difference between do_sync and i is that i gets incremented at every iteration, where do_sync gets incremented only when the first conditional is false. But I still see do_sync as a loop variable.
-- Steve
Borislav Petkov
maintainerRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
Quoted 3 lines · Steven Rostedt
The difference between
do_syncand i is that i gets incremented at every iteration, wheredo_syncgets incremented only when the first conditional is false. But I still seedo_syncas a loop variable.
I'd prefer it this way:
do_sync = 0;
for (i = 0; i < nr_entries; i++) {
if (tp[i].text[0] == INT3_INSN_OPCODE)
continue;
text_poke(tp[i].addr, tp[i].text, sizeof(int3));
do_sync++;
}
if (do_sync)
on_each_cpu(do_sync_core, NULL, 1);Clear and simple. We clear it, the loop runs and we check it after the loop ends. Clear and simple pattern which you see everywhere in code. All well known and uneventful.
Now if the do_sync clearing is in the for () brackets, you have to stop and look for it and wonder, why is that thing there, is there anything special about it?
And with the amount of code going through us every day, I'd prefer well known and uneventful in any day of the week.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteRe: [PATCH v3 1/6] x86/alternatives: Teach text_poke_bp() to emulate instructions
In reply to Peter ZijlstraQuoted 12 lines · Peter Zijlstra
In preparation for
static_calland variable sizejump_labelsupport, teachtext_poke_bp()to emulate instructions, namely:JMP32, JMP8, CALL, NOP2,
NOP_ATOMIC5, INT3The current
text_poke_bp()takes a @handler argument which is used as a jump target when the temporary INT3 is hit by a different CPU.When patching CALL instructions, this doesn't work because we'd miss the PUSH of the return address. Instead, teach
poke_int3_handler()to emulate an instruction, typically the instruction we're patching in.
Quoted 8 lines
This fits almost all
text_poke_bp()users, exceptarch_unoptimize_kprobe()which restores random text, and for that site we have to build an explicit emulate instruction.Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Masami Hiramatsu <[email protected]>Cc Steven Rostedt <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>
Thanks!
-- Daniel
Peter Zijlstra
maintainerPATCH v3 2/6 x86/alternatives: Update int3_emulate_push() comment
In reply to Peter ZijlstraUpdate the comment now that we've merged x86_32 support.
--- arch/x86/include/asm/text-patching.h | 3 +++ 1 file changed, 3 insertions(+)
arch/x86/include/asm/text-patching.h +3 -0
@@ -85,6 +85,9 @@ static inline void int3_emulate_push(str* stack where the break point happened, and the saving of * pt_regs. We can extend the original stack because of * this gap. See the idtentry macro's create_gap option. +* +* Similarly entry_32.S will have a gap on the stack for (any) hardware +* exception and pt_regs; see FIXUP_FRAME. */ regs->sp -=sizeof (unsigned long ); *(unsigned long *)regs->sp = val;
Peter Zijlstra
maintainerPATCH v3 3/6 x86/alternatives,jump_label: Provide better text_poke() batching interface
In reply to Peter ZijlstraAdding another text_poke_bp_batch() user made me realize the interface is all sorts of wrong. The text poke vector should be internal to the implementation.
This then results in a trivial interface:
text_poke_queue() - which has the 'normal' text_poke_bp() interface text_poke_finish() - which takes no arguments and flushes any
pending text_poke()s.
--- arch/x86/include/asm/text-patching.h | 15 +---- arch/x86/kernel/alternative.c | 64 ++++++++++++++++++++-- arch/x86/kernel/jump_label.c | 99 ++++++++++++----------------------- 3 files changed, 96 insertions(+), 82 deletions(-)
arch/x86/include/asm/text-patching.h +4 -11
@@ -25,14 +25,6 @@ static inline void apply_paravirt(struct*/ #define POKE_MAX_OPCODE_SIZE 5 -struct text_poke_loc { -void *addr; -int len; -s32 rel32; -u8 opcode; -const char text[POKE_MAX_OPCODE_SIZE ]; -}; -extern void text_poke_early (void *addr,const void *opcode,size_t len);/*
@@ -53,9 +45,10 @@ extern void *text_poke(void *addr, constextern void *text_poke_kgdb (void *addr,const void *opcode,size_t len);extern int poke_int3_handler (struct pt_regs *regs);extern void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate); -extern void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries); -extern void text_poke_loc_init (struct text_poke_loc *tp,void *addr, -const void *opcode,size_t len,const void *emulate); + +extern void text_poke_queue (void *addr,const void *opcode,size_t len,const void *emulate); +extern void text_poke_finish (void ); +extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +58 -6
94 lines not on this page · raw message
arch/x86/kernel/jump_label.c +34 -65
164 lines not on this page · raw message
Re: [PATCH v3 3/6] x86/alternatives,jump_label: Provide better text_poke() batching interface
Quoted 6 lines · Peter Zijlstra
Adding another
text_poke_bp_batch()user made me realize the interface is all sorts of wrong. The text poke vector should be internal to the implementation.This then results in a trivial interface:
Quoted 8 lines
text_poke_queue()- which has the 'normal'text_poke_bp()interfacetext_poke_finish()- which takes no arguments and flushes any
pending text_poke()s.Signed-off-by Peter Zijlstra (Intel) <[email protected]>Reviewed-by Masami Hiramatsu <[email protected]>Cc Steven Rostedt <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>
Thanks!
-- Daniel
Peter Zijlstra
maintainerPATCH v3 4/6 x86/alternatives: Add and use text_gen_insn() helper
In reply to Peter ZijlstraProvide a simple helper function to create common instruction encodings.
--- arch/x86/include/asm/text-patching.h | 2 + arch/x86/kernel/alternative.c | 36 +++++++++++++++++++++++++++++++++++ arch/x86/kernel/jump_label.c | 31 ++++++++++-------------------- arch/x86/kernel/kprobes/opt.c | 7 ------ 4 files changed, 50 insertions(+), 26 deletions(-)
arch/x86/include/asm/text-patching.h +2 -0
@@ -49,6 +49,8 @@ extern void text_poke_bp(void *addr, conextern void text_poke_queue (void *addr,const void *opcode,size_t len,const void *emulate);extern void text_poke_finish (void ); +extern void *text_gen_insn (u8 opcode,const void *addr,const void *dest); +extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;
arch/x86/kernel/alternative.c +36 -0
@@ -1237,3 +1237,39 @@ void text_poke_bp(void *addr, const voidtext_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); } + +union text_poke_insn { +u8 text[POKE_MAX_OPCODE_SIZE ]; +struct { +u8 opcode; +s32 disp; + } __attribute__((packed)); +}; + +void *text_gen_insn (u8 opcode,const void *addr,const void *dest) +{ +static union text_poke_insn insn;/* text_mutex */ +int size =0 ; + +lockdep_assert_held (&text_mutex); + + insn.opcode = opcode; + +#define __CASE (insn) \ + case insn##_INSN_OPCODE: size = insn##_INSN_SIZE; break + +switch (opcode) { +__CASE (INT3 ); +__CASE (CALL ); +__CASE (JMP32 ); +__CASE (JMP8 ); + } + +if (size >1 ) { + insn.disp = (long )dest - (long )(addr + size); +if (size ==2 ) +BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 )); + } + +return &insn.text; +}
arch/x86/kernel/jump_label.c +11 -20
@@ -16,15 +16,7 @@#include <asm/alternative.h> #include <asm/text-patching.h> -union jump_code_union { -char code[JUMP_LABEL_NOP_SIZE ]; -struct { -char jump; -int offset; - } __attribute__((packed)); -}; - -static void bug_at (unsigned char *ip,int line) +static void bug_at (const void *ip,int line) {/* * The location is not an op that we were expecting.
@@ -38,33 +30,32 @@ static void bug_at(unsigned char *ip, instatic const void *__jump_label_set_jump_code (struct jump_entry *entry,enum jump_label_type type,int init) { -static union jump_code_union code;/* relies on text_mutex */ const unsigned char default_nop[] = {STATIC_KEY_INIT_NOP };const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; -const void *expect; +const void *expect, *code; +const void *addr, *dest;int line; -lockdep_assert_held (&text_mutex); + addr = (void *)jump_entry_code (entry); + dest = (void *)jump_entry_target (entry); - code.jump =JMP32_INSN_OPCODE ; - code.offset =jump_entry_target (entry) - - (jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); + code =text_gen_insn (JMP32_INSN_OPCODE , addr, dest);if (init) { expect = default_nop; line = __LINE__; }else if (type ==JUMP_LABEL_JMP ) { expect = ideal_nop; line = __LINE__; }else { - expect = code.code; line = __LINE__; + expect = code; line = __LINE__; } -if (memcmp ((void *)jump_entry_code (entry), expect,JUMP_LABEL_NOP_SIZE )) -bug_at ((void *)jump_entry_code (entry), line); +if (memcmp (addr, expect,JUMP_LABEL_NOP_SIZE )) +bug_at (addr, line);if (type ==JUMP_LABEL_NOP ) -memcpy (&code, ideal_nop,JUMP_LABEL_NOP_SIZE ); + code = ideal_nop; -return &code; +return code; } static void inline __jump_label_transform(struct jump_entry *entry,
arch/x86/kernel/kprobes/opt.c +1 -6
@@ -447,18 +447,13 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; -u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); - emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; - *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - - ((long )op->kp.addr +RELATIVEJUMP_SIZE )); -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - emulate_buff); +text_gen_insn (JMP32_INSN_OPCODE , op->kp.addr, op->optinsn.insn)); }/*
Re: [PATCH v3 4/6] x86/alternatives: Add and use text_gen_insn() helper
Quoted 2 lines · On Mon, 07 Oct 2019 10:17:20 +0200 Peter Zijlstra <[email protected]> wrote:
Provide a simple helper function to create common instruction encodings.
Thanks for using correct INSN_OPCODE:)
This looks good to me.
Thanks,
Quoted 152 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Cc Steven Rostedt <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Cc Masami Hiramatsu <[email protected]>--- arch/x86/include/asm/text-patching.h | 2 + arch/x86/kernel/alternative.c | 36 +++++++++++++++++++++++++++++++++++ arch/x86/kernel/jump_label.c | 31 ++++++++++-------------------- arch/x86/kernel/kprobes/opt.c | 7 ------ 4 files changed, 50 insertions(+), 26 deletions(-)arch/x86/include/asm/text-patching.h +2
-0@@ -49,6 +49,8 @@ extern void text_poke_bp(void *addr, conextern void text_poke_queue (void *addr,const void *opcode,size_t len,const void *emulate);extern void text_poke_finish (void ); +extern void *text_gen_insn (u8 opcode,const void *addr,const void *dest); +extern int after_bootmem;extern __ro_after_init struct mm_struct *poking_mm;extern __ro_after_init unsigned long poking_addr;arch/x86/kernel/alternative.c +36
-0@@ -1237,3 +1237,39 @@ void text_poke_bp(void *addr, const voidtext_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); } + +union text_poke_insn { +u8 text[POKE_MAX_OPCODE_SIZE ]; +struct { +u8 opcode; +s32 disp; + } __attribute__((packed)); +}; + +void *text_gen_insn (u8 opcode,const void *addr,const void *dest) +{ +static union text_poke_insn insn;/* text_mutex */ +int size =0 ; + +lockdep_assert_held (&text_mutex); + + insn.opcode = opcode; + +#define __CASE (insn) \ + case insn##_INSN_OPCODE: size = insn##_INSN_SIZE; break + +switch (opcode) { +__CASE (INT3 ); +__CASE (CALL ); +__CASE (JMP32 ); +__CASE (JMP8 ); + } + +if (size >1 ) { + insn.disp = (long )dest - (long )(addr + size); +if (size ==2 ) +BUG_ON ((insn.disp >>31 ) != (insn.disp >>7 )); + } + +return &insn.text; +}arch/x86/kernel/jump_label.c +11
-20@@ -16,15 +16,7 @@#include <asm/alternative.h> #include <asm/text-patching.h> -union jump_code_union { -char code[JUMP_LABEL_NOP_SIZE ]; -struct { -char jump; -int offset; - } __attribute__((packed)); -}; - -static void bug_at (unsigned char *ip,int line) +static void bug_at (const void *ip,int line) {/* * The location is not an op that we were expecting. @@ -38,33 +30,32 @@ static void bug_at(unsigned char *ip, instatic const void *__jump_label_set_jump_code (struct jump_entry *entry,enum jump_label_type type,int init) { -static union jump_code_union code;/* relies on text_mutex */ const unsigned char default_nop[] = {STATIC_KEY_INIT_NOP };const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; -const void *expect; +const void *expect, *code; +const void *addr, *dest;int line; -lockdep_assert_held (&text_mutex); + addr = (void *)jump_entry_code (entry); + dest = (void *)jump_entry_target (entry); - code.jump =JMP32_INSN_OPCODE ; - code.offset =jump_entry_target (entry) - - (jump_entry_code (entry) +JUMP_LABEL_NOP_SIZE ); + code =text_gen_insn (JMP32_INSN_OPCODE , addr, dest);if (init) { expect = default_nop; line = __LINE__; }else if (type ==JUMP_LABEL_JMP ) { expect = ideal_nop; line = __LINE__; }else { - expect = code.code; line = __LINE__; + expect = code; line = __LINE__; } -if (memcmp ((void *)jump_entry_code (entry), expect,JUMP_LABEL_NOP_SIZE )) -bug_at ((void *)jump_entry_code (entry), line); +if (memcmp (addr, expect,JUMP_LABEL_NOP_SIZE )) +bug_at (addr, line);if (type ==JUMP_LABEL_NOP ) -memcpy (&code, ideal_nop,JUMP_LABEL_NOP_SIZE ); + code = ideal_nop; -return &code; +return code; } static void inline __jump_label_transform(struct jump_entry *entry,arch/x86/kernel/kprobes/opt.c +1
-6@@ -447,18 +447,13 @@ void arch_optimize_kprobes(struct list_hvoid arch_unoptimize_kprobe (struct optimized_kprobe *op) {u8 insn_buff[RELATIVEJUMP_SIZE ]; -u8 emulate_buff[RELATIVEJUMP_SIZE ];/* Set int3 to first byte for kprobes */ insn_buff[0 ] =BREAKPOINT_INSTRUCTION ;memcpy (insn_buff +1 , op->optinsn.copied_insn,RELATIVE_ADDR_SIZE ); - emulate_buff[0 ] =RELATIVEJUMP_OPCODE ; - *(s32 *)(&emulate_buff[1 ]) = (s32)((long )op->optinsn.insn - - ((long )op->kp.addr +RELATIVEJUMP_SIZE )); -text_poke_bp (op->kp.addr, insn_buff,RELATIVEJUMP_SIZE , - emulate_buff); +text_gen_insn (JMP32_INSN_OPCODE , op->kp.addr, op->optinsn.insn)); }/*
Signature · 1 line
Masami Hiramatsu <[email protected]>
Peter Zijlstra
maintainerRe: [PATCH v3 4/6] x86/alternatives: Add and use text_gen_insn() helper
Quoted 9 lines · Masami Hiramatsu
Quoted 2 lines · On Mon, 07 Oct 2019 10:17:20 +0200 Peter Zijlstra <[email protected]> wrote:
Provide a simple helper function to create common instruction encodings.
Thanks for using correct
INSN_OPCODE:)
This looks good to me.
Right, I have it on my todo list to convert all of kprobes over. But feel tree to beat me to it :-)
Peter Zijlstra
maintainerPATCH v3 5/6 x86/ftrace: Use text_poke()
In reply to Peter ZijlstraMove ftrace over to using the generic x86 text_poke functions; this avoids having a second/different copy of that code around.
This also avoids ftrace violating the (new) W^X rule and avoids fragmenting the kernel text page-tables, due to no longer having to toggle them RW.
--- arch/x86/include/asm/ftrace.h | 2 arch/x86/kernel/alternative.c | 4 arch/x86/kernel/ftrace.c | 630 ++++++------------------------------------ arch/x86/kernel/traps.c | 9 4 files changed, 93 insertions(+), 552 deletions(-)
arch/x86/include/asm/ftrace.h +0 -2
@@ -35,8 +35,6 @@ struct dyn_arch_ftrace { /* No extra data needed for x86 */ }; -int ftrace_int3_handler (struct pt_regs *regs); -#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR#endif /* CONFIG_DYNAMIC_FTRACE */
arch/x86/kernel/alternative.c +2 -2
@@ -941,7 +941,7 @@ static struct bp_patching_desc {int nr_entries; } bp_patching; -static int patch_cmp (const void *key,const void *elt) +static int notrace patch_cmp(const void *key, const void *elt) {struct text_poke_loc *tp = (struct text_poke_loc *) elt;
@@ -953,7 +953,7 @@ static int patch_cmp(const void *key, co }NOKPROBE_SYMBOL (patch_cmp); -int poke_int3_handler (struct pt_regs *regs) +int notrace poke_int3_handler(struct pt_regs *regs) {struct text_poke_loc *tp;void *ip;
arch/x86/kernel/ftrace.c +91 -539
799 lines not on this page · raw message
arch/x86/kernel/traps.c +0 -9
@@ -568,15 +568,6 @@ NOKPROBE_SYMBOL(do_general_protection); dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) { -#ifdef CONFIG_DYNAMIC_FTRACE -/* -* ftrace must be first, everything else may cause a recursive crash. -* See note by declaration of modifying_ftrace_code in ftrace.c -*/ -if (unlikely (atomic_read (&modifying_ftrace_code)) && -ftrace_int3_handler (regs)) -return ; -#endif if (poke_int3_handler (regs))return ;
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 11 lines · On Mon, 07 Oct 2019 10:17:21 +0200 Peter Zijlstra <[email protected]> wrote:
Move ftrace over to using the generic x86
text_pokefunctions; this avoids having a second/different copy of that code around.This also avoids ftrace violating the (new) W^X rule and avoids fragmenting the kernel text page-tables, due to no longer having to toggle them RW.
Cc Steven Rostedt <[email protected]>Cc Daniel Bristot de Oliveira <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
I'll review the other series in this thread, but I'm assuming they don't rely on this series? Or do they?
-- Steve
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 3 lines · Steven Rostedt
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
Works for me; or we can have a topic branch in tip we both can use. Ingo?
Quoted 2 lines
I'll review the other series in this thread, but I'm assuming they don't rely on this series? Or do they?
Indeed, this series stands on it's own. The rest depends on this.
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 8 lines · On Tue, 8 Oct 2019 19:11:37 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 3 lines · Steven Rostedt
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
Works for me; or we can have a topic branch in tip we both can use. Ingo?
In case you want to use this branch, I just pushed:
git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
ftrace/text-poke
That is these 6 patches applied to v5.4-rc2
I'm going to start running them through my tests. I can work on these directly. And if Ingo wants to pull this into tip, then we can do that, and apply the other patches on top of them.
Ingo, would that work for you?
-- Steve
Quoted 5 lines
Quoted 3 lines
I'll review the other series in this thread, but I'm assuming they don't rely on this series? Or do they?
Indeed, this series stands on it's own. The rest depends on this.
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Steven RostedtQuoted 3 lines · On Tue, 8 Oct 2019 10:43:35 -0400 Steven Rostedt <[email protected]> wrote:
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
I did a bit of hammering and found two bugs. One I sent a patch to fix (adding a module when tracing is enabled), but the other bug I triggered, I'm too tired to debug right now. But figured I'd mention it anyway.
If you add on the kernel command line:
ftrace=function ftrace_filter=schedule
You will get this (note, I had KASAN enabled, so it showed more info):
[ 1.274356] ftrace: allocating 34274 entries in 134 pages [ 1.320059] Starting tracer 'function' [ 1.323724] ================================================================== [ 1.330798] BUG: KASAN: null-ptr-deref in __get_locked_pte+0x21/0x210 [ 1.337186] Read of size 8 at addr 0000000000000050 by task swapper/0 [ 1.343579] [ 1.345049] CPU: 0 PID: 0 Comm: swapper Not tainted 5.4.0-rc2-test+ #50 [ 1.351613] Hardware name: Hewlett-Packard HP Compaq Pro 6300 SFF/339A, BIOS K01 v03.03 07/14/2016 [ 1.360510] Call Trace: [ 1.362932] dump_stack+0x7c/0xc0 [ 1.366213] ? __get_locked_pte+0x21/0x210 [ 1.370272] ? __get_locked_pte+0x21/0x210 [ 1.374331] __kasan_report.cold.10+0x5/0x3e [ 1.378566] ? __get_locked_pte+0x21/0x210 [ 1.382625] kasan_report+0xe/0x20 [ 1.385993] __get_locked_pte+0x21/0x210 [ 1.389880] ? 0xffffffffa000005c [ 1.393162] __text_poke+0x1ca/0x5b0 [ 1.396705] ? optimize_nops.isra.8+0xd0/0xd0 [ 1.401023] ? insn_get_length+0x4f/0x70 [ 1.404910] ? text_poke_loc_init+0x186/0x220 [ 1.409229] ? text_poke_kgdb+0x10/0x10 [ 1.413031] ? 0xffffffffa0000000 [ 1.416312] text_poke_bp_batch+0xb4/0x1e0 [ 1.420372] ? __text_poke+0x5b0/0x5b0 [ 1.424088] ? do_raw_spin_lock+0x113/0x1d0 [ 1.428233] ? 0xffffffffa000005c [ 1.431515] ? 0xffffffffa0000000 [ 1.434797] text_poke_bp+0x7a/0xa0 [ 1.438253] ? text_poke_queue+0xb0/0xb0 [ 1.442139] ? 0xffffffffa0000000 [ 1.445421] ? 0xffffffffa000005c [ 1.448706] ? find_vmap_area+0x56/0x80 [ 1.452505] arch_ftrace_update_trampoline+0x114/0x380 [ 1.457603] ? ftrace_caller+0x4e/0x4e [ 1.461316] ? 0xffffffffa0000091 [ 1.464599] ? arch_ftrace_update_code+0x10/0x10 [ 1.469179] ? mutex_lock+0x86/0xd0 [ 1.472634] ? __mutex_lock_slowpath+0x10/0x10 [ 1.477039] ? mutex_unlock+0x1d/0x40 [ 1.480668] ? arch_jump_label_transform_queue+0x7a/0x90 [ 1.485937] ? __jump_label_update+0x91/0x140 [ 1.490256] ? mutex_unlock+0x1d/0x40 [ 1.493885] ? tracing_start_sched_switch.cold.0+0x60/0x60 [ 1.499327] __register_ftrace_function+0xaf/0xf0 [ 1.503991] ftrace_startup+0x24/0x130 [ 1.507706] register_ftrace_function+0x2d/0x80 [ 1.512198] function_trace_init+0xc1/0x100 [ 1.516346] tracing_set_tracer+0x1fb/0x3c0 [ 1.520492] ? free_snapshot+0x50/0x50 [ 1.524205] ? __kasan_kmalloc.constprop.6+0xc1/0xd0 [ 1.529131] ? __list_add_valid+0x29/0xa0 [ 1.533106] register_tracer+0x235/0x26c [ 1.536993] early_trace_init+0x29b/0x3a0 [ 1.540967] start_kernel+0x2a3/0x5f7 [ 1.544595] ? mem_encrypt_init+0x6/0x6 [ 1.548396] ? load_ucode_intel_bsp+0x5e/0xa3 [ 1.552715] ? init_intel_microcode+0xc3/0xc3 [ 1.557036] ? load_ucode_bsp+0xcc/0x154 [ 1.560923] secondary_startup_64+0xa4/0xb0 [ 1.565070] ==================================================================
-- Steve
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 4 lines · Steven Rostedt
On Tue, 8 Oct 2019 10:43:35 -0400
Steven Rostedt <[email protected]> wrote:
Quoted 8 lines
Quoted 3 lines
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
I did a bit of hammering and found two bugs. One I sent a patch to fix (adding a module when tracing is enabled), but the other bug I triggered, I'm too tired to debug right now. But figured I'd mention it anyway.
I'm thinking this should fix it... Just not sure this is the right plce, then again, we're doing the same thing in jump_label and static_call, so perhaps we should do it like this.
arch/x86/kernel/alternative.c +6 -1
@@ -1230,10 +1230,15 @@ void text_poke_queue(void *addr, const v* dynamically allocated memory. This function should be used when it is * not possible to allocate memory. */ -void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + } +text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 23 lines · On Thu, 10 Oct 2019 11:20:54 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Steven Rostedt
On Tue, 8 Oct 2019 10:43:35 -0400
Steven Rostedt <[email protected]> wrote:Quoted 8 lines
Quoted 3 lines
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
I did a bit of hammering and found two bugs. One I sent a patch to fix (adding a module when tracing is enabled), but the other bug I triggered, I'm too tired to debug right now. But figured I'd mention it anyway.
I'm thinking this should fix it... Just not sure this is the right plce, then again, we're doing the same thing in
jump_labelandstatic_call, so perhaps we should do it like this.arch/x86/kernel/alternative.c +0
-0@@ -1230,10 +1230,15 @@ void text_poke_queue(void *addr, const v* dynamically allocated memory. This function should be used when it is * not possible to allocate memory.
arch/x86/kernel/alternative.c void text_poke_queue(void *addr, const v · line 1240
Quoted 10 lines
arch/x86/kernel/alternative.c +5
-1*/ -void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + }
We need a new system state. SYSTEM_UP ? (Arg, that name is confusing, SYSTEM_BOOTING_SMP?) Or perhaps just test num_online_cpus()?
if (unlikely(system_state == SYSTEM_BOOTING && num_online_cpus() == 1)
?
Because we can't do the above once we have more than one CPU running.
-- Steve
Quoted 4 lines
arch/x86/kernel/alternative.c +1
-0+text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 40 lines · Steven Rostedt
Quoted 23 lines · On Thu, 10 Oct 2019 11:20:54 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Steven Rostedt
On Tue, 8 Oct 2019 10:43:35 -0400
Steven Rostedt <[email protected]> wrote:Quoted 8 lines
Quoted 3 lines
BTW, I'd really like to take this patch series through my tree. That way I can really hammer it, as well as I have code that will be built on top of it.
I did a bit of hammering and found two bugs. One I sent a patch to fix (adding a module when tracing is enabled), but the other bug I triggered, I'm too tired to debug right now. But figured I'd mention it anyway.
I'm thinking this should fix it... Just not sure this is the right plce, then again, we're doing the same thing in
jump_labelandstatic_call, so perhaps we should do it like this.arch/x86/kernel/alternative.c +0
-0@@ -1230,10 +1230,15 @@ void text_poke_queue(void *addr, const v* dynamically allocated memory. This function should be used when it is * not possible to allocate memory. arch/x86/kernel/alternative.c void text_poke_queue(void *addr, const v · line 1240
Quoted 10 lines
arch/x86/kernel/alternative.c +5
-1*/ -void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + }We need a new system state.
SYSTEM_UP? (Arg, that name is confusing,SYSTEM_BOOTING_SMP?) Or perhaps just testnum_online_cpus()?
Quoted 6 lines
if (unlikely(system_state == SYSTEM_BOOTING && num_online_cpus() == 1)?
Because we can't do the above once we have more than one CPU running.
We loose BOOTING _long_ before we gain SMP.
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 3 lines · On Thu, 10 Oct 2019 16:05:13 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 1 line
Because we can't do the above once we have more than one CPU running.
We loose BOOTING _long_ before we gain SMP.
Ah, yep. But I finally got it working with the following patch:
arch/x86/include/asm/text-patching.h +1 -1
@@ -25,7 +25,7 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,*/ #define POKE_MAX_OPCODE_SIZE 5 -extern void text_poke_early (void *addr,const void *opcode,size_t len); +extern void *text_poke_early (void *addr,const void *opcode,size_t len);/* * Clear and restore the kernel write-protection flag on the local CPU.
arch/x86/kernel/alternative.c +16 -7
@@ -267,7 +267,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)extern struct alt_instr __alt_instructions[], __alt_instructions_end[];extern s32 __smp_locks[], __smp_locks_end[]; -void text_poke_early (void *addr,const void *opcode,size_t len); +void *text_poke_early (void *addr,const void *opcode,size_t len);/* * Are we looking at a near JMP with a 1 or 4-byte displacement.
@@ -756,8 +756,8 @@ void __init alternative_instructions(void)* instructions. And on the local CPU you need to be protected against NMI or * MCE handlers seeing an inconsistent instruction while you patch. */ -void __init_or_module text_poke_early(void *addr, const void *opcode, -size_t len) +void *__init_or_module text_poke_early(void *addr, const void *opcode, +size_t len) {unsigned long flags;
@@ -780,6 +780,7 @@ void __init_or_module text_poke_early(void *addr, const void *opcode,* that causes hangs on some VIA CPUs. */ } +return NULL ; }__ro_after_init struct mm_struct *poking_mm;
@@ -1058,10 +1059,14 @@ static int tp_vec_nr;*/ static void text_poke_bp_batch (struct text_poke_loc *tp,unsigned int nr_entries) { +void *(*poke)(void *addr,const void *opcode,size_t len) = text_poke;unsigned char int3 =INT3_INSN_OPCODE ;unsigned int i;int do_sync; +if (system_state ==SYSTEM_BOOTING ) + poke = text_poke_early; +lockdep_assert_held (&text_mutex); bp_patching.vec = tp;
@@ -1077,7 +1082,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries* First step: add a int3 trap to the address that will be patched. */ for (i =0 ; i < nr_entries; i++) -text_poke (tp[i].addr, &int3,sizeof (int3)); +poke (tp[i].addr, &int3,sizeof (int3));on_each_cpu (do_sync_core,NULL ,1 );
@@ -1086,8 +1091,8 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries*/ for (do_sync =0 , i =0 ; i < nr_entries; i++) {if (tp[i].len -sizeof (int3) >0 ) { -text_poke ((char *)tp[i].addr +sizeof (int3), - (const char *)tp[i].text +sizeof (int3), +poke ((char *)tp[i].addr +sizeof (int3), + (const char *)tp[i].text +sizeof (int3), tp[i].len -sizeof (int3)); do_sync++; }
@@ -1110,7 +1115,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entriesif (tp[i].text[0 ] ==INT3_INSN_OPCODE )continue ; -text_poke (tp[i].addr, tp[i].text,sizeof (int3)); +poke (tp[i].addr, tp[i].text,sizeof (int3)); do_sync++; }
@@ -1234,6 +1239,10 @@ void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulat {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + }text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +13 -3
@@ -308,7 +308,8 @@ union ftrace_op_code_union {#define RET_SIZE 1static unsigned long -create_trampoline (struct ftrace_ops *ops,unsigned int *tramp_size) +create_trampoline (struct ftrace_ops *ops,unsigned int *tramp_size, +unsigned int *pages) {unsigned long start_offset;unsigned long end_offset;
@@ -394,8 +395,11 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)set_vm_flush_reset_perms (trampoline); -set_memory_ro ((unsigned long )trampoline, npages); +/* We can't use text_poke_bp() at start up */ +if (system_state !=SYSTEM_BOOTING ) +set_memory_ro ((unsigned long )trampoline, npages);set_memory_x ((unsigned long )trampoline, npages); + *pages = npages;return (unsigned long )trampoline; fail:tramp_free (trampoline);
@@ -423,7 +427,9 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)ftrace_func_t func;unsigned long offset;unsigned long ip; +unsigned int pages;unsigned int size; +bool set_ro = false;const char *new;if (ops->trampoline) {
@@ -434,7 +440,9 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)if (!(ops->flags &FTRACE_OPS_FL_ALLOC_TRAMP ))return ; }else { - ops->trampoline =create_trampoline (ops, &size); +if (system_state ==SYSTEM_BOOTING ) + set_ro = true; + ops->trampoline =create_trampoline (ops, &size, &pages);if (!ops->trampoline)return ; ops->trampoline_size = size;
@@ -450,6 +458,8 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops) new =ftrace_call_replace (ip, (unsigned long )func);text_poke_bp ((void *)ip, new,MCOUNT_INSN_SIZE ,NULL );mutex_unlock (&text_mutex); +if (set_ro) +set_memory_ro ((unsigned long )ops->trampoline, pages); }/* Return the address of the function the trampoline calls */
Is it really important to use text_poke() on text that is coming live? That is, I really hate the above "set_ro" hack. This is because you moved the ro setting to create_trampoline() and then forcing the text_poke() on text that has just been created. I prefer to just modify it and then setting it to ro before it gets executed. Otherwise we need to do all these dances.
The same is with the module code. My patch was turning text to read-write that is not to be executed yet. Really, what's wrong with that?
-- Steve
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 8 lines · Steven Rostedt
Quoted 3 lines · On Thu, 10 Oct 2019 16:05:13 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 1 line
Because we can't do the above once we have more than one CPU running.
We loose BOOTING _long_ before we gain SMP.
Ah, yep. But I finally got it working with the following patch:
That looks like it can be done simpler; but my head is exploding, I'll have to look at this in the AM.
Quoted 1 line
Is it really important to use
text_poke()on text that is coming live?
What is really important is that we never have memory that is both writable and executable (W^X).
Moving as much poking to before marking it RO (or moving the marking RO later if that is the same thing) is a sane approach.
But once it gains X, we must never again mark it W, without first removing X.
Quoted 5 lines
That is, I really hate the above "
set_ro" hack. This is because you moved the ro setting tocreate_trampoline()and then forcing thetext_poke()on text that has just been created. I prefer to just modify it and then setting it to ro before it gets executed. Otherwise we need to do all these dances.
I thought create_trampoline() finished the whole thing; if it does not, either make create_trampoline() do everything, or add a finish_trampoline() callback to mark it complete.
Quoted 3 lines
The same is with the module code. My patch was turning text to read-write that is not to be executed yet. Really, what's wrong with that?
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Really the best solution is to move all the poking into ftrace_module_init(), before we mark it RO+X. That's what I'm going to do for jump_label and static_call as well, I just need to add that extra notifier callback.
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 9 lines · On Thu, 10 Oct 2019 19:28:19 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 5 lines
That is, I really hate the above "
set_ro" hack. This is because you moved the ro setting tocreate_trampoline()and then forcing thetext_poke()on text that has just been created. I prefer to just modify it and then setting it to ro before it gets executed. Otherwise we need to do all these dances.I thought
create_trampoline()finished the whole thing; if it does not, either makecreate_trampoline()do everything, or add afinish_trampoline()callback to mark it complete.
I'm good with a finish_trampoline(). I can make a patch that does that.
Quoted 8 lines
Quoted 4 lines
The same is with the module code. My patch was turning text to read-write that is not to be executed yet. Really, what's wrong with that?
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Hmm, I wonder if we can work with a way to make this work in the module code as well. Moving the place it sets 'x' and 'ro' around :-/
Quoted 3 lines
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
OK.
Quoted 5 lines
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.
I'll have to think about other ways to handle the other archs with the all or nothing approach. Perhaps we can use my patch as an arch dependent situation, I'll try another approach.
-- Steve
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 9 lines · Steven Rostedt
Quoted 6 lines · On Thu, 10 Oct 2019 19:28:19 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 5 lines
That is, I really hate the above "
set_ro" hack. This is because you moved the ro setting tocreate_trampoline()and then forcing thetext_poke()on text that has just been created. I prefer to just modify it and then setting it to ro before it gets executed. Otherwise we need to do all these dances.
Quoted 5 lines
Quoted 3 lines
I thought
create_trampoline()finished the whole thing; if it does not, either makecreate_trampoline()do everything, or add afinish_trampoline()callback to mark it complete.I'm good with a
finish_trampoline(). I can make a patch that does that.
I found it easier to just make create_trampoline do it all. The below patch seems to cure both issues for me.
---
arch/x86/kernel/alternative.c +11 -1
@@ -1213,6 +1213,11 @@ void text_poke_queue(void *addr, const v {struct text_poke_loc *tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + } +text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];
@@ -1230,10 +1235,15 @@ void text_poke_queue(void *addr, const v* dynamically allocated memory. This function should be used when it is * not possible to allocate memory. */ -void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + } +text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +31 -12
106 lines not on this page · raw message
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
arch/x86/kernel/ftrace.c void arch_ftrace_update_trampoline(struc · line 449
Quoted 2 lines · Peter Zijlstra
arch/x86/kernel/ftrace.c +2
-0+if (!ops->trampoline) { + ops->trampoline =create_trampoline (ops, &size,ftrace_ops_get_func (ops));
And now that I look at what I send, I see we already pass ops, so no need to pass ftrace_ops_get_func().
Let me respin that.
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 8 lines · Peter Zijlstra
Review
Quoted 2 lines · Peter Zijlstra
(quoted diff) +2
-0+ if (!ops->trampoline) { + ops->trampoline = create_trampoline(ops, &size, ftrace_ops_get_func(ops));And now that I look at what I send, I see we already pass ops, so no need to pass
ftrace_ops_get_func().Let me respin that.
---
arch/x86/kernel/alternative.c +12 -2
@@ -1209,10 +1209,15 @@ void text_poke_finish(void)text_poke_flush (NULL ); } -void text_poke_queue (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_queue (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc *tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + } +text_poke_flush (addr); tp = &tp_vec[tp_vec_nr++];
@@ -1230,10 +1235,15 @@ void text_poke_queue(void *addr, const v* dynamically allocated memory. This function should be used when it is * not possible to allocate memory. */ -void text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) +void __ref text_poke_bp (void *addr,const void *opcode,size_t len,const void *emulate) {struct text_poke_loc tp; +if (unlikely (system_state ==SYSTEM_BOOTING )) { +text_poke_early (addr, opcode, len); +return ; + } +text_poke_loc_init (&tp, addr, opcode, len, emulate);text_poke_bp_batch (&tp,1 ); }
arch/x86/kernel/ftrace.c +30 -10
@@ -34,6 +34,8 @@#ifdef CONFIG_DYNAMIC_FTRACE +static int ftrace_poke_late =0 ; +int ftrace_arch_code_modify_prepare (void ) __acquires(&text_mutex) {
@@ -43,12 +45,15 @@ int ftrace_arch_code_modify_prepare(void* ftrace has it set to "read/write". */ mutex_lock (&text_mutex); + ftrace_poke_late =1 ;return 0 ; }int ftrace_arch_code_modify_post_process (void ) __releases(&text_mutex) { +text_poke_finish (); + ftrace_poke_late =0 ;mutex_unlock (&text_mutex);return 0 ; }
@@ -116,7 +121,10 @@ ftrace_modify_code_direct(unsigned longreturn ret;/* replace the text with the new text */ -text_poke_early ((void *)ip, new_code,MCOUNT_INSN_SIZE ); +if (ftrace_poke_late) +text_poke_queue ((void *)ip, new_code,MCOUNT_INSN_SIZE ,NULL ); +else +text_poke_early ((void *)ip, new_code,MCOUNT_INSN_SIZE );return 0 ; }
@@ -313,6 +321,7 @@ create_trampoline(struct ftrace_ops *opsunsigned long start_offset;unsigned long end_offset;unsigned long op_offset; +unsigned long call_offset;unsigned long offset;unsigned long npages;unsigned long size;
@@ -329,10 +338,12 @@ create_trampoline(struct ftrace_ops *ops start_offset = (unsigned long )ftrace_regs_caller; end_offset = (unsigned long )ftrace_regs_caller_end; op_offset = (unsigned long )ftrace_regs_caller_op_ptr; + call_offset = (unsigned long )ftrace_regs_call; }else { start_offset = (unsigned long )ftrace_caller; end_offset = (unsigned long )ftrace_epilogue; op_offset = (unsigned long )ftrace_caller_op_ptr; + call_offset = (unsigned long )ftrace_call; } size = end_offset - start_offset;
@@ -389,6 +400,15 @@ create_trampoline(struct ftrace_ops *ops/* put in the new offset to the ftrace_ops */ memcpy (trampoline + op_offset, &op_ptr,OP_REF_SIZE ); +/* put in the call to the function */ +mutex_lock (&text_mutex); + call_offset -= start_offset; +memcpy (trampoline + call_offset, +text_gen_insn (CALL_INSN_OPCODE , + trampoline + call_offset, +ftrace_ops_get_func (ops)),CALL_INSN_SIZE ); +mutex_unlock (&text_mutex); +/* ALLOC_TRAMP flags lets us know we created it */ ops->flags |=FTRACE_OPS_FL_ALLOC_TRAMP ;
@@ -426,23 +446,23 @@ void arch_ftrace_update_trampoline(strucunsigned int size;const char *new; -if (ops->trampoline) { -/* -* The ftrace_ops caller may set up its own trampoline. -* In such a case, this code must not modify it. -*/ -if (!(ops->flags &FTRACE_OPS_FL_ALLOC_TRAMP )) -return ; - }else { +if (!ops->trampoline) { ops->trampoline =create_trampoline (ops, &size);if (!ops->trampoline)return ; ops->trampoline_size = size; +return ; } +/* +* The ftrace_ops caller may set up its own trampoline. +* In such a case, this code must not modify it. +*/ +if (!(ops->flags &FTRACE_OPS_FL_ALLOC_TRAMP )) +return ; + offset =calc_trampoline_call_offset (ops->flags &FTRACE_OPS_FL_SAVE_REGS ); ip = ops->trampoline + offset; - func =ftrace_ops_get_func (ops);mutex_lock (&text_mutex);
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter ZijlstraQuoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.
OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
That is, I'm struggling to explain why we cannot flip prepare_coming_module() and complete_formation().
Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 11 lines · On Fri, 11 Oct 2019 14:59:03 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
Quoted 5 lines
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
For someone that doesn't use modules, you are making me very nervous with all the changes you are making to the module code! ;-)
-- Steve
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 14 lines · Steven Rostedt
Quoted 11 lines · On Fri, 11 Oct 2019 14:59:03 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
Quoted 8 lines
Quoted 5 lines
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
For someone that doesn't use modules, you are making me very nervous with all the changes you are making to the module code! ;-)
Hey, today I build a kernel with modules just for you :-)
And whatever it takes right, I just want to clean this crap up.
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter Zijlstra+++ Peter Zijlstra [11/10/19 14:59 +0200]:
Quoted 11 lines · Peter Zijlstra
Quoted 5 lines
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
Quoted 5 lines
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
I'm having trouble visualizing what changes you're planning on making. I get that you want to squash ftrace_module_enable() into ftrace_module_init(), before module_enable_ro(). I'm fine with that as long as the races Steven described are addressed for affected arches. And livepatch should be OK as long as klp_module_coming() is always called *after* ftrace_module_enable(). But are you moving klp_module_coming() before the module_enable_* calls as well? And if so, why?
Quoted 6 lines
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Not sure if relevant, but just thought I'd clarify: IIRC, klp_module_coming() is not poking the coming module, but it calls module_enable_ro() on itself (the livepatch module) so it can apply relocations and such on the new code, which lives inside the livepatch module, and it needs to possibly do this numerous times over the lifetime of the patch module for any coming module it is responsible for patching (i.e., call module_enable_ro() on the patch module, not necessarily the coming module). So I am not be sure why klp_module_coming() should be moved before complete_formation(). I hope I'm remembering the details correctly, livepatch folks feel free to chime in if I'm incorrect here.
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 1 line · Jessica Yu
I'm having trouble visualizing what changes you're planning on making.
I want all the text poking (jump_label, ftrace, klp whatever) to happen _before_ we do the protection changes.
I also want to avoid flipping the protection state around unnecessarily, because that clearly is just wasted work.
Quoted 3 lines
I get that you want to squash
ftrace_module_enable()intoftrace_module_init(), beforemodule_enable_ro(). I'm fine with that as long as the races Steven described are addressed for affected arches.
Right, the problem is set_all_modules_text_*(), that relies on COMING having made the protection changes.
After the x86 changes, there's only 2 more architectures that use that function. I'll work on getting those converted and then we can delete that function and worry no more about it.
Quoted 4 lines
And livepatch should be OK as long as
klp_module_coming()is always called *after*ftrace_module_enable(). But are you movingklp_module_coming()before the module_enable_* calls as well? And if so, why?
I wanted to move the COMING notifier callback before the protection changes, because that is the easiest way to convert jump_label and static_call and AFAICT nothing really cared aside from ftrace.
The alternative is introducing additional module states, which just adds complexity we don't really need if we can just flip things around a little.
Quoted 7 lines
Quoted 6 lines
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Quoted 11 lines
Not sure if relevant, but just thought I'd clarify: IIRC,
klp_module_coming()is not poking the coming module, but it callsmodule_enable_ro()on itself (the livepatch module) so it can apply relocations and such on the new code, which lives inside the livepatch module, and it needs to possibly do this numerous times over the lifetime of the patch module for any coming module it is responsible for patching (i.e., callmodule_enable_ro()on the patch module, not necessarily the coming module). So I am not be sure whyklp_module_coming()should be moved beforecomplete_formation(). I hope I'm remembering the details correctly, livepatch folks feel free to chime in if I'm incorrect here.
You mean it does module_disable_ro() ? That would be broken and it needs to be fixed. Can some livepatch person explain what it does and why?
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Peter Zijlstra
Quoted 14 lines · Jessica Yu
Quoted 2 lines
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Not sure if relevant, but just thought I'd clarify: IIRC,
klp_module_coming()is not poking the coming module, but it callsmodule_enable_ro()on itself (the livepatch module) so it can apply relocations and such on the new code, which lives inside the livepatch module, and it needs to possibly do this numerous times over the lifetime of the patch module for any coming module it is responsible for patching (i.e., callmodule_enable_ro()on the patch module, not necessarily the coming module). So I am not be sure whyklp_module_coming()should be moved beforecomplete_formation(). I hope I'm remembering the details correctly, livepatch folks feel free to chime in if I'm incorrect here.
Quoted 2 lines
You mean it does
module_disable_ro()? That would be broken and it needs to be fixed. Can some livepatch person explain what it does and why?
mbenes confirmed; what would be needed is for the live-patch module to have all module dependent parts to be in their own section and have the sections be page-aligned. Then we can do the protection on sections instead of on the whole module.
Damn, and I thought I was so close to getting W^X sorted :/
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter Zijlstra[ live-patching ML CCed ]
Quoted 21 lines · Peter Zijlstra
Quoted 7 lines · Jessica Yu
Quoted 6 lines
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Quoted 11 lines
Not sure if relevant, but just thought I'd clarify: IIRC,
klp_module_coming()is not poking the coming module, but it callsmodule_enable_ro()on itself (the livepatch module) so it can apply relocations and such on the new code, which lives inside the livepatch module, and it needs to possibly do this numerous times over the lifetime of the patch module for any coming module it is responsible for patching (i.e., callmodule_enable_ro()on the patch module, not necessarily the coming module). So I am not be sure whyklp_module_coming()should be moved beforecomplete_formation(). I hope I'm remembering the details correctly, livepatch folks feel free to chime in if I'm incorrect here.
Quoted 2 lines
You mean it does
module_disable_ro()? That would be broken and it needs to be fixed. Can some livepatch person explain what it does and why?
Yes, it does. klp_module_coming() calls module_disable_ro() on all patching modules which patch the coming module in order to call apply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.
Miroslav
Joe Lawrence
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 4 lines · Miroslav Benes
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.
FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
-- Joe
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
+++ Joe Lawrence [15/10/19 11:06 -0400]:
Quoted 7 lines · Miroslav Benes
Quoted 4 lines
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
I wasn't at Plumbers sadly, was this idea documented/talked about in detail somewhere? (recording, slides, etherpad, etc?). What do you mean by updates are kept on disk? Maybe someone can explain it more in detail? :)
Joe Lawrence
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 13 lines · Jessica Yu
+++ Joe Lawrence [15/10/19 11:06 -0400]:
Quoted 11 lines · Miroslav Benes
Quoted 4 lines
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Quoted 5 lines
I wasn't at Plumbers sadly, was this idea documented/talked about in detail somewhere? (recording, slides, etherpad, etc?). What do you mean by updates are kept on disk? Maybe someone can explain it more in detail? :)
Livepatching folks -- I don't have the LPC summary link (etherpad?) that Jiri put together. Does someone have that handy for Jessica?
Thanks,
-- Joe
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 3 lines · On Tue, 15 Oct 2019 18:17:43 -0400 Joe Lawrence <[email protected]> wrote:
Livepatching folks -- I don't have the LPC summary link (etherpad?) that Jiri put together. Does someone have that handy for Jessica?
Yes, and I'll be posting this on the LPC web site soon. But here's the write up that Jiri sent me (with the etherpad link at the end):
Live patching miniconference covered 8 topics overall.
(1) First session, titled "What happened in kernel live patching over the last year" was led by Miroslav Benes. It was quite a natural followup to where we ended at the LPC 2018 miniconf, summarizing which of the points that have been agreed on back then have already been fully implemented, where obstacles have been enounctered, etc.
The most prominent feature that has been merged during past year was "atomic replace", which allows for easier stacking of patches. This is especially useful for distros, as it naturally aligns with the way patches are being distributed by them.
Another big step forward since LPC 2018 miniconf was addition of livepatching selftests, which already tremendously helped in various cases, as it e.g. helped to track down quite a few issues during development of reliable stacktraces on s390. Proposal has been made that all major KLP features in the future should be accompanied by accompanying selftest, which the audience agreed on.
One of the last year's discussion topics / pain points were GCC optimizations which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
(2) Second session, titled "Rethinking late module patching" was led by Miroslav Benes again.
The problem statement is: in case when there is a patch loaded for module that is yet to be loaded, it has to be patched before it starts executing. The current solution relies on hooks in the module loader, and module is patched when its being linked. It gets a bit nasty with the arch-specifics of the module loader handling all the relocations, patching of alternatives, etc. One of the issues is that all the paravirt / jump label patching has to be done after relocations are resolved, this is getting a bit fragile and not well maintainable.
Miroslav sketched out the possible solutions:
- livepatch would immediately load all the modules for which it has patch via dependency; half-loading modules (not promoting to final LIVE state)
- splitting the currently one big monolithic livepatch to a per-object structure; might cause issues with consistency model
- "blue sky" idea from Joe Lawrence: livepatch loaded modules, binary-patch .ko on disk, blacklist vulnerable version
Miroslav proposed to actually stick to the current solution, and improve selftests coverage for all the considered-fragile arch-specific module linking code hooks. The discussion then mostly focused, based on proposals from several attendees (most prominently Steven Rostedt and Amit Shah), on expanding on the "blue sky" idea.
The final proposal converged to having a separate .ko for livepatches that's installed on the disk along with the module. This addresses the module signature issue (as signature does not actually change), as well as module removal case (the case where a module was previously loaded while a livepatch is applied, and then later unloaded and reloaded). The slight downside is that this will require changes to the module loader to also look for livepatches when loading a module. When unloading the module, the livepatch module will also need to be unloaded. Steven approved of this approach over his previous suggestion.
(3) Third session, titled "Source-based livepatch creation tooling", was led by Nicolai Stange.
The primary objective of the session was basing on the source-based creation of livepatches, while avoiding the tedious (and error-prone task) of copying a lot of kernel code around (from the source tree to the livepatch). Nicolai spent par of last year writing a klp-ccp (KLP Copy and Paste) utility, which automates a big chunk of the process.
Nicolai then presented the still open issues with the tool and with the process around it, most promonent ones being:
- obtaining original GCC commandline that was used to build the original kernel
- externalizability of static functions; we need to know whether GCC emitted static function into the patched object
Miroslav proposed to extend existing IPA dumping capabiity of GCC to emit also the information about dead code elimination; DWARF information is guaranteed not to be reliable when it comes to IPA optimizations.
(4) Fourth session, titled "Objtool on power -- update", was led by Kamalesh Babulal.
Kamalesh reported that as a followup to last year's miniconference, the objtool support for powerpc actually came to life. It hasn't yet been posted upstream, but is currently available on github [1].
Kamalesh further reported, that decoder has basic functionality (stack operations + validation, branches, unreachable code, switch table (through gcc plugin), conditional branches, prologue sequences). It turns out that stack validation on powerpc is easier than on x86, as the ABI is much more strict there; which leaves the validation phase to mostly focus on hand-written assembly.
The next steps are basing on arm64 objtool code which already abstracted out the arch-specific bits, and further optimizations can be stacked on top of that (switch table detection, more testing, different gcc versions).
(5) Fifth session, titled "Do we need a Livepatch Developers Guide?", was led by Joe Lawrence.
Joe postulated, that Current in-kernel documentation provides very good documentation for individual features the infrastructure provides to the livepatch author, but Joe further suggested to also include something along the lines of what they currently have for kpatch, which takes a more general look from the point of view of livepatch developer.
Proposals that have been brought up for discussion:
- FAQ
- collecting already existing CVE fixes and ammend them with a lot of commentary
- creating a livepatch blog on people.kernel.org
Mark Brown asked for documenting what architectures need to implement in order to support livepatching.
Amit Shah asked if the 'kpatch' and 'kpatch-build' script/program be renamed to 'livepatch'-friendly names so that kernel sources can also reference them for the user docs part of it.
Both Mark's and Amit's remarks have been considered very valid and useful, and agreement was reached that they will be taken care of.
(6) Sixth session, titled "API for state changes made by callbacks", was led by Petr Mladek.
Petr described his proposal for API for changing, updating and disabling patches (by callbacks). Example where this was needed: L1TF fix, which needed to change PTE semantics (particular bits). This can't be done before all the code understands this new PTE format/semantics. Therefore pre-patch and post-patch callbacks had to do the actual modifications to all the existing PTEs. What is also currently missing is tracking compatibilities / dependencies between individual livepatches.
Petr's proposal (v2) is already on ML.
struct klp_state is being introduced which tracks the actual states of the patch. klp_is_patch_compatible() checks the compatibility of the current states to the states that the new livepatch is going to bring.
No principal issues / objections have been raised, and it's appreciated by the patch author(s), so v3 will be submitted and pre-merge bikeshedding will start.
(7) Seventh session, titled "klp-convert and livepatch relocations", was led by Joe Lawrence.
Joe started the session with problem statement: accessing non exported / static symbols from inside the patch module. One possible workardound is manually via kallsyms. Second workaround is klp-convert, which actually creates proper relocations inside the livepatch module from the symbol database during the final .ko link.
Currently module loader looks for special livepatch relocations and resolves those during runtime; kernel support for these relocations have so far been added for x86 only. Special livepatch relocations are supported and processed also on other architectures. Special quirks/sections are not yet supported. Plus klp-convert would still be needed even with late module patching update. vmlinux or modules could have ambiguous static symbols.
It turns out that the features / bugs below have to be resolved before we can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and have a good regression tests in place
- one day we might (or might not) add support for out-of-tree modules which need klp-convert
- BFD bug 24456 (multiple relocations to the same .text section)
(8) Eight sesstion, titled "Making livepatching infrastructure better", was led by Kamalesh Babulal.
The primary goal of the discussion as presented by Kamalesh was simple: how to improve our testing coverage. Currently we have sample modules + kselftests. We seem to be currently missing specific unit cases and tests for corner cases. What Kamalesh would also like to see would be more stress testing oriented tests for the infrastructure. We should make sure that projects like kernelCI are running with CONFIG_LIVEPATCH=y.
Another thing Kamalesh currently sees as missing are failure test cases too. It should be checked with sosreport and supportconfig guys whether those diagnostic tools do provide necessary coverage of (at least) livepatching sysfs state. This is especially a task for distro people to figure out. Nicolai proposed as one of the testcases identity patching, as that should reveal issues directly in the infrastructure.
(9) Last, ninth session, titled "Open sourcing live patching services", was led by Alice Ferrazzi.
This session followed up on previous suggestion of having public repository for livepatches against LTS kernel.
Alice reported on improviement of elivepatch since last year as having moved everything to docker.
Alice proposed to more share livepatch sources; SUSE does publish those [2][3], but it's important to mention that livepatches are very closely tied to particular kernel version.
[1] https://github.com/kamalesh-babulal/linux/tree/objtool-v1
[2] On https://kernel.suse.com/
[3] Example source-based SUSE's livepatch is at https://kernel.suse.com/cgit/kernel-livepatch/tree/uname_patch/kgr_patch_uname.c?id=d4e00de0b0a3f858fec4e83640f12e1f17298667
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
Fortunately, the impact is negligible.
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter ZijlstraQuoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for CONFIG_LIVEPATCH, since kpatch-build doesn't need it.
Signature · 1 line
Josh
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 10 lines · Josh Poimboeuf
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
Quoted 7 lines
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for
CONFIG_LIVEPATCH, since kpatch-build doesn't need it.
I also just discovered that -flive-patching is responsible for all those "unreachable instruction" objtool warnings which Randy has been dutifully bugging me about over the last several months. For some reason it subtly breaks GCC implicit noreturn detection for local functions.
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
If there aren't any objections I'll be posting a patch soon to revert.
Signature · 1 line
Josh
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 19 lines · Josh Poimboeuf
Quoted 10 lines · Josh Poimboeuf
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
Quoted 7 lines
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for
CONFIG_LIVEPATCH, since kpatch-build doesn't need it.
Quoted 5 lines
I also just discovered that -flive-patching is responsible for all those "unreachable instruction" objtool warnings which Randy has been dutifully bugging me about over the last several months. For some reason it subtly breaks GCC implicit noreturn detection for local functions.
Ugh, that is unfortunate. Have you reported it?
Quoted 2 lines
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
Can you explain this? The option makes GCC to avoid optimizations which are difficult to detect and would make live patching unsafe. I consider it useful as it is, so if you shared the other downsides and what you meant by real upstream code, we could discuss it.
Quoted 1 line
If there aren't any objections I'll be posting a patch soon to revert.
I think it would be a setback.
Regards
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 21 lines · Miroslav Benes
Quoted 19 lines · Josh Poimboeuf
Quoted 17 lines · Josh Poimboeuf
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for
CONFIG_LIVEPATCH, since kpatch-build doesn't need it.
Quoted 7 lines
Quoted 5 lines
I also just discovered that -flive-patching is responsible for all those "unreachable instruction" objtool warnings which Randy has been dutifully bugging me about over the last several months. For some reason it subtly breaks GCC implicit noreturn detection for local functions.
Ugh, that is unfortunate. Have you reported it?
Not yet (but I plan to).
Quoted 7 lines
Quoted 2 lines
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
Can you explain this? The option makes GCC to avoid optimizations which are difficult to detect and would make live patching unsafe. I consider it useful as it is, so if you shared the other downsides and what you meant by real upstream code, we could discuss it.
Only SLES needs it right? Why inflict it on other livepatch users? By "real upstream code" I mean there's no (documented) way to create live patches using the method which relies on this flag. So I don't see any upstream benefits for having it enabled.
Signature · 1 line
Josh
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 9 lines
Quoted 8 lines
Quoted 3 lines
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
Can you explain this? The option makes GCC to avoid optimizations which are difficult to detect and would make live patching unsafe. I consider it useful as it is, so if you shared the other downsides and what you meant by real upstream code, we could discuss it.
Quoted 4 lines
Only SLES needs it right? Why inflict it on other livepatch users? By "real upstream code" I mean there's no (documented) way to create live patches using the method which relies on this flag. So I don't see any upstream benefits for having it enabled.
I'd put it differently. SLES and upstream need it, RHEL does not need it. Or anyone using kpatch-build. It is perfectly fine to prepare live patches just from the source code using upstream live patching infrastructure. After all, SLES is nothing else than upstream here. We were creating live patches manually for quite a long time and only recently we have been using Nicolai's klp-ccp automation (https://github.com/SUSE/klp-ccp).
So, everyone using upstream directly relies on the flag, which seems to be a clear benefit to me. Reverting the patch would be a step back.
Also I think we're moving in the right direction to make the life of upstream user easier with a proposal of klp-ccp and Petr's patch set to split live patch modules. It is a path from inconvenient to comfortable and not from impossible to possible.
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 9 lines · Miroslav Benes
Quoted 9 lines
Quoted 8 lines
Quoted 3 lines
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
Can you explain this? The option makes GCC to avoid optimizations which are difficult to detect and would make live patching unsafe. I consider it useful as it is, so if you shared the other downsides and what you meant by real upstream code, we could discuss it.
Quoted 7 lines
Quoted 4 lines
Only SLES needs it right? Why inflict it on other livepatch users? By "real upstream code" I mean there's no (documented) way to create live patches using the method which relies on this flag. So I don't see any upstream benefits for having it enabled.
I'd put it differently. SLES and upstream need it, RHEL does not need it. Or anyone using kpatch-build.
I'm confused about why you think upstream needs it.
Is all the tooling available somewhere? Is there documentation available which describes how to build patches using that method from start to finish? Are there actual users other than SUSE?
BTW, kpatch-build has a *lot* of users other than RHEL. All its tooling and documentation are available on Github.
Quoted 2 lines
It is perfectly fine to prepare live patches just from the source code using upstream live patching infrastructure.
Do you mean the dangerous method used by the livepatch sample code which completely ignores interprocedural optimizations? I wouldn't call that perfectly fine.
Quoted 6 lines
After all, SLES is nothing else than upstream here. We were creating live patches manually for quite a long time and only recently we have been using Nicolai's klp-ccp automation (https://github.com/SUSE/klp-ccp).
So, everyone using upstream directly relies on the flag, which seems to be a clear benefit to me. Reverting the patch would be a step back.
Who exactly is "everyone using upstream"?
From what I can tell, kpatch-build is the only known way (to those outside of SUSE) to make safe patches for an upstream kernel. And it doesn't need this flag and the problems associated with it: performance, LTO incompatibility, clang incompatibility (I think?), the GCC dead code issue.
Signature · 1 line
Josh
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 41 lines · Josh Poimboeuf
Quoted 9 lines · Miroslav Benes
Quoted 9 lines
Quoted 8 lines
Quoted 3 lines
At this point, I only see downsides of -flive-patching, at least until we actually have real upstream code which needs it.
Can you explain this? The option makes GCC to avoid optimizations which are difficult to detect and would make live patching unsafe. I consider it useful as it is, so if you shared the other downsides and what you meant by real upstream code, we could discuss it.
Quoted 7 lines
Quoted 4 lines
Only SLES needs it right? Why inflict it on other livepatch users? By "real upstream code" I mean there's no (documented) way to create live patches using the method which relies on this flag. So I don't see any upstream benefits for having it enabled.
I'd put it differently. SLES and upstream need it, RHEL does not need it. Or anyone using kpatch-build.
I'm confused about why you think upstream needs it.
Is all the tooling available somewhere? Is there documentation available which describes how to build patches using that method from start to finish? Are there actual users other than SUSE?
BTW, kpatch-build has a *lot* of users other than RHEL. All its tooling and documentation are available on Github.
Quoted 2 lines
It is perfectly fine to prepare live patches just from the source code using upstream live patching infrastructure.
Do you mean the dangerous method used by the livepatch sample code which completely ignores interprocedural optimizations? I wouldn't call that perfectly fine.
Quoted 6 lines
After all, SLES is nothing else than upstream here. We were creating live patches manually for quite a long time and only recently we have been using Nicolai's klp-ccp automation (https://github.com/SUSE/klp-ccp).
So, everyone using upstream directly relies on the flag, which seems to be a clear benefit to me. Reverting the patch would be a step back.
Quoted 7 lines
Who exactly is "everyone using upstream"?
Quoted 1 line
From what I can tell, kpatch-build is the only known way (to those
outside of SUSE) to make safe patches for an upstream kernel. And it doesn't need this flag and the problems associated with it: performance, LTO incompatibility, clang incompatibility (I think?), the GCC dead code issue.
I don't think we have something special at SUSE not generally available...
...and I don't think it is really important to discuss that and replying to the above, because there is a legitimate use case which relies on the flag. We decided to support different use cases right at the beginning.
I understand it currently complicates things for objtool, but objtool is sensitive to GCC code generation by definition. "Issues" appear with every new GCC version. I see no difference here and luckily it is not so difficult to fix it.
I am happy to help with acting on those objtool warning reports you mentioned in the other email. Just Cc me where appropriate. We will take a look.
Regards
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 6 lines · Miroslav Benes
I don't think we have something special at SUSE not generally available...
...and I don't think it is really important to discuss that and replying to the above, because there is a legitimate use case which relies on the flag. We decided to support different use cases right at the beginning.
Quoted 8 lines
I understand it currently complicates things for objtool, but objtool is sensitive to GCC code generation by definition. "Issues" appear with every new GCC version. I see no difference here and luckily it is not so difficult to fix it.
I am happy to help with acting on those objtool warning reports you mentioned in the other email. Just Cc me where appropriate. We will take a look.
As I said, the objtool warnings aren't even the main issue.
There are N users[*] of CONFIG_LIVEPATCH, where N is perhaps dozens. For N-1 users, they have to suffer ALL the drawbacks, with NONE of the benefits.
And, even if they wanted those benefits, they have no idea how to get them because the patch creation process isn't documented.
And, there's no direct upstream usage of the flag, i.e. the only user does so in a distro which can easily modify KCFLAGS in the spec file.
As best as I can tell, these are facts, which you seem to keep glossing over. Did I get any of the facts wrong?
[*] The term 'user' describes the creator/distributor of the
live patches.
Signature · 1 line
Josh
Petr Mladek
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 12 lines · Josh Poimboeuf
Quoted 11 lines · Miroslav Benes
I don't think we have something special at SUSE not generally available...
...and I don't think it is really important to discuss that and replying to the above, because there is a legitimate use case which relies on the flag. We decided to support different use cases right at the beginning.
I understand it currently complicates things for objtool, but objtool is sensitive to GCC code generation by definition. "Issues" appear with every new GCC version. I see no difference here and luckily it is not so difficult to fix it.
Quoted 5 lines
Quoted 3 lines
I am happy to help with acting on those objtool warning reports you mentioned in the other email. Just Cc me where appropriate. We will take a look.
As I said, the objtool warnings aren't even the main issue.
Great.
Anyway, I think that we might make your life easier with using the proposed -Wsuggest-attribute=noreturn.
Also it might be possible to create the list of global noreturn functions using some gcc tool. Similar way that we get the list of functions that need to be livepatched explicitly because of the problematic optimizations.
It sounds like a win-win approach.
Quoted 3 lines
There are N users[*] of
CONFIG_LIVEPATCH, where N is perhaps dozens. For N-1 users, they have to suffer ALL the drawbacks, with NONE of the benefits.
You wrote in the other mail:
Quoted 2 lines
The problems associated with it: performance, LTO incompatibility, clang incompatibility (I think?), the GCC dead code issue.
SUSE performance team did extensive testing and did not found any real performance issues. It was discussed when the option was enabled upstream.
Are the other problems affecting real life usage, please? Could you be more specific about them, please?
Quoted 2 lines
And, even if they wanted those benefits, they have no idea how to get them because the patch creation process isn't documented.
I do not understand this. All the sample modules and selftests are using source based livepatches. It is actually the only somehow documented way. Sure, the documentation might get improved. Patches are welcome.
The option is not currently needed by the selftests only because there is no selftest for this type of problems. But the problems are real. They would actually deserve selftests. Again, patches are welcome.
My understanding is that the source based livepatches is the future. N-1 users are just waiting until the 1 user develops more helper tools for this. I would really like to hear about some serious problems before we do this step back in upstream.
Best Regards,
Petr
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Petr Mladek
Quoted 16 lines · Josh Poimboeuf
Quoted 14 lines · Miroslav Benes
I don't think we have something special at SUSE not generally available...
...and I don't think it is really important to discuss that and replying to the above, because there is a legitimate use case which relies on the flag. We decided to support different use cases right at the beginning.
I understand it currently complicates things for objtool, but objtool is sensitive to GCC code generation by definition. "Issues" appear with every new GCC version. I see no difference here and luckily it is not so difficult to fix it.
I am happy to help with acting on those objtool warning reports you mentioned in the other email. Just Cc me where appropriate. We will take a look.
Quoted 6 lines
Quoted 1 line
As I said, the objtool warnings aren't even the main issue.
Great.
Anyway, I think that we might make your life easier with using the proposed -Wsuggest-attribute=noreturn.
Maybe. Though if I understand correctly, this doesn't help for any of the new warnings because they're for static functions, and this only warns about global functions.
Quoted 6 lines
Also it might be possible to create the list of global noreturn functions using some gcc tool. Similar way that we get the list of functions that need to be livepatched explicitly because of the problematic optimizations.
It sounds like a win-win approach.
I don't quite get how that could be done in an automated way, but ideas about how to implement it would certainly be welcome.
Quoted 9 lines
Quoted 3 lines
There are N users[*] of
CONFIG_LIVEPATCH, where N is perhaps dozens. For N-1 users, they have to suffer ALL the drawbacks, with NONE of the benefits.You wrote in the other mail:
Quoted 2 lines
The problems associated with it: performance, LTO incompatibility, clang incompatibility (I think?), the GCC dead code issue.
Quoted 6 lines
SUSE performance team did extensive testing and did not found any real performance issues. It was discussed when the option was enabled upstream.
Are the other problems affecting real life usage, please? Could you be more specific about them, please?
The original commit mentioned 1-3% scheduler degradation. And I'd expect things to worsen over time as interprocedural optimizations improve.
Also, LTO is coming whether we like it or not. As is Clang. Those are real-world things which will need to work with livepatching sooner or later.
Quoted 5 lines
Quoted 2 lines
And, even if they wanted those benefits, they have no idea how to get them because the patch creation process isn't documented.
I do not understand this. All the sample modules and selftests are using source based livepatches.
We're talking in circles. Have you read the thread?
The samples are a (dangerous) joke. With or without -flive-patching.
Quoted 2 lines
It is actually the only somehow documented way. Sure, the documentation might get improved. Patches are welcome.
Are you suggesting for *me* to send documentation for how *you* build patches?
Quoted 5 lines
The option is not currently needed by the selftests only because there is no selftest for this type of problems. But the problems are real. They would actually deserve selftests. Again, patches are welcome.
My understanding is that the source based livepatches is the future.
I think that still remains to be seen.
Quoted 2 lines
N-1 users are just waiting until the 1 user develops more helper tools for this.
No. N-1 users have no idea how to make (safe) source-based patches in the first place. And if *you* don't need the tools, why would anyone else? Why not document the process and encourage the existence of other users so they can get involved and help with the tooling?
Quoted 2 lines
I would really like to hear about some serious problems before we do this step back in upstream.
Sometimes you need to take 1 step back before you can take 2 steps forward. I regret ACKing the original patch. It was too early.
Signature · 1 line
Josh
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 6 lines · Josh Poimboeuf
Quoted 2 lines
Anyway, I think that we might make your life easier with using the proposed -Wsuggest-attribute=noreturn.
Maybe. Though if I understand correctly, this doesn't help for any of the new warnings because they're for static functions, and this only warns about global functions.
Could you please provide a pointer where those have been reported/analyzed?
For the cases I've seen so far, it has always been gcc deciding under certain circumstances not to propagate __attribute__((__noreturn__)) from callee to caller even in the cases when caller unconditionally called callee.
AFAIU, the behavior is (and always will) be dependent on the state of gcc optimizations, and therefore I don't see any other way than adding __noreturn anotation transitively everywhere in order to silence objtool.
So those cases have to be fixed anyway.
What are the other cases please? Either I have completely missed those, or they haven't been mentioned in this thread.
Thanks,
Signature · 2 lines
Jiri Kosina SUSE Labs
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Jiri Kosina
Quoted 6 lines · Josh Poimboeuf
Quoted 2 lines
Anyway, I think that we might make your life easier with using the proposed -Wsuggest-attribute=noreturn.
Maybe. Though if I understand correctly, this doesn't help for any of the new warnings because they're for static functions, and this only warns about global functions.
Could you please provide a pointer where those have been reported/analyzed?
For the cases I've seen so far, it has always been gcc deciding under certain circumstances not to propagate __attribute__((__noreturn__)) from callee to caller even in the cases when caller unconditionally called callee.
Quoted 8 lines
AFAIU, the behavior is (and always will) be dependent on the state of gcc optimizations, and therefore I don't see any other way than adding
__noreturnanotation transitively everywhere in order to silence objtool.So those cases have to be fixed anyway.
What are the other cases please? Either I have completely missed those, or they haven't been mentioned in this thread.
For example, see:
Many of those callees are static noreturns, for which we've *never* needed annotations. Disabling -fipa-pure-const has apparently changed that.
-Wsuggest-attribute=noreturn doesn't seem to suggest annotations for static functions, probably because most reasonable setups use -O2 which allows GCC to detect them.
Signature · 1 line
Josh
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 14 lines · Josh Poimboeuf
Quoted 5 lines
For the cases I've seen so far, it has always been gcc deciding under certain circumstances not to propagate __attribute__((__noreturn__)) from callee to caller even in the cases when caller unconditionally called callee.
Quoted 8 lines
AFAIU, the behavior is (and always will) be dependent on the state of gcc optimizations, and therefore I don't see any other way than adding
__noreturnanotation transitively everywhere in order to silence objtool.So those cases have to be fixed anyway.
What are the other cases please? Either I have completely missed those, or they haven't been mentioned in this thread.
Quoted 7 lines
For example, see:
Many of those callees are static noreturns, for which we've *never* needed annotations. Disabling -fipa-pure-const has apparently changed that.
For some reason I thought you were talking about static inlines, sorry for the noise.
Yeah, so I agree with you -- whether we need those anotations depends on compiler implementation of optimizations, and most importantly on (the current state of) internal implementation of specific optimizations in gcc.
Leaving live patching completely aside for the sake of this discussion for now -- I believe we either fully rely on gcc to propagate the 'noreturn' propery throughout the callstack upward, or we don't.
If we don't, then we do need the anotations (both the global and static ones), and problem solved.
If we do, well, where is the 'this is *the* behavior of any current/future clang^Wcompiler' invariant guarantee?
Thanks,
Signature · 2 lines
Jiri Kosina SUSE Labs
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Josh PoimboeufQuoted 13 lines
Quoted 13 lines
Quoted 3 lines
There are N users[*] of
CONFIG_LIVEPATCH, where N is perhaps dozens. For N-1 users, they have to suffer ALL the drawbacks, with NONE of the benefits.You wrote in the other mail:
Quoted 2 lines
The problems associated with it: performance, LTO incompatibility, clang incompatibility (I think?), the GCC dead code issue.
SUSE performance team did extensive testing and did not found any real performance issues. It was discussed when the option was enabled upstream.
Quoted 6 lines
Quoted 2 lines
Are the other problems affecting real life usage, please? Could you be more specific about them, please?
The original commit mentioned 1-3% scheduler degradation. And I'd expect things to worsen over time as interprocedural optimizations improve.
Or maybe not.
Anyway, -flive-patching does not disable all interprocedural optimizations. By far. Only a subset of optimizations whose usage on the linux kernel is reportedly even not that prominent (compared to heavily C++ template based source codes). Reportedly, because we did some tests but nothing exhaustive. So I'd leave any expectations aside now.
The fact is that -fno-ipa-pure-const caused the objtool issue. One could argue that it should be fixed anyway, because it relies on GCC internal implementation which could easily change, and we luckily found it out thanks to -flive-patching. But you pointed out that was not even the main problem here, so I'd leave it for the separate subthread which Jiri started.
Regarding the scheduler degradation. hackbench performance degradation to make it clear. It might be interesting to find out what really changed there. Which disabled optimization caused it and how. Maybe it could be gained back if proven again (because it may have changed, right?).
It all sound artificial to me though. I am not saying the degradation is not there, but many people also lived with frame pointers enabled for quite a long time and no one seemed to be bothered. And that was even more serious because the decline was bigger and it was measurable in many workflows. Not just a schedule microbenchmark. That is why Petr asked about real life reports, I guess.
Quoted 3 lines
Also, LTO is coming whether we like it or not. As is Clang. Those are real-world things which will need to work with livepatching sooner or later.
Yes, but we are not there yet. Once a user has problem with that, we will try to solve it.
LTO might not be a big problem. The number of ipa clones would probably grow, but that is not directly dangerous. It remains to be seen.
I don't know much about Clang.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
And, even if they wanted those benefits, they have no idea how to get them because the patch creation process isn't documented.
I do not understand this. All the sample modules and selftests are using source based livepatches.
We're talking in circles. Have you read the thread?
The samples are a (dangerous) joke. With or without -flive-patching.
How come?
In my opinion, the samples and selftests try to show the way to prepare a (simple, yes) live patch. We try to ensure it always works (selftests should).
After all, there is not much more we do at SUSE to prepare a live patch.
1. take a patch and put all touched functions in a live patch
2. if the functions cannot be patched, patch their callers
3. do the function closure and/or add references (relocations or kallsyms trick) so it can all be compiled.
4. done
See? Samples and selftests are not different. Our live patches are not different (https://kernel.suse.com/cgit/kernel-livepatch/). Can we implement the samples and selftests without -flive-patching? No, not really. Or we could, but no guarantees they would work.
For 2., we use -fdump-ipa-clones and Martin Liska's tool
(https://github.com/marxin/kgraft-analysis-tool) to parse the output.
Yes, sometimes it is more complicated. Source based approach allows us to cope with that quite well. But that is case by case and cannot be easily documented.
Do we lack the documentation of our approach? Definitely. We are moving to klp-ccp automation now (https://github.com/SUSE/klp-ccp) and once done completely, we will hopefully have some documention. CCing Nicolai if he wants to add something.
Should it be upstream? I don't know. I don't think so. For the same reason kpatch-build documentation is not upstream either. Use cases of the infrastructure differ. Maybe there are users who use it in a completely different way. I don't know. In fact, it does not matter to me. I think we should support it all if they make sense.
And that is my message which (in my opinion) makes more sense. Definitely more sense than your "kpatch-build is the only safe way to prepare a live patch" mantra you are trying to sell here for whatever reason. I don't agree with it.
Quoted 5 lines
Quoted 2 lines
It is actually the only somehow documented way. Sure, the documentation might get improved. Patches are welcome.
Are you suggesting for *me* to send documentation for how *you* build patches?
I don't think that is what Petr meant (he will definitely correct me). If you think there is a space for improvement in our upstream documentation of the infrastructure, you are welcome to send patches. The space is definitely there.
Quoted 8 lines
Quoted 5 lines
The option is not currently needed by the selftests only because there is no selftest for this type of problems. But the problems are real. They would actually deserve selftests. Again, patches are welcome.
My understanding is that the source based livepatches is the future.
I think that still remains to be seen.
Quoted 7 lines
Quoted 2 lines
N-1 users are just waiting until the 1 user develops more helper tools for this.
No. N-1 users have no idea how to make (safe) source-based patches in the first place. And if *you* don't need the tools, why would anyone else? Why not document the process and encourage the existence of other users so they can get involved and help with the tooling?
I replied to this one above. You are right we should document our approach better. I think it is off topic of the thread and problem here.
Regards
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 6 lines · Miroslav Benes
The fact is that -fno-ipa-pure-const caused the objtool issue. One could argue that it should be fixed anyway, because it relies on GCC internal implementation which could easily change, and we luckily found it out thanks to -flive-patching. But you pointed out that was not even the main problem here, so I'd leave it for the separate subthread which Jiri started.
It's not an objtool "issue". The warnings were correct. And objtool *has* to rely on GCC internals.
And why would this particular internal implementation ever change (detecting static noreturns)? I don't see why optimizing the call interface to a pure/const static function would break GCC's implicit noreturn detection anyway. It smells like a GCC bug.
Quoted 4 lines
Regarding the scheduler degradation. hackbench performance degradation to make it clear. It might be interesting to find out what really changed there. Which disabled optimization caused it and how. Maybe it could be gained back if proven again (because it may have changed, right?).
Fixing the scheduler performance regression would be a good thing to have done *before* merging the patch.
Quoted 6 lines
It all sound artificial to me though. I am not saying the degradation is not there, but many people also lived with frame pointers enabled for quite a long time and no one seemed to be bothered. And that was even more serious because the decline was bigger and it was measurable in many workflows. Not just a schedule microbenchmark. That is why Petr asked about real life reports, I guess.
Many people were happy to get rid of frame pointers.
Quoted 10 lines
Quoted 1 line
The samples are a (dangerous) joke. With or without -flive-patching.
How come?
In my opinion, the samples and selftests try to show the way to prepare a (simple, yes) live patch. We try to ensure it always works (selftests should).
After all, there is not much more we do at SUSE to prepare a live patch.
Quoted 7 lines
1. take a patch and put all touched functions in a live patch
2. if the functions cannot be patched, patch their callers
3. do the function closure and/or add references (relocations or kallsyms trick) so it can all be compiled.
4. doneSee? Samples and selftests are not different.
How much ABI optimization analysis was done before creating the samples? (hint: none)
And how would somebody using the samples as a guide know to do all that?
Quoted 10 lines
Do we lack the documentation of our approach? Definitely. We are moving to klp-ccp automation now (https://github.com/SUSE/klp-ccp) and once done completely, we will hopefully have some documention. CCing Nicolai if he wants to add something.
Should it be upstream? I don't know. I don't think so. For the same reason kpatch-build documentation is not upstream either. Use cases of the infrastructure differ. Maybe there are users who use it in a completely different way. I don't know. In fact, it does not matter to me. I think we should support it all if they make sense.
Of course the documentation should be in-tree. Otherwise the samples are *very* misleading.
Quoted 4 lines
And that is my message which (in my opinion) makes more sense. Definitely more sense than your "kpatch-build is the only safe way to prepare a live patch" mantra you are trying to sell here for whatever reason. I don't agree with it.
Of course I didn't say that.
The only thing I'm trying to "sell" is that this flag has several drawbacks and no benefits for the upstream community. Why do you keep dancing around that unavoidable fact?
Quoted 10 lines
Quoted 5 lines
Quoted 2 lines
It is actually the only somehow documented way. Sure, the documentation might get improved. Patches are welcome.
Are you suggesting for *me* to send documentation for how *you* build patches?
I don't think that is what Petr meant (he will definitely correct me). If you think there is a space for improvement in our upstream documentation of the infrastructure, you are welcome to send patches. The space is definitely there.
If you want to use the -flive-patching flag for CONFIG_LIVEPATCH, then yes, there's a huge gap in the documentation. I don't understand why you seem to be suggesting that I'm the one whose qualified to write that documentation.
Quoted 10 lines
Quoted 7 lines
Quoted 2 lines
N-1 users are just waiting until the 1 user develops more helper tools for this.
No. N-1 users have no idea how to make (safe) source-based patches in the first place. And if *you* don't need the tools, why would anyone else? Why not document the process and encourage the existence of other users so they can get involved and help with the tooling?
I replied to this one above. You are right we should document our approach better. I think it is off topic of the thread and problem here.
It's actually very much on-topic. It's one of the main reasons why I wanted to revert the patch. Surely that's clear by now?
In retrospect, the prerequisites for merging it should have been:
1) Document how source-based patches can be safely generated;
2) Fix the scheduler performance regression;
3) Figure out if there are any other regressions by detecting which function interfaces are affected by the flag and seeing if they're hot path;
4) Provide a way for the N-1 users to opt-out
5) Fix the objtool warnings (or is it a GCC bug)
6) Make -flive-patching compatible with LTO (or at least acknowledge that it should and will be done soon)
7) At least make it build- or runtime-incompatible with Clang-built kernels to prevent people from assuming it's safe.
If you don't want to revert the patch, then address my concerns instead of minimizing and deflecting at every opportunity.
Signature · 1 line
Josh
Petr Mladek
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 1 line · Josh Poimboeuf
In retrospect, the prerequisites for merging it should have been:
OK, let me do one more move in this game.
Quoted 1 line
1) Document how source-based patches can be safely generated;
I agree that the information are really scattered over many files in Documentation/livepatch/. Anyway, there is a lot of useful hints:
+ structure and behavior of the livepatch module, link to a sample, limitations, are described in livepatch.rst
+ many other catches are described in the other files:
callbacks, module-elf-fomat, cumulative-patches, system-state.
Yes, it would be great to have a better structure, more information. But do not get me wrong. Anyone, Joe definitely, is able to create livepatch from sources by this information.
Anyone could play with it, ask questions, and improve the documentation. Better documentation would help but it is not a blocker, definitely.
Quoted 1 line
2) Fix the scheduler performance regression;
The optimizations are disabled only when livepatching is enabled. I would consider this as a prize for the feature. There are many things like this.
As it was said. It was 1-3 percent in scheduler microbenchmark. It would make sense to fix it only when it causes such a regression in real workloads. Do you have any?
Quoted 3 lines
3) Figure out if there are any other regressions by detecting which function interfaces are affected by the flag and seeing if they're hot path;
IMHO, benchmarks are much more effective and we spent non-trivial resources when running them.
Quoted 1 line
4) Provide a way for the N-1 users to opt-out
AFAIK, the only prize is the 1-3 percent scheduler performance degradation. If you really do not want to pay this prize, let's make it configurable.
But the option is definitely needed when source livepatches are used. There is no other reasonable way to detect and workaround these problems. For this, it has to be in upstream kernel. It is in line with the effort to make livepatching less and less error prone.
And please, let's stop playing this multi-user games. There is at least one known user of source based livepatches. By coincidence, it is also a big contributor to this subsystem. Adding an extra option into CFLAGS is quite error prone. You can imagine how complicated is a kernel rpm spec file for more kernel flavors. The only safe way is to have the optimization tight with the CONFIG option in kernel sources.
Quoted 1 line
5) Fix the objtool warnings (or is it a GCC bug)
Nobody was aware of them. I wonder if they even existed at that time. We have a simple fix now. Let's continue in the thread started by Jikos if we could get a better solution.
Quoted 2 lines
6) Make -flive-patching compatible with LTO (or at least acknowledge that it should and will be done soon)
Is LTO officially supported upstream?
Are all patches, features tested for LTO compactibility? Is there any simple way to build and run LTO kernel?
Quoted 2 lines
7) At least make it build- or runtime-incompatible with Clang-built kernels to prevent people from assuming it's safe.
Same questions as for LTO.
Quoted 2 lines
If you don't want to revert the patch, then address my concerns instead of minimizing and deflecting at every opportunity.
I would really like to keep focusing on realistic problems and realistic solutions:
+ make the optimization configurable if you resist on it
+ fix the objtool warnings
Anything else is out of scope of this thread from my POV.
Best Regards,
Petr
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 10 lines · Petr Mladek
Quoted 1 line · Josh Poimboeuf
In retrospect, the prerequisites for merging it should have been:
OK, let me do one more move in this game.
Quoted 1 line
1) Document how source-based patches can be safely generated;
I agree that the information are really scattered over many files in
Documentation/livepatch/.
Once again you're blithely ignoring my point and pretending I'm saying something else. And you did that again further down in the email, but what's the point of arguing if you're not going to listen.
This has nothing to do with the organization of the existing documentation. When did I say that?
Adding the -flive-patching flag doesn't remove *all* function-ABI-breaking optimizations. It's only a partial solution. The rest of the solution involves tooling and processes which need to be documented. But you already know that.
If we weren't co-maintainers I would have reverted the patch days ago. I've tried to give you all the benefit of the doubt. But you seem to be playing company politics.
I would ask that you please put on your upstream hats and stop playing politics. If the patch creation process is a secret, then by all means, keep it secret. But then keep your GCC flag to yourself.
Signature · 1 line
Josh
Petr Mladek
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 7 lines · Josh Poimboeuf
Quoted 6 lines · Petr Mladek
Quoted 1 line · Josh Poimboeuf
In retrospect, the prerequisites for merging it should have been:
OK, let me do one more move in this game.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
1) Document how source-based patches can be safely generated;
I agree that the information are really scattered over many files in
Documentation/livepatch/.Once again you're blithely ignoring my point and pretending I'm saying something else. And you did that again further down in the email, but what's the point of arguing if you're not going to listen.
I have exactly the same feeling but the opposite way.
Quoted 3 lines
I would ask that you please put on your upstream hats and stop playing politics. If the patch creation process is a secret, then by all means, keep it secret. But then keep your GCC flag to yourself.
The thing is that we do not have any magic secret.
Best Regards,
Petr
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Josh PoimboeufQuoted 28 lines · Josh Poimboeuf
Quoted 27 lines · Miroslav Benes
Quoted 24 lines · Josh Poimboeuf
Quoted 10 lines · Josh Poimboeuf
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
Quoted 7 lines
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for
CONFIG_LIVEPATCH, since kpatch-build doesn't need it.I also just discovered that -flive-patching is responsible for all those "unreachable instruction" objtool warnings which Randy has been dutifully bugging me about over the last several months. For some reason it subtly breaks GCC implicit noreturn detection for local functions.
Quoted 3 lines
Quoted 1 line
Ugh, that is unfortunate. Have you reported it?
Not yet (but I plan to).
My findings so far...
I bisected through GCC options which -flive-patching disables and -fno-ipa-pure-const is the culprit. I got no warnings without the option with my config.
Then I found out allmodconfig was ok even with -flive-patching. CONFIG_GCOV is the difference. CONFIG_GCOV=y seems to make the warnings go away here.
/me goes staring
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 36 lines · Miroslav Benes
Quoted 28 lines · Josh Poimboeuf
Quoted 27 lines · Miroslav Benes
Quoted 24 lines · Josh Poimboeuf
Quoted 10 lines · Josh Poimboeuf
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines
which are not compatible with livepatching. GCC upstream now has -flive-patching option, which disables all those interfering optimizations.
Which, IIRC, has a significant performance impact and should thus really not be used...
If distros ship that crap, I'm going to laugh at them the next time they want a single digit performance improvement because *important*.
Quoted 7 lines
I have a crazy plan to try to use objtool to detect function changes at a binary level, which would hopefully allow us to drop this flag.
But regardless, I wonder if we enabled this flag prematurely. We still don't have a reasonable way to use it for creating source-based live patches upstream, and it should really be optional for
CONFIG_LIVEPATCH, since kpatch-build doesn't need it.I also just discovered that -flive-patching is responsible for all those "unreachable instruction" objtool warnings which Randy has been dutifully bugging me about over the last several months. For some reason it subtly breaks GCC implicit noreturn detection for local functions.
Quoted 3 lines
Quoted 1 line
Ugh, that is unfortunate. Have you reported it?
Not yet (but I plan to).
My findings so far...
Quoted 7 lines
I bisected through GCC options which -flive-patching disables and -fno-ipa-pure-const is the culprit. I got no warnings without the option with my config.
Then I found out allmodconfig was ok even with -flive-patching.
CONFIG_GCOVis the difference.CONFIG_GCOV=yseems to make the warnings go away here.
Sorry, that was a red herring. See 867ac9d73709 ("objtool: Fix gcov check for older versions of GCC").
I started looking at some btrfs reports and then found out those were already fixed.
https://lore.kernel.org/linux-btrfs/[email protected]/
arch/x86/kernel/cpu/mce/core.o: warning: objtool: mce_panic()+0x11b: unreachable instruction was next...
Broken code (-fno-ipa-pure-const):
...
1186: e8 a5 fe ff ff callq 1030 <wait_for_panic>
118b: e9 23 ff ff ff jmpq 10b3 <mce_panic+0x43></end of function>
Working code (-fipa-pure-const):
753: e8 88 fe ff ff callq 5e0 <wait_for_panic>
758: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1)
75f: 00 mce_panic() has:
if (atomic_inc_return(&mce_panicked) > 1)
wait_for_panic();
barrier();
bust_spinlocks(1); jmpq in the broken code goes to bust_spinlocks(1), because GCC does not know that wait_for_panic() is noreturn... because it is not. wait_for_panic() calls panic() unconditionally in the end, which is noreturn.
So the question is why ipa-pure-const optimization knows about panic()'s noreturn. The answer is that it is right one of the things the optimization does. It propagates inner noreturns to its callers. (Martin Jambor CCed).
Marking wait_for_panic() as noreturn (__noreturn), of course, fixes it then. Now I don't know what the right fix should be. Should we mark all these sites as noreturn, or is it ok for the kernel to rely on GCC behaviour in this case? Could we teach objtool to recognize this?
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 35 lines · Miroslav Benes
I started looking at some btrfs reports and then found out those were already fixed.
https://lore.kernel.org/linux-btrfs/[email protected]/
arch/x86/kernel/cpu/mce/core.o: warning: objtool: mce_panic()+0x11b: unreachable instruction was next...Broken code (-fno-ipa-pure-const):
...1186: e8 a5 fe ff ff callq 1030 <wait_for_panic> 118b: e9 23 ff ff ff jmpq 10b3 <mce_panic+0x43></end of function>
Working code (-fipa-pure-const):
753: e8 88 fe ff ff callq 5e0 <wait_for_panic> 758: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) 75f: 00
mce_panic()has:if (atomic_inc_return(&mce_panicked) > 1) wait_for_panic(); barrier(); bust_spinlocks(1);jmpq in the broken code goes to
bust_spinlocks(1), because GCC does not know thatwait_for_panic()is noreturn... because it is not.wait_for_panic()callspanic()unconditionally in the end, which is noreturn.So the question is why ipa-pure-const optimization knows about panic()'s noreturn. The answer is that it is right one of the things the optimization does. It propagates inner noreturns to its callers. (Martin Jambor CCed).
Quoted 4 lines
Marking
wait_for_panic()as noreturn (__noreturn), of course, fixes it then. Now I don't know what the right fix should be. Should we mark all these sites as noreturn, or is it ok for the kernel to rely on GCC behaviour in this case? Could we teach objtool to recognize this?
Thanks for looking at it. I cam to a similar conclusion and I already had the manual noreturns added (see patch below) before I realized that -flive-patching was the culprit.
The patch works, but the problem is that more warnings will pop up in the future and it'll be my job to fix them...
Global noreturns are already a pain today. There's no way for objtool to know whether GCC considered a function to be noreturn, we have already have to keep a hard-coded list of global noreturns in objtool. It's been a constant source of annoyance and this will add to that.
drivers/gpu/drm/ttm/ttm_bo.c +1 -1
@@ -192,7 +192,7 @@ static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, } } -static void ttm_bo_ref_bug (struct kref *list_kref) +static void __noreturn ttm_bo_ref_bug (struct kref *list_kref) {BUG (); }
drivers/message/fusion/mptbase.h +1 -1
@@ -945,7 +945,7 @@ extern int mpt_raid_phys_disk_get_num_paths(MPT_ADAPTER *ioc, u8 phys_disk_num);extern int mpt_set_taskmgmt_in_progress_flag (MPT_ADAPTER *ioc);extern void mpt_clear_taskmgmt_in_progress_flag (MPT_ADAPTER *ioc); -extern void mpt_halt_firmware (MPT_ADAPTER *ioc); +extern void mpt_halt_firmware (MPT_ADAPTER *ioc)__noreturn ;/*
fs/btrfs/extent_io.c +1 -1
@@ -655,7 +655,7 @@ alloc_extent_state_atomic(struct extent_state *prealloc)return prealloc; } -static void extent_io_tree_panic (struct extent_io_tree *tree,int err) +static void __noreturn extent_io_tree_panic (struct extent_io_tree *tree,int err) {struct inode *inode = tree->private_data;
fs/btrfs/relocation.c +1 -1
@@ -321,7 +321,7 @@ static struct rb_node *tree_search(struct rb_root *root, u64 bytenr)return NULL ; } -static void backref_tree_panic (struct rb_node *rb_node,int errno,u64 bytenr) +static void __noreturn backref_tree_panic (struct rb_node *rb_node,int errno,u64 bytenr) {struct btrfs_fs_info *fs_info =NULL ;
include/linux/cred.h +1 -1
@@ -175,7 +175,7 @@ extern void __init cred_init(void);* check for validity of credentials */ #ifdef CONFIG_DEBUG_CREDENTIALS -extern void __invalid_creds (const struct cred *,const char *,unsigned ); +extern void __noreturn __invalid_creds (const struct cred *,const char *,unsigned );extern void __validate_process_creds (struct task_struct *,const char *,unsigned );
include/linux/sched/task.h +1 -1
@@ -86,7 +86,7 @@ static inline void exit_thread(struct task_struct *tsk) { }#endif -extern void do_group_exit (int ); +extern void __noreturn do_group_exit (int );extern void exit_files (struct task_struct *);extern void exit_itimers (struct signal_struct *);
net/core/skbuff.c +2 -2
@@ -98,8 +98,8 @@ EXPORT_SYMBOL(sysctl_max_skb_frags);* Keep out of line to prevent kernel bloat. * __builtin_return_address is not used because it is not always reliable. */ -static void skb_panic (struct sk_buff *skb,unsigned int sz,void *addr, -const char msg[]) +static void __noreturn +skb_panic (struct sk_buff *skb,unsigned int sz,void *addr,const char msg[]) { pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", msg, addr, skb->len, sz, skb->head, skb->data,
tools/objtool/check.c +3 -0
@@ -145,6 +145,9 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func,"machine_real_restart" ,"rewind_stack_do_exit" ,"kunit_try_catch_throw" , +"__invalid_creds" , +"do_group_exit" , + "mpt_halt_firmware", };if (!func)
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Hello,
Quoted 2 lines · Josh Poimboeuf
Global noreturns are already a pain today. There's no way for objtool to know whether GCC considered a function to be noreturn,
You should be able to get a good idea with -Wsuggest-attribute=noreturn:
$ cat a.c
int __attribute__((noreturn)) my_abort (void)
{
__builtin_abort ();
}int foo (void)
{
return my_abort ();
}
int bar (int flag)
{
if (flag)
foo ();
return 4;
}
$ gcc -S -O2 -Wsuggest-attribute=noreturn a.c
a.c: In function ‘foo’:
a.c:6:5: warning: function might be candidate for attribute ‘noreturn’ [-Wsuggest-attribute=noreturn]
6 | int foo (void)
| ^~~GCC 9 and newer even have -fdiagnostics-format=json if you are into that kind of thing.
Hope this helps a little,
Martin
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Steven RostedtQuoted 15 lines · Steven Rostedt
(7) Seventh session, titled "klp-convert and livepatch relocations", was led by Joe Lawrence.
Joe started the session with problem statement: accessing non exported / static symbols from inside the patch module. One possible workardound is manually via kallsyms. Second workaround is klp-convert, which actually creates proper relocations inside the livepatch module from the symbol database during the final .ko link.
Currently module loader looks for special livepatch relocations and resolves those during runtime; kernel support for these relocations have so far been added for x86 only. Special livepatch relocations are supported and processed also on other architectures. Special quirks/sections are not yet supported. Plus klp-convert would still be needed even with late module patching update. vmlinux or modules could have ambiguous static symbols.
Quoted 4 lines
It turns out that the features / bugs below have to be resolved before we can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and have a good regression tests in place
I suppose all the patches in this series-of-series here will make life harder for KLP, static_call() and 2 byte jumps etc..
Quoted 3 lines
- one day we might (or might not) add support for out-of-tree modules which need klp-convert
- BFD bug 24456 (multiple relocations to the same .text section)
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Peter Zijlstra
Quoted 15 lines · Steven Rostedt
(7) Seventh session, titled "klp-convert and livepatch relocations", was led by Joe Lawrence.
Joe started the session with problem statement: accessing non exported / static symbols from inside the patch module. One possible workardound is manually via kallsyms. Second workaround is klp-convert, which actually creates proper relocations inside the livepatch module from the symbol database during the final .ko link.
Currently module loader looks for special livepatch relocations and resolves those during runtime; kernel support for these relocations have so far been added for x86 only. Special livepatch relocations are supported and processed also on other architectures. Special quirks/sections are not yet supported. Plus klp-convert would still be needed even with late module patching update. vmlinux or modules could have ambiguous static symbols.
Quoted 7 lines
Quoted 4 lines
It turns out that the features / bugs below have to be resolved before we can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and have a good regression tests in placeI suppose all the patches in this series-of-series here will make life harder for KLP,
static_call()and 2 byte jumps etc..
Yes, I think so. We'll have to deal with that once it lands. That is why we want to get rid of all this arch-specific code in livepatch and reinvent the late module patching. So it is perhaps better to start working on it sooner than later. Adding Petr, who hesitantly signed up for the task...
Miroslav
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 24 lines · Miroslav Benes
Quoted 22 lines · Peter Zijlstra
Quoted 19 lines · Steven Rostedt
(7) Seventh session, titled "klp-convert and livepatch relocations", was led by Joe Lawrence.
Joe started the session with problem statement: accessing non exported / static symbols from inside the patch module. One possible workardound is manually via kallsyms. Second workaround is klp-convert, which actually creates proper relocations inside the livepatch module from the symbol database during the final .ko link.
Currently module loader looks for special livepatch relocations and resolves those during runtime; kernel support for these relocations have so far been added for x86 only. Special livepatch relocations are supported and processed also on other architectures. Special quirks/sections are not yet supported. Plus klp-convert would still be needed even with late module patching update. vmlinux or modules could have ambiguous static symbols.It turns out that the features / bugs below have to be resolved before we can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and have a good regression tests in place
Quoted 8 lines
Quoted 2 lines
I suppose all the patches in this series-of-series here will make life harder for KLP,
static_call()and 2 byte jumps etc..Yes, I think so. We'll have to deal with that once it lands. That is why we want to get rid of all this arch-specific code in livepatch and reinvent the late module patching. So it is perhaps better to start working on it sooner than later. Adding Petr, who hesitantly signed up for the task...
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.
The upside is that almost no work would be required on patch modules creation side. The downside is that klp_modinfo must stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.
Jessica, do you think it would be feasible?
Petr, Joe, Josh, am I missing something or would it work?
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
+++ Miroslav Benes [16/10/19 15:29 +0200]:
Quoted 46 lines · Miroslav Benes
Quoted 25 lines · Peter Zijlstra
Quoted 23 lines · Steven Rostedt
Quoted 20 lines
(7) Seventh session, titled "klp-convert and livepatch relocations", was led by Joe Lawrence.
Joe started the session with problem statement: accessing non exported / static symbols from inside the patch module. One possible workardound is manually via kallsyms. Second workaround is klp-convert, which actually creates proper relocations inside the livepatch module from the symbol database during the final .ko link.
Currently module loader looks for special livepatch relocations and resolves those during runtime; kernel support for these relocations have so far been added for x86 only. Special livepatch relocations are supported and processed also on other architectures. Special quirks/sections are not yet supported. Plus klp-convert would still be needed even with late module patching update. vmlinux or modules could have ambiguous static symbols.It turns out that the features / bugs below have to be resolved before we can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and have a good regression tests in placeQuoted 8 lines
Quoted 2 lines
I suppose all the patches in this series-of-series here will make life harder for KLP,
static_call()and 2 byte jumps etc..Yes, I think so. We'll have to deal with that once it lands. That is why we want to get rid of all this arch-specific code in livepatch and reinvent the late module patching. So it is perhaps better to start working on it sooner than later. Adding Petr, who hesitantly signed up for the task...
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific
module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.
Quoted 6 lines
The upside is that almost no work would be required on patch modules creation side. The downside is that
klp_modinfomust stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.Jessica, do you think it would be feasible?
I think that does sound feasible. I'm trying to visualize how that would look. I guess there would need to be various livepatching hooks called during the different stages (apply_relocate_add(), module_finalize(), module_enable_ro/x()).
So maybe something like the following?
When a livepatch module loads:
apply_relocate_add()
klp hook: apply .klp.rela.$objname relocations *only* for already loaded modules
module_finalize()
klp hook: apply .klp.arch.$objname changes for already loaded modules
module_enable_ro()
klp hook: only enable ro/x for .klp.text.$objname for already loaded modules
When a to-be-patched module loads:
apply_relocate_add()
klp hook: for each patch module that patches the coming module, apply .klp.rela.$objname relocations for this object
module_finalize()
klp hook: for each patch module that patches the coming module, apply .klp.arch.$objname changes for this object
module_enable_ro()
klp hook: for each patch module, apply ro/x permissions for .klp.text.$objname for this object
Then, in klp_module_coming, we only need to do the callbacks and enable the patch, and get rid of the module_disable_ro->apply relocs->module_enable_ro block.
Does that sound like what you had in mind or am I totally off?
Thanks!
Jessica
Petr Mladek
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 27 lines · Jessica Yu
+++ Miroslav Benes [16/10/19 15:29 +0200]:
Quoted 12 lines · Miroslav Benes
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific
module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.Quoted 6 lines
The upside is that almost no work would be required on patch modules creation side. The downside is that
klp_modinfomust stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.Jessica, do you think it would be feasible?
I think that does sound feasible. I'm trying to visualize how that would look. I guess there would need to be various livepatching hooks called during the different stages (
apply_relocate_add(),module_finalize(), module_enable_ro/x()).So maybe something like the following?
Quoted 9 lines
When a livepatch module loads:
apply_relocate_add()
klp hook: apply .klp.rela.$objname relocations *only* for already loaded modules
module_finalize()
klp hook: apply .klp.arch.$objname changes for already loaded modules
module_enable_ro()
klp hook: only enable ro/x for .klp.text.$objname for already loaded modules
Just for record. We should also set ro for the not-yet used .klp.text.$objname at this stage so that it can't be modified easily "by accident".
Quoted 11 lines
When a to-be-patched module loads:
apply_relocate_add()
klp hook: for each patch module that patches the coming module, apply .klp.rela.$objname relocations for this object
module_finalize()
klp hook: for each patch module that patches the coming module, apply .klp.arch.$objname changes for this object
module_enable_ro()
klp hook: for each patch module, apply ro/x permissions for .klp.text.$objname for this object
Quoted 5 lines
Then, in
klp_module_coming, we only need to do the callbacks and enable the patch, and get rid of themodule_disable_ro->applyrelocs->module_enable_roblock.Does that sound like what you had in mind or am I totally off?
Makes sense to me.
Well, I wonder if it is really any better from what we have now. We would still need special delayed handling for the module-specific elf sections. Also we still would not need to clear the modifications in these sections when the livepatched object gets unloaded.
I am afraid that the real difference might come when we split the livepatch into per-livepatched object modules. This would move the complexity to another parts of the code ;-) I am unable to say what approach is easier and more safe to maintain at the moment.
Best Regards,
Petr
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
+++ Petr Mladek [18/10/19 15:40 +0200]:
Quoted 38 lines · Jessica Yu
Quoted 27 lines
+++ Miroslav Benes [16/10/19 15:29 +0200]:
Quoted 12 lines · Miroslav Benes
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific
module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.Quoted 6 lines
The upside is that almost no work would be required on patch modules creation side. The downside is that
klp_modinfomust stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.Jessica, do you think it would be feasible?
I think that does sound feasible. I'm trying to visualize how that would look. I guess there would need to be various livepatching hooks called during the different stages (
apply_relocate_add(),module_finalize(), module_enable_ro/x()).So maybe something like the following?
Quoted 9 lines
When a livepatch module loads:
apply_relocate_add()
klp hook: apply .klp.rela.$objname relocations *only* for already loaded modules
module_finalize()
klp hook: apply .klp.arch.$objname changes for already loaded modules
module_enable_ro()
klp hook: only enable ro/x for .klp.text.$objname for already loaded modules
Quoted 3 lines
Just for record. We should also set ro for the not-yet used .klp.text.$objname at this stage so that it can't be modified easily "by accident".
If we also set ro protection already for .klp.text.$objname for not-yet loaded modules, I think this would unfortunately mean we would still have to do the protection flipping for late module patching that Peter was trying to avoid, right?
That is, we *still* end up having to do the whole module_disable_ro() -> apply_relocate_add() -> module_finalize() -> module_enable_ro() thing for late module patching, except now we've moved that work to the module loader instead of in klp_module_coming.. It sounds just as complicated as the current way :/
However, I think this complaint would not apply if livepatch switches to the one patch module per module model..
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Petr MladekQuoted 58 lines · Petr Mladek
Quoted 27 lines · Jessica Yu
+++ Miroslav Benes [16/10/19 15:29 +0200]:
Quoted 12 lines · Miroslav Benes
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific
module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.Quoted 6 lines
The upside is that almost no work would be required on patch modules creation side. The downside is that
klp_modinfomust stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.Jessica, do you think it would be feasible?
I think that does sound feasible. I'm trying to visualize how that would look. I guess there would need to be various livepatching hooks called during the different stages (
apply_relocate_add(),module_finalize(), module_enable_ro/x()).So maybe something like the following?
Quoted 9 lines
When a livepatch module loads:
apply_relocate_add()
klp hook: apply .klp.rela.$objname relocations *only* for already loaded modules
module_finalize()
klp hook: apply .klp.arch.$objname changes for already loaded modules
module_enable_ro()
klp hook: only enable ro/x for .klp.text.$objname for already loaded modulesJust for record. We should also set ro for the not-yet used .klp.text.$objname at this stage so that it can't be modified easily "by accident".
Quoted 15 lines
When a to-be-patched module loads:
apply_relocate_add()
klp hook: for each patch module that patches the coming module, apply .klp.rela.$objname relocations for this object
module_finalize()
klp hook: for each patch module that patches the coming module, apply .klp.arch.$objname changes for this object
module_enable_ro()
klp hook: for each patch module, apply ro/x permissions for .klp.text.$objname for this objectThen, in
klp_module_coming, we only need to do the callbacks and enable the patch, and get rid of themodule_disable_ro->applyrelocs->module_enable_roblock.
Quoted 5 lines
Quoted 1 line
Does that sound like what you had in mind or am I totally off?
Makes sense to me.
Well, I wonder if it is really any better from what we have now.
AFAICT, this would still have a lot of the same problems we have today. It has a lot of complexity. It needs arch-specific livepatch code and sections, and introduces special cases in the module code.
I'd much prefer the proposal from LPC to have per-module live patches. It's simpler and has less things that can go wrong IMO.
Signature · 1 line
Josh
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Jessica YuQuoted 94 lines · Jessica Yu
+++ Miroslav Benes [16/10/19 15:29 +0200]:
Quoted 58 lines · Miroslav Benes
Quoted 37 lines · Peter Zijlstra
Quoted 35 lines · Steven Rostedt
Quoted 32 lines
(7) Seventh session, titled "klp-convert and livepatch relocations", was led
by Joe Lawrence.Joe started the session with problem statement: accessing non exported / static
symbols from inside the patch module. One possible workardound is manually via
kallsyms. Second workaround is klp-convert, which actually creates proper
relocations inside the livepatch module from the symbol database during the
final .ko link.
Currently module loader looks for special livepatch relocations and resolves
those during runtime; kernel support for these relocations have so far been
added for x86 only. Special livepatch relocations are supported and processed
also on other architectures. Special quirks/sections are not yet supported.
Plus klp-convert would still be needed even with late module patching update.
vmlinux or modules could have ambiguous static symbols.It turns out that the features / bugs below have to be resolved before we
can claim the klp-convert support for relocation complete:
- handle all the corner cases (jump labels, static keys, ...) properly and
have a good regression tests in placeQuoted 8 lines
Quoted 2 lines
I suppose all the patches in this series-of-series here will make life harder for KLP,
static_call()and 2 byte jumps etc..Yes, I think so. We'll have to deal with that once it lands. That is why we want to get rid of all this arch-specific code in livepatch and reinvent the late module patching. So it is perhaps better to start working on it sooner than later. Adding Petr, who hesitantly signed up for the task...
Thinking about it more... crazy idea. I think we could leverage these new ELF .text per vmlinux/module sections for the reinvention I was talking about. If we teach module loader to relocate (and apply alternatives and so on, everything in arch-specific
module_finalize()) not the whole module in case of live patch modules, but separate ELF .text sections, it could solve the issue with late module patching we have. It is a variation on Steven's idea. When live patch module is loaded, only its section for present modules would be processed. Then whenever a to-be-patched module is loaded, its .text section in all present patch module would be processed.Quoted 6 lines
The upside is that almost no work would be required on patch modules creation side. The downside is that
klp_modinfomust stay. Module loader needs to be hacked a lot in both cases. So it remains to be seen which idea is easier to implement.Jessica, do you think it would be feasible?
I think that does sound feasible. I'm trying to visualize how that would look. I guess there would need to be various livepatching hooks called during the different stages (
apply_relocate_add(),module_finalize(), module_enable_ro/x()).So maybe something like the following?
When a livepatch module loads:
apply_relocate_add()
klp hook: apply .klp.rela.$objname relocations *only* for already loaded modules
module_finalize()
klp hook: apply .klp.arch.$objname changes for already loaded modules
module_enable_ro()
klp hook: only enable ro/x for .klp.text.$objname for already loaded modulesWhen a to-be-patched module loads:
apply_relocate_add()
klp hook: for each patch module that patches the coming module, apply .klp.rela.$objname relocations for this object
module_finalize()
klp hook: for each patch module that patches the coming module, apply .klp.arch.$objname changes for this object
module_enable_ro()
klp hook: for each patch module, apply ro/x permissions for .klp.text.$objname for this object
Quoted 5 lines
Then, in
klp_module_coming, we only need to do the callbacks and enable the patch, and get rid of themodule_disable_ro->applyrelocs->module_enable_roblock.Does that sound like what you had in mind or am I totally off?
Sort of. What I had in mind was that we could get rid of all special .klp ELF section if module loader guarantees that only sections for loaded modules are processed. Then .klp.rela.$objname is not needed and proper .rela.text.$objname (or whatever its text section is named) should be sufficient. The same for the rest (.klp.arch).
Only then it would be useful.
Miroslav
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 7 lines · Miroslav Benes
Quoted 1 line
Does that sound like what you had in mind or am I totally off?
Sort of. What I had in mind was that we could get rid of all special .klp ELF section if module loader guarantees that only sections for loaded modules are processed. Then .klp.rela.$objname is not needed and proper .rela.text.$objname (or whatever its text section is named) should be sufficient. The same for the rest (.klp.arch).
If I understand correctly, using kvm as an example to-be-patched module, we'd have:
.text.kvm
.rela.text.kvm
.altinstructions.kvm
.rela.altinstructions.kvm
__jump_table.kvm
.rela__jump_table.kvm
etc. i.e. any "special" sections would need to be renamed.
Is that right?
But also I think *any* sections which need relocations would need to be renamed, for example:
.rodata.kvm
.rela.rodata.kvm
.orc_unwind_ip.kvm
.rela.orc_unwind_ip.kvm
It's an interesting idea.
We'd have to be careful about ordering issues. For example, there are module-specific jump labels stored in mod->jump_entries. Right now that's just a pointer to the module's __jump_table section. With late module patching, when kvm is loaded we'd have to insert the klp module's __jump_table.kvm entries into kvm's mod->jump_entries list somehow.
Presumably we'd also have that issue for other sections. Handling that _might_ be as simple as just hacking up find_module_sections() to re-allocate sections and append "patched sections" to them.
But then you still have to worry about when to apply the relocations. If you apply them before patching the sections, then relative relocations would have the wrong values. If you apply them after, then you have to figure out where the appended relocations are.
And if we allow unpatching then we'd presumably have to be able to remove entries from the module specific section lists.
So I get the feeling a lot of complexity would creep in. Even just thinking about it requires more mental gymnastics than the one-patch-per-module idea, so I view that as a bad sign.
Signature · 1 line
Josh
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 19 lines · Josh Poimboeuf
Quoted 7 lines · Miroslav Benes
Quoted 1 line
Does that sound like what you had in mind or am I totally off?
Sort of. What I had in mind was that we could get rid of all special .klp ELF section if module loader guarantees that only sections for loaded modules are processed. Then .klp.rela.$objname is not needed and proper .rela.text.$objname (or whatever its text section is named) should be sufficient. The same for the rest (.klp.arch).
If I understand correctly, using kvm as an example to-be-patched module, we'd have:
.text.kvm
.rela.text.kvm
.altinstructions.kvm
.rela.altinstructions.kvm
__jump_table.kvm
.rela__jump_table.kvm
Quoted 3 lines
etc. i.e. any "special" sections would need to be renamed.
Is that right?
Yes.
Quoted 7 lines
But also I think *any* sections which need relocations would need to be renamed, for example:
.rodata.kvm
.rela.rodata.kvm
.orc_unwind_ip.kvm
.rela.orc_unwind_ip.kvm
Correct.
Quoted 7 lines
It's an interesting idea.
We'd have to be careful about ordering issues. For example, there are module-specific jump labels stored in
mod->jump_entries. Right now that's just a pointer to the module's__jump_tablesection. With late module patching, when kvm is loaded we'd have to insert the klp module's __jump_table.kvm entries into kvm'smod->jump_entrieslist somehow.
Yes.
Quoted 8 lines
Presumably we'd also have that issue for other sections. Handling that _might_ be as simple as just hacking up
find_module_sections()to re-allocate sections and append "patched sections" to them.But then you still have to worry about when to apply the relocations. If you apply them before patching the sections, then relative relocations would have the wrong values. If you apply them after, then you have to figure out where the appended relocations are.
Ah, right. That is a valid remark.
Quoted 2 lines
And if we allow unpatching then we'd presumably have to be able to remove entries from the module specific section lists.
Correct.
Quoted 3 lines
So I get the feeling a lot of complexity would creep in. Even just thinking about it requires more mental gymnastics than the one-patch-per-module idea, so I view that as a bad sign.
Yes, the devil is in the details. It would be better if the approach helped even someone/something else in the kernel. Without it, it is probably better to stick to Steven's proposal and handle the complexity elsewhere.
Thanks
Miroslav
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Joe LawrenceQuoted 10 lines · Joe Lawrence
Quoted 4 lines · Miroslav Benes
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/
It should not be a problem to create a live patch module like that and the code in kernel/livepatch/ is almost ready. Something like module_section_disable_ro(mod, section) (and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.
Miroslav
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Miroslav Benes
Quoted 10 lines · Joe Lawrence
Quoted 4 lines · Miroslav Benes
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/
Quoted 5 lines
It should not be a problem to create a live patch module like that and the code in
kernel/livepatch/is almost ready. Something likemodule_section_disable_ro(mod, section)(and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.
Just had a browse around the module code, and while the section info is in load_info, it doesn't get retained for active modules.
So I suppose I'll go add that for KLP enabled things.
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
+++ Peter Zijlstra [16/10/19 11:23 +0200]:
Quoted 24 lines · Miroslav Benes
Quoted 17 lines · Joe Lawrence
Quoted 11 lines · Miroslav Benes
Quoted 4 lines
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/Quoted 5 lines
It should not be a problem to create a live patch module like that and the code in
kernel/livepatch/is almost ready. Something likemodule_section_disable_ro(mod, section)(and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.
Quoted 4 lines
Just had a browse around the module code, and while the section info is in
load_info, it doesn't get retained for active modules.So I suppose I'll go add that for KLP enabled things.
Elf section info does get saved for livepatch modules though, see mod->klp_info. And wouldn't this only be needed for livepatch modules?
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 25 lines · Jessica Yu
+++ Peter Zijlstra [16/10/19 11:23 +0200]:
Quoted 24 lines · Miroslav Benes
Quoted 22 lines · Joe Lawrence
Quoted 11 lines · Miroslav Benes
Quoted 4 lines
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/It should not be a problem to create a live patch module like that and the code in
kernel/livepatch/is almost ready. Something likemodule_section_disable_ro(mod, section)(and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.
Quoted 7 lines
Quoted 4 lines
Just had a browse around the module code, and while the section info is in
load_info, it doesn't get retained for active modules.So I suppose I'll go add that for KLP enabled things.
Elf section info does get saved for livepatch modules though, see
mod->klp_info. And wouldn't this only be needed for livepatch modules?
Right, I just found that, but it is x86 only for some mysterious reason. And yes, it's KLP only.
I was thikning of adding a KLP only list of {name,addr,size} sections that start with ".klp" in layout_sections(). Would that not work across architectures?
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Miroslav BenesQuoted 17 lines · Miroslav Benes
Quoted 10 lines · Joe Lawrence
Quoted 4 lines · Miroslav Benes
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/
Quoted 5 lines
It should not be a problem to create a live patch module like that and the code in
kernel/livepatch/is almost ready. Something likemodule_section_disable_ro(mod, section)(and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.
Something a little like so.. completely fresh of the keyboard.
---
include/linux/module.h +12 -0
@@ -853,6 +853,18 @@ static inline void module_enable_ro(consstatic inline void module_disable_ro (const struct module *mod) { }#endif +#if defined(CONFIG_STRICT_MODULE_RWX ) && defined(CONFIG_LIVEPATCH ) +extern void module_section_disable_ro (struct module *mod,const char *sec); +extern void module_section_enable_ro (struct module *mod,const char *sec); +extern void module_section_disable_x (struct module *mod,const char *sec); +extern void module_section_enable_x (struct module *mod,const char *sec); +#else +static inline void module_section_disable_ro (struct module *mod,const char *sec) { } +static inline void module_section_enable_ro (struct module *mod,const char *sec) { } +static inline void module_section_disable_x (struct module *mod,const char *sec) { } +static inline void module_section_enable_x (struct module *mod,const char *sec) { } +#endif +#ifdef CONFIG_GENERIC_BUG void module_bug_finalize (const Elf_Ehdr *,const Elf_Shdr *,struct module *);
kernel/module.c +48 -0
54 lines not on this page · raw message
Miroslav Benes
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 18 lines · Peter Zijlstra
Quoted 17 lines · Miroslav Benes
Quoted 10 lines · Joe Lawrence
Quoted 4 lines · Miroslav Benes
Yes, it does.
klp_module_coming()callsmodule_disable_ro()on all patching modules which patch the coming module in order to callapply_relocate_add(). New (patching) code for a module can be relocated only when the relevant module is loaded.FWIW, would the LPC blue-sky2 model (ie, Steve's suggestion @ plumber's where livepatches only patch a single object and updates are kept on disk to handle coming module updates as they are loaded) eliminate those outstanding relocations and the need to perform this late permission flipping?
Yes, it should, but we don't have to wait for it. PeterZ proposed a different solution to this specific issue in
https://lore.kernel.org/lkml/[email protected]/
Quoted 7 lines
Quoted 5 lines
It should not be a problem to create a live patch module like that and the code in
kernel/livepatch/is almost ready. Something likemodule_section_disable_ro(mod, section)(and similar for X protection) should be enough. Module reloads would still require juggling with the protections, but I think it is all feasible.Something a little like so.. completely fresh of the keyboard.
Yes, but I noticed you found different and better way through text_poke() (I was not aware that text_poke() works around the protections).
Miroslav
Quoted 80 lines
---
include/linux/module.h +12
-0@@ -853,6 +853,18 @@ static inline void module_enable_ro(consstatic inline void module_disable_ro (const struct module *mod) { }#endif +#if defined(CONFIG_STRICT_MODULE_RWX ) && defined(CONFIG_LIVEPATCH ) +extern void module_section_disable_ro (struct module *mod,const char *sec); +extern void module_section_enable_ro (struct module *mod,const char *sec); +extern void module_section_disable_x (struct module *mod,const char *sec); +extern void module_section_enable_x (struct module *mod,const char *sec); +#else +static inline void module_section_disable_ro (struct module *mod,const char *sec) { } +static inline void module_section_enable_ro (struct module *mod,const char *sec) { } +static inline void module_section_disable_x (struct module *mod,const char *sec) { } +static inline void module_section_enable_x (struct module *mod,const char *sec) { } +#endif +#ifdef CONFIG_GENERIC_BUG void module_bug_finalize (const Elf_Ehdr *,const Elf_Shdr *,struct module *);kernel/module.c +48
-0@@ -2107,6 +2107,54 @@ static void free_module_elf(struct modulkfree (mod->klp_info->secstrings);kfree (mod->klp_info); } + +#ifdef CONFIG_STRICT_MODULE_RWX + +static void __frob_section (struct Elf_Shdr *sec,int (*set_memory)(unsigned long start,int num_pages)) +{ +BUG_ON ((unsigned long )sec->sh_addr & (PAGE_SIZE -1 )); +BUG_ON ((unsigned long )sec->sh_size & (PAGE_SIZE -1 )); +set_memory ((unsigned long )sec->sh_addr, sec->sh_size >>PAGE_SHIFT ); +} + +static void frob_section (struct module *mod,const char *section, +int (*set_memory)(unsigned long start,int num_pages)) +{ +struct klp_modinfo *info = mod->klp_info; +const char *secname; +Elf_Shdr *s; + +for (s = info->sechdrs; s < info->sechdrs + info->hdr.e_shnum; s++) { + secname = mod->klp_info->secstrings + s->sh_name; +if (strcmp (secname, section)) +continue ; + +__frob_section (s, set_memory); + } +} + +void module_section_disable_ro (struct module *mod,const char *section) +{ +frob_section (mod, section, set_memory_rw); +} + +void module_section_enable_ro (struct module *mod,const char *section) +{ +frob_section (mod, section, set_memory_ro); +} + +void module_section_disable_x (struct module *mod,const char *section) +{ +frob_section (mod, section, set_memory_nx); +} + +void module_section_enable_x (struct module *mod,const char *section) +{ +frob_section (mod, section, set_memory_x); +} + +#endif /* ONFIG_STRICT_MODULE_RWX */ +#else /* !CONFIG_LIVEPATCH */ static int copy_module_elf (struct module *mod,struct load_info *info) {
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter ZijlstraQuoted 6 lines · Peter Zijlstra
Right, the problem is set_all_modules_text_*(), that relies on COMING having made the protection changes.
After the x86 changes, there's only 2 more architectures that use that function. I'll work on getting those converted and then we can delete that function and worry no more about it.
Here's a possible patch for arch/arm, which only leaves arch/nds32/.
--- arch/arm/kernel/ftrace.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
arch/arm/kernel/ftrace.c +8 -8
@@ -22,6 +22,7 @@#include <asm/ftrace.h> #include <asm/insn.h> #include <asm/set_memory.h> +#include <asm/patch.h> #ifdef CONFIG_THUMB2_KERNEL #define NOP 0xf85deb04/* pop.w {lr} */
@@ -31,13 +32,15 @@#ifdef CONFIG_DYNAMIC_FTRACE +static int patch_text_remap =0 ; +static int __ftrace_modify_code (void *data) {int *command = data; -set_kernel_text_rw (); + patch_text_remap =1 ;ftrace_modify_all_code (*command); -set_kernel_text_ro (); + patch_text_remap =0 ;return 0 ; }
@@ -59,13 +62,13 @@ static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)int ftrace_arch_code_modify_prepare (void ) { -set_all_modules_text_rw (); + patch_text_remap =1 ;return 0 ; }int ftrace_arch_code_modify_post_process (void ) { -set_all_modules_text_ro (); + patch_text_remap =0 ;/* Make sure any TLB misses during machine stop are cleared. */ flush_tlb_all ();return 0 ;
@@ -97,10 +100,7 @@ static int ftrace_modify_code(unsigned long pc, unsigned long old,return -EINVAL ; } -if (probe_kernel_write ((void *)pc, &new,MCOUNT_INSN_SIZE )) -return -EPERM ; - -flush_icache_range (pc, pc +MCOUNT_INSN_SIZE ); +__patch_text_real ((void *)pc, new, patch_text_remap);return 0 ; }
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 9 lines · Peter Zijlstra
Quoted 6 lines · Peter Zijlstra
Right, the problem is set_all_modules_text_*(), that relies on COMING having made the protection changes.
After the x86 changes, there's only 2 more architectures that use that function. I'll work on getting those converted and then we can delete that function and worry no more about it.
Here's a possible patch for
arch/arm, which only leavesarch/nds32/.
*sigh*, so I'd written the patching code for nds32, but then discovered it doesn't have STRICT_MODULE_RWX and therefore we can simply delete the thing.
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter Zijlstra+++ Peter Zijlstra [15/10/19 15:56 +0200]:
Quoted 8 lines · Jessica Yu
Quoted 1 line
I'm having trouble visualizing what changes you're planning on making.
I want all the text poking (
jump_label, ftrace, klp whatever) to happen _before_ we do the protection changes.I also want to avoid flipping the protection state around unnecessarily, because that clearly is just wasted work.
OK, that makes sense, thanks for clarifying.
Quoted 11 lines
Quoted 3 lines
I get that you want to squash
ftrace_module_enable()intoftrace_module_init(), beforemodule_enable_ro(). I'm fine with that as long as the races Steven described are addressed for affected arches.Right, the problem is set_all_modules_text_*(), that relies on COMING having made the protection changes.
After the x86 changes, there's only 2 more architectures that use that function. I'll work on getting those converted and then we can delete that function and worry no more about it.
Quoted 8 lines
Quoted 4 lines
And livepatch should be OK as long as
klp_module_coming()is always called *after*ftrace_module_enable(). But are you movingklp_module_coming()before the module_enable_* calls as well? And if so, why?I wanted to move the COMING notifier callback before the protection changes, because that is the easiest way to convert
jump_labelandstatic_calland AFAICT nothing really cared aside from ftrace.
I think I would be fine with this as long as no notifiers/users rely on the assumption that COMING == module enabled protections already. I've yet to audit all the module notifiers (but I trust you've done this already), and I think ftrace was the only user that relied on this. For livepatch it's probably not immediately fixable without some serious overhaul.
Quoted 3 lines
The alternative is introducing additional module states, which just adds complexity we don't really need if we can just flip things around a little.
Yeah, I would prefer not adding more states if possible :-)
Quoted 19 lines
Quoted 7 lines
Quoted 6 lines
The fact that it is executable; also the fact that you do it right after we mark it ro. Flipping the memory protections back and forth is just really poor everything.
Once this ftrace thing is sorted, we'll change x86 to _refuse_ to make executable (kernel) memory writable.
Quoted 11 lines
Not sure if relevant, but just thought I'd clarify: IIRC,
klp_module_coming()is not poking the coming module, but it callsmodule_enable_ro()on itself (the livepatch module) so it can apply relocations and such on the new code, which lives inside the livepatch module, and it needs to possibly do this numerous times over the lifetime of the patch module for any coming module it is responsible for patching (i.e., callmodule_enable_ro()on the patch module, not necessarily the coming module). So I am not be sure whyklp_module_coming()should be moved beforecomplete_formation(). I hope I'm remembering the details correctly, livepatch folks feel free to chime in if I'm incorrect here.
Quoted 2 lines
You mean it does
module_disable_ro()? That would be broken and it needs to be fixed. Can some livepatch person explain what it does and why?
Gah, sorry, yes I meant module_disable_ro().
Steven Rostedt
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Peter ZijlstraQuoted 11 lines · On Fri, 11 Oct 2019 14:59:03 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
Quoted 5 lines
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
You are mainly worried about making text that is executable into read-write again. What if we kept my one patch that just changed the module in ftrace_module_enable() from read-only to read-write, but before we ever set it executable.
Jessica, would this patch break anything?
It moves the setting of the module execution to after calling both ftrace_module_enable() and klp_module_coming().
This would make it possible to do the module code and still keep with the no executable code becoming writable.
-- Steve
kernel/module.c +5 -1
@@ -3728,7 +3728,6 @@ static int complete_formation(struct module *mod, struct load_info *info)module_enable_ro (mod, false);module_enable_nx (mod); -module_enable_x (mod);/* Mark state as coming so strong_try_module_get() ignores us, * but kallsyms etc. can see us. */
@@ -3751,6 +3750,11 @@ static int prepare_coming_module(struct module *mod)if (err)return err; +/* Make module executable after ftrace is enabled */ +mutex_lock (&module_mutex); +module_enable_x (mod); +mutex_unlock (&module_mutex); +blocking_notifier_call_chain (&module_notify_list,MODULE_STATE_COMING , mod);return 0 ;
Peter Zijlstra
maintainerRe: [PATCH v3 5/6] x86/ftrace: Use text_poke()
Quoted 17 lines · Steven Rostedt
Quoted 14 lines · On Fri, 11 Oct 2019 14:59:03 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().
Quoted 7 lines
Quoted 2 lines
Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
You are mainly worried about making text that is executable into read-write again. What if we kept my one patch that just changed the module in
ftrace_module_enable()from read-only to read-write, but before we ever set it executable.
This still flips the protections back and forth, which is still really ugly. And afaict the only reason this is required is that set_all_modules_text_*() stuff.
So please, instead of tinkering around, lets just kill that horrible interface and be done with it. There's only 2 users left, fixing those can't be too hard.
Re: [PATCH v3 5/6] x86/ftrace: Use text_poke()
In reply to Steven Rostedt+++ Steven Rostedt [15/10/19 09:28 -0400]:
Quoted 49 lines
Quoted 11 lines · On Fri, 11 Oct 2019 14:59:03 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 4 lines · Peter Zijlstra
Really the best solution is to move all the poking into
ftrace_module_init(), before we mark it RO+X. That's what I'm going to do forjump_labelandstatic_callas well, I just need to add that extra notifier callback.OK, so I started writing that patch... or rather, I wrote the patch and started on the Changelog when I ran into trouble describing why we need it.
Quoted 5 lines
That is, I'm struggling to explain why we cannot flip
prepare_coming_module()andcomplete_formation().Yes, it breaks ftrace, but I'm thinking that is all it breaks. So let me see if we can cure that.
You are mainly worried about making text that is executable into read-write again. What if we kept my one patch that just changed the module in
ftrace_module_enable()from read-only to read-write, but before we ever set it executable.Jessica, would this patch break anything?
It moves the setting of the module execution to after calling both
ftrace_module_enable()andklp_module_coming().This would make it possible to do the module code and still keep with the no executable code becoming writable.
-- Steve
kernel/module.c +0
-1@@ -3728,7 +3728,6 @@ static int complete_formation(struct module *mod, struct load_info *info)module_enable_ro (mod, false);module_enable_nx (mod); -module_enable_x (mod);/* Mark state as coming so strong_try_module_get() ignores us, * but kallsyms etc. can see us. */ @@ -3751,6 +3750,11 @@ static int prepare_coming_module(struct module *mod)if (err)
kernel/module.c static int prepare_coming_module(struct module *mod) · line 3760
Quoted 10 lines
kernel/module.c +5
-0return err; +/* Make module executable after ftrace is enabled */ +mutex_lock (&module_mutex); +module_enable_x (mod); +mutex_unlock (&module_mutex); +blocking_notifier_call_chain (&module_notify_list,MODULE_STATE_COMING , mod);return 0 ;
As long as we enable x before parse_args(), which this patch does, then I don't think this change would break anything.
Peter Zijlstra
maintainerPATCH v3 6/6 x86/mm: Remove set_kernel_text_r[ow]()
In reply to Peter ZijlstraWith the last and only user of these functions gone (ftrace) remove them as well to avoid ever growing new users.
--- arch/x86/include/asm/set_memory.h | 2 -- arch/x86/mm/init_32.c | 28 ---------------------------- arch/x86/mm/init_64.c | 36 ------------------------------------ 3 files changed, 66 deletions(-)
arch/x86/include/asm/set_memory.h +0 -2
@@ -80,8 +80,6 @@ int set_direct_map_invalid_noflush(strucint set_direct_map_default_noflush (struct page *page);extern int kernel_set_to_readonly; -void set_kernel_text_rw (void ); -void set_kernel_text_ro (void );#ifdef CONFIG_X86_64 static inline int set_mce_nospec (unsigned long pfn)
arch/x86/mm/init_32.c +0 -28
@@ -874,34 +874,6 @@ void arch_remove_memory(int nid, u64 staint kernel_set_to_readonly__read_mostly ; -void set_kernel_text_rw (void ) -{ -unsigned long start =PFN_ALIGN (_text); -unsigned long size =PFN_ALIGN (_etext) - start; - -if (!kernel_set_to_readonly) -return ; - -pr_debug ("Set kernel text: %lx - %lx for read write\n" , - start, start+size); - -set_pages_rw (virt_to_page (start), size >>PAGE_SHIFT ); -} - -void set_kernel_text_ro (void ) -{ -unsigned long start =PFN_ALIGN (_text); -unsigned long size =PFN_ALIGN (_etext) - start; - -if (!kernel_set_to_readonly) -return ; - -pr_debug ("Set kernel text: %lx - %lx for read only\n" , - start, start+size); - -set_pages_ro (virt_to_page (start), size >>PAGE_SHIFT ); -} -static void mark_nxdata_nx (void ) {/*
arch/x86/mm/init_64.c +0 -36
@@ -1260,42 +1260,6 @@ void __init mem_init(void)int kernel_set_to_readonly; -void set_kernel_text_rw (void ) -{ -unsigned long start =PFN_ALIGN (_text); -unsigned long end =PFN_ALIGN (__stop___ex_table); - -if (!kernel_set_to_readonly) -return ; - -pr_debug ("Set kernel text: %lx - %lx for read write\n" , - start, end); - -/* -* Make the kernel identity mapping for text RW. Kernel text -* mapping will always be RO. Refer to the comment in -* static_protections() in pageattr.c -*/ -set_memory_rw (start, (end - start) >>PAGE_SHIFT ); -} - -void set_kernel_text_ro (void ) -{ -unsigned long start =PFN_ALIGN (_text); -unsigned long end =PFN_ALIGN (__stop___ex_table); - -if (!kernel_set_to_readonly) -return ; - -pr_debug ("Set kernel text: %lx - %lx for read only\n" , - start, end); - -/* -* Set the kernel identity mapping for text RO. -*/ -set_memory_ro (start, (end - start) >>PAGE_SHIFT ); -} -void mark_rodata_ro (void ) {unsigned long start =PFN_ALIGN (_text);
Steven Rostedt
maintainerRe: [PATCH v3 0/6] Rewrite x86/ftrace to use text_poke()
In reply to Peter ZijlstraQuoted 3 lines · On Mon, 07 Oct 2019 10:17:16 +0200 Peter Zijlstra <[email protected]> wrote:
Ftrace was one of the last W^X violators; these patches move it over to the generic
text_poke()interface and thereby get rid of this oddity.
Ingo, Thomas or H.Peter,
If it's OK with you, can you ack this series so that I can pull it through my tree? I have some code that will conflict with this that I want to write on top of it. As well as hammer it out to make sure ftrace works fine.
Thanks!
-- Steve