x86/alternative: Optimize single-byte NOPs at an arbitrary position
Borislav Petkov
maintainerPATCH 1/1 x86/alternative: Optimize single-byte NOPs at an arbitrary position
From: Borislav Petkov <[email protected]>
Up until now the assumption was that an alternative patching site would have some instructions at the beginning and trailing single-byte NOPs (0x90) padding. Therefore, the patching machinery would go and optimize those single-byte NOPs into longer ones.
However, this assumption is broken on 32-bit when code like hv_do_hypercall() in hyperv_init() would use the ratpoline speculation killer CALL_NOSPEC. The 32-bit version of that macro would align certain insns to 16 bytes, leading to the compiler issuing a one or more single-byte NOPs, depending on the holes it needs to fill for alignment.
That would lead to the warning in optimize_nops() to fire:
------------[ cut here ]------------ Not a NOP at 0xc27fb598 WARNING: CPU: 0 PID: 0 at arch/x86/kernel/alternative.c:211 optimize_nops.isra.13
due to that function verifying whether all of the following bytes really are single-byte NOPs.
Therefore, carve out the NOP padding into a separate function and call it for each NOP range beginning with a single-byte NOP.
--- arch/x86/kernel/alternative.c | 62 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 18 deletions(-)
arch/x86/kernel/alternative.c +44 -18
@@ -182,42 +182,68 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) n_dspl, (unsigned long)orig_insn + n_dspl + repl_len); } +/* +* @instr: instruction byte stream +* @instrlen: length of the above +* @off: offset within @instr where the first NOP has been detected +*/ +static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off) +{ +unsigned long flags; +int i = off, nnops; + +while (i < instrlen) { +if (instr[i] !=0x90 ) +break ; + + i++; + } + + nnops = i - off; + +if (nnops <=1 ) +return nnops; + +local_irq_save (flags); +add_nops (instr + off, nnops); +local_irq_restore (flags); + +DUMP_BYTES (instr, instrlen,"%px: [%d:%d) optimized NOPs: " , + instr, off, i); + +return nnops; +} + +/* * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr) { -unsigned long flags;struct insn insn; -int nop, i =0 ; +int i =0 ;/* -* Jump over the non-NOP insns, the remaining bytes must be single-byte -* NOPs, optimize them. +* Jump over the non-NOP insns and optimize single-byte NOPs into bigger +* ones. */ for (;;) {if (insn_decode_kernel (&insn, &instr[i]))return ; +/* +* See if this and any potentially following NOPs can be +* optimized. +*/ if (insn.length ==1 && insn.opcode.bytes[0 ] ==0x90 ) -break ; - -if ((i += insn.length) >= a->instrlen) -return ; - } + i +=optimize_nops_range (instr, a->instrlen, i); +else + i += insn.length; -for (nop = i; i < a->instrlen; i++) { -if (WARN_ONCE (instr[i] !=0x90 ,"Not a NOP at 0x%px\n" , &instr[i])) +if (i >= a->instrlen)return ; } - -local_irq_save (flags); -add_nops (instr + nop, i - nop); -local_irq_restore (flags); - -DUMP_BYTES (instr, a->instrlen,"%px: [%d:%d) optimized NOPs: " , - instr, nop, a->instrlen); }/*
Signature · 1 line
2.29.2
Quoted 7 lines · Borislav Petkov
From: Borislav Petkov <[email protected]>
Up until now the assumption was that an alternative patching site would have some instructions at the beginning and trailing single-byte NOPs (
0x90) padding. Therefore, the patching machinery would go and optimize those single-byte NOPs into longer ones.
Quoted 5 lines
However, this assumption is broken on 32-bit when code like
hv_do_hypercall()inhyperv_init()would use the ratpoline speculation killerCALL_NOSPEC. The 32-bit version of that macro would align certain insns to 16 bytes, leading to the compiler issuing a one or more single-byte NOPs, depending on the holes it needs to fill for alignment.
Who again insisted that wouldn't happen? :-)
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 189
Quoted 13 lines
arch/x86/kernel/alternative.c +5
-0@@ -182,42 +182,68 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) n_dspl, (unsigned long)orig_insn + n_dspl + repl_len); } +/* +* @instr: instruction byte stream +* @instrlen: length of the above +* @off: offset within @instr where the first NOP has been detected +*/
That's almost a proper comment, might as well finish it
/*
* optimize_nops_range() - Optimize a sequence of single byte NOPs (0x90)
* @instr: instruction byte stream
* @instrlen: length of the above
* @off: offset within @instr where the first NOP has been detected *
* Return: number of NOPs found (and replaced)
*/
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 200
Quoted 11 lines
arch/x86/kernel/alternative.c +11
-0+static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off) +{ +unsigned long flags; +int i = off, nnops; + +while (i < instrlen) { +if (instr[i] !=0x90 ) +break ; + + i++; + }
for (; i < instrlen && instr[i] == 0x90; i++) ;
perhaps? (possibly too dense, up to you)
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 205
Quoted 5 lines
arch/x86/kernel/alternative.c +5
-0+ + nnops = i - off; + +if (nnops <=1 ) +return nnops;
!nnops would be an error, no?
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 217
Quoted 12 lines
arch/x86/kernel/alternative.c +12
-0+ +local_irq_save (flags); +add_nops (instr + off, nnops); +local_irq_restore (flags); + +DUMP_BYTES (instr, instrlen,"%px: [%d:%d) optimized NOPs: " , + instr, off, i); + +return nnops; +} + +
We really needs that extra line?
Quoted 20 lines
arch/x86/kernel/alternative.c +6
-0/* * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr) {struct insn insn; +int i =0 ;/* +* Jump over the non-NOP insns and optimize single-byte NOPs into bigger +* ones. */ for (;;) {if (insn_decode_kernel (&insn, &instr[i]))return ; +/* +* See if this and any potentially following NOPs can be +* optimized.
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 247
Quoted 10 lines
arch/x86/kernel/alternative.c +5
-0+*/ if (insn.length ==1 && insn.opcode.bytes[0 ] ==0x90 ) + i +=optimize_nops_range (instr, a->instrlen, i); +else + i += insn.length; +if (i >= a->instrlen)return ; } }
Anyway, irrespective of nits:
Borislav Petkov
maintainerQuoted 1 line · Peter Zijlstra
Who again insisted that wouldn't happen? :-)
Yours truly, ofc. :-P
Quoted 10 lines
That's almost a proper comment, might as well finish it
/*
*optimize_nops_range()- Optimize a sequence of single byte NOPs (0x90)
* @instr: instruction byte stream
* @instrlen: length of the above
* @off: offset within @instr where the first NOP has been detected *
* Return: number of NOPs found (and replaced)
*/
Done.
Quoted 4 lines
for (; i < instrlen && instr[i] == 0x90; i++) ;perhaps? (possibly too dense, up to you)
Yeah, let's leave it simple so that one can read it at a quick glance.
Quoted 7 lines
arch/x86/kernel/alternative.c recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff) · line 205
Quoted 5 lines
arch/x86/kernel/alternative.c +5
-0+ + nnops = i - off; + +if (nnops <=1 ) +return nnops;!nnops would be an error, no?
Yeah, just being overly cautious. It should not be 0 since we're being called for the byte at offset being 0x90. But I have said "should not be" before and have fallen flat on my face.
Quoted 1 line
We really needs that extra line?
Zapped.
Quoted 3 lines
Anyway, irrespective of nits:
Thanks, all except one incorporated.
Signature · 4 lines
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquetteThe following commit has been merged into the x86/urgent branch of tip:
The following commit has been merged into the x86/urgent branch of tip:
Commit-ID: f01775b09efa3b9d1f4a2519678f11d274362da9 Gitweb: https://git.kernel.org/tip/f01775b09efa3b9d1f4a2519678f11d274362da9 Author: Borislav Petkov <[email protected]> AuthorDate: Tue, 01 Jun 2021 17:51:22 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 02 Jun 2021 12:36:07 +02:00
x86/alternative: Optimize single-byte NOPs at an arbitrary position
Up until now the assumption was that an alternative patching site would have some instructions at the beginning and trailing single-byte NOPs (0x90) padding. Therefore, the patching machinery would go and optimize those single-byte NOPs into longer ones.
However, this assumption is broken on 32-bit when code like hv_do_hypercall() in hyperv_init() would use the ratpoline speculation killer CALL_NOSPEC. The 32-bit version of that macro would align certain insns to 16 bytes, leading to the compiler issuing a one or more single-byte NOPs, depending on the holes it needs to fill for alignment.
That would lead to the warning in optimize_nops() to fire:
------------[ cut here ]------------ Not a NOP at 0xc27fb598 WARNING: CPU: 0 PID: 0 at arch/x86/kernel/alternative.c:211 optimize_nops.isra.13
due to that function verifying whether all of the following bytes really are single-byte NOPs.
Therefore, carve out the NOP padding into a separate function and call it for each NOP range beginning with a single-byte NOP.
--- arch/x86/kernel/alternative.c | 64 ++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 18 deletions(-)
arch/x86/kernel/alternative.c +46 -18
@@ -183,41 +183,69 @@ done: }/* +* optimize_nops_range() - Optimize a sequence of single byte NOPs (0x90) +* +* @instr: instruction byte stream +* @instrlen: length of the above +* @off: offset within @instr where the first NOP has been detected +* +* Return: number of NOPs found (and replaced). +*/ +static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off) +{ +unsigned long flags; +int i = off, nnops; + +while (i < instrlen) { +if (instr[i] !=0x90 ) +break ; + + i++; + } + + nnops = i - off; + +if (nnops <=1 ) +return nnops; + +local_irq_save (flags); +add_nops (instr + off, nnops); +local_irq_restore (flags); + +DUMP_BYTES (instr, instrlen,"%px: [%d:%d) optimized NOPs: " , instr, off, i); + +return nnops; +} + +/* * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr) { -unsigned long flags;struct insn insn; -int nop, i =0 ; +int i =0 ;/* -* Jump over the non-NOP insns, the remaining bytes must be single-byte -* NOPs, optimize them. +* Jump over the non-NOP insns and optimize single-byte NOPs into bigger +* ones. */ for (;;) {if (insn_decode_kernel (&insn, &instr[i]))return ; +/* +* See if this and any potentially following NOPs can be +* optimized. +*/ if (insn.length ==1 && insn.opcode.bytes[0 ] ==0x90 ) -break ; - -if ((i += insn.length) >= a->instrlen) -return ; - } + i +=optimize_nops_range (instr, a->instrlen, i); +else + i += insn.length; -for (nop = i; i < a->instrlen; i++) { -if (WARN_ONCE (instr[i] !=0x90 ,"Not a NOP at 0x%px\n" , &instr[i])) +if (i >= a->instrlen)return ; } - -local_irq_save (flags); -add_nops (instr + nop, i - nop); -local_irq_restore (flags); - -DUMP_BYTES (instr, a->instrlen,"%px: [%d:%d) optimized NOPs: " , - instr, nop, a->instrlen); }/*
The following commit has been merged into the x86/urgent branch of tip:
The following commit has been merged into the x86/urgent branch of tip:
Commit-ID: 2b31e8ed96b260ce2c22bd62ecbb9458399e3b62 Gitweb: https://git.kernel.org/tip/2b31e8ed96b260ce2c22bd62ecbb9458399e3b62 Author: Borislav Petkov <[email protected]> AuthorDate: Tue, 01 Jun 2021 17:51:22 +02:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Thu, 03 Jun 2021 16:33:09 +02:00
x86/alternative: Optimize single-byte NOPs at an arbitrary position
Up until now the assumption was that an alternative patching site would have some instructions at the beginning and trailing single-byte NOPs (0x90) padding. Therefore, the patching machinery would go and optimize those single-byte NOPs into longer ones.
However, this assumption is broken on 32-bit when code like hv_do_hypercall() in hyperv_init() would use the ratpoline speculation killer CALL_NOSPEC. The 32-bit version of that macro would align certain insns to 16 bytes, leading to the compiler issuing a one or more single-byte NOPs, depending on the holes it needs to fill for alignment.
That would lead to the warning in optimize_nops() to fire:
------------[ cut here ]------------ Not a NOP at 0xc27fb598 WARNING: CPU: 0 PID: 0 at arch/x86/kernel/alternative.c:211 optimize_nops.isra.13
due to that function verifying whether all of the following bytes really are single-byte NOPs.
Therefore, carve out the NOP padding into a separate function and call it for each NOP range beginning with a single-byte NOP.
--- arch/x86/kernel/alternative.c | 64 ++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 18 deletions(-)
arch/x86/kernel/alternative.c +46 -18
@@ -183,41 +183,69 @@ done: }/* +* optimize_nops_range() - Optimize a sequence of single byte NOPs (0x90) +* +* @instr: instruction byte stream +* @instrlen: length of the above +* @off: offset within @instr where the first NOP has been detected +* +* Return: number of NOPs found (and replaced). +*/ +static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off) +{ +unsigned long flags; +int i = off, nnops; + +while (i < instrlen) { +if (instr[i] !=0x90 ) +break ; + + i++; + } + + nnops = i - off; + +if (nnops <=1 ) +return nnops; + +local_irq_save (flags); +add_nops (instr + off, nnops); +local_irq_restore (flags); + +DUMP_BYTES (instr, instrlen,"%px: [%d:%d) optimized NOPs: " , instr, off, i); + +return nnops; +} + +/* * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr) { -unsigned long flags;struct insn insn; -int nop, i =0 ; +int i =0 ;/* -* Jump over the non-NOP insns, the remaining bytes must be single-byte -* NOPs, optimize them. +* Jump over the non-NOP insns and optimize single-byte NOPs into bigger +* ones. */ for (;;) {if (insn_decode_kernel (&insn, &instr[i]))return ; +/* +* See if this and any potentially following NOPs can be +* optimized. +*/ if (insn.length ==1 && insn.opcode.bytes[0 ] ==0x90 ) -break ; - -if ((i += insn.length) >= a->instrlen) -return ; - } + i +=optimize_nops_range (instr, a->instrlen, i); +else + i += insn.length; -for (nop = i; i < a->instrlen; i++) { -if (WARN_ONCE (instr[i] !=0x90 ,"Not a NOP at 0x%px\n" , &instr[i])) +if (i >= a->instrlen)return ; } - -local_irq_save (flags); -add_nops (instr + nop, i - nop); -local_irq_restore (flags); - -DUMP_BYTES (instr, a->instrlen,"%px: [%d:%d) optimized NOPs: " , - instr, nop, a->instrlen); }/*