x86: Add stright-line-speculation mitigations
Subsystems (18)
Peter Zijlstra
maintainerv2 cover x86: Add stright-line-speculation mitigations
Respin and split of the SLS patch.
Since the old patch hit a snag, I've split up the patch as per popular demand.
New this time are patches 1 and 5 and some minor edits to the last patch, the rest is mostly identical.
Patch 1 removes a 32bit (abuse) of the RET macro; which got exposed by the build robot due to a change in the last patch (defining RET even for !CONFIG_SLS). It would've been possible to revert to the old method of relying on AS being case insensitive and RET being a valid instruction etc.. but I figured that code was convoluted enough to warrant cleaning up anyway.
Patch 5 is what made Boris' machine go BUG when he tried to use the SLS patch.
Combined it seems to boot and build a kernel for me, so it must be perfect this time (fingers crossed).
Peter Zijlstra
maintainerPATCH v2 1/6 x86/atomic64_386_32: Rename things
Principally to get rid of #define RET in this code to make place for a new RET, but also to clarify the code, rename a bunch of things:
s/UNLOCK/IRQ_RESTORE/
s/LOCK/IRQ_SAVE/
s/BEGIN/BEGIN_IRQ_SAVE/
s/\<RET\>/RET_IRQ_RESTORE/
s/RET_ENDP/\tRET_IRQ_RESTORE\rENDP/
which then leaves RET unused so it can be removed.
--- arch/x86/lib/atomic64_386_32.S | 84 ++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 38 deletions(-)
arch/x86/lib/atomic64_386_32.S +46 -38
@@ -9,81 +9,83 @@ #include <asm/alternative.h> /* if you want SMP support, implement these with real spinlocks */ -.macro LOCK reg +.macro IRQ_SAVE reg pushfl cli .endm -.macro UNLOCK reg +.macro IRQ_RESTORE reg popfl .endm -#define BEGIN(op) \ +#define BEGIN_IRQ_SAVE(op) \ .macro endp; \ SYM_FUNC_END(atomic64_##op##_386); \ .purgem endp; \ .endm; \ SYM_FUNC_START(atomic64_##op##_386); \ - LOCK v; + IRQ_SAVE v; #define ENDP endp -#define RET \ - UNLOCK v; \ +#define RET_IRQ_RESTORE \ + IRQ_RESTORE v; \ ret -#define RET_ENDP \ - RET; \ - ENDP - #define v %ecx -BEGIN(read) +BEGIN_IRQ_SAVE(read) movl (v), %eax movl 4(v), %edx -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(set) +BEGIN_IRQ_SAVE(set) movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(xchg) +BEGIN_IRQ_SAVE(xchg) movl (v), %eax movl 4(v), %edx movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add) +BEGIN_IRQ_SAVE(add) addl %eax, (v) adcl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add_return) +BEGIN_IRQ_SAVE(add_return) addl (v), %eax adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub) +BEGIN_IRQ_SAVE(sub) subl %eax, (v) sbbl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub_return) +BEGIN_IRQ_SAVE(sub_return) negl %edx negl %eax sbbl $0, %edx
@@ -91,47 +93,52 @@ BEGIN(sub_return) adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc) +BEGIN_IRQ_SAVE(inc) addl $1, (v) adcl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc_return) +BEGIN_IRQ_SAVE(inc_return) movl (v), %eax movl 4(v), %edx addl $1, %eax adcl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec) +BEGIN_IRQ_SAVE(dec) subl $1, (v) sbbl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec_return) +BEGIN_IRQ_SAVE(dec_return) movl (v), %eax movl 4(v), %edx subl $1, %eax sbbl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(add_unless) +BEGIN_IRQ_SAVE(add_unless) addl %eax, %ecx adcl %edx, %edi addl (v), %eax
@@ -143,7 +150,7 @@ BEGIN(add_unless) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: cmpl %edx, %edi jne 1b
@@ -153,7 +160,7 @@ ENDP #undef v #define v %esi -BEGIN(inc_not_zero) +BEGIN_IRQ_SAVE(inc_not_zero) movl (v), %eax movl 4(v), %edx testl %eax, %eax
@@ -165,7 +172,7 @@ BEGIN(inc_not_zero) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: testl %edx, %edx jne 1b
@@ -174,7 +181,7 @@ ENDP #undef v #define v %esi -BEGIN(dec_if_positive) +BEGIN_IRQ_SAVE(dec_if_positive) movl (v), %eax movl 4(v), %edx subl $1, %eax
@@ -183,5 +190,6 @@ BEGIN(dec_if_positive) movl %eax, (v) movl %edx, 4(v) 1: -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v
[tip: x86/core] x86/lib/atomic64_386_32: Rename things
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 22da5a07c75e1104caf6a42f189c97b83d070073 Gitweb: https://git.kernel.org/tip/22da5a07c75e1104caf6a42f189c97b83d070073 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:39 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 08 Dec 2021 11:57:08 +01:00
x86/lib/atomic64_386_32: Rename things
Principally, in order to get rid of #define RET in this code to make place for a new RET, but also to clarify the code, rename a bunch of things:
s/UNLOCK/IRQ_RESTORE/
s/LOCK/IRQ_SAVE/
s/BEGIN/BEGIN_IRQ_SAVE/
s/\<RET\>/RET_IRQ_RESTORE/
s/RET_ENDP/\tRET_IRQ_RESTORE\rENDP/
which then leaves RET unused so it can be removed.
--- arch/x86/lib/atomic64_386_32.S | 84 ++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 38 deletions(-)
arch/x86/lib/atomic64_386_32.S +46 -38
@@ -9,81 +9,83 @@ #include <asm/alternative.h> /* if you want SMP support, implement these with real spinlocks */ -.macro LOCK reg +.macro IRQ_SAVE reg pushfl cli .endm -.macro UNLOCK reg +.macro IRQ_RESTORE reg popfl .endm -#define BEGIN(op) \ +#define BEGIN_IRQ_SAVE(op) \ .macro endp; \ SYM_FUNC_END(atomic64_##op##_386); \ .purgem endp; \ .endm; \ SYM_FUNC_START(atomic64_##op##_386); \ - LOCK v; + IRQ_SAVE v; #define ENDP endp -#define RET \ - UNLOCK v; \ +#define RET_IRQ_RESTORE \ + IRQ_RESTORE v; \ ret -#define RET_ENDP \ - RET; \ - ENDP - #define v %ecx -BEGIN(read) +BEGIN_IRQ_SAVE(read) movl (v), %eax movl 4(v), %edx -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(set) +BEGIN_IRQ_SAVE(set) movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(xchg) +BEGIN_IRQ_SAVE(xchg) movl (v), %eax movl 4(v), %edx movl %ebx, (v) movl %ecx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add) +BEGIN_IRQ_SAVE(add) addl %eax, (v) adcl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(add_return) +BEGIN_IRQ_SAVE(add_return) addl (v), %eax adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub) +BEGIN_IRQ_SAVE(sub) subl %eax, (v) sbbl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %ecx -BEGIN(sub_return) +BEGIN_IRQ_SAVE(sub_return) negl %edx negl %eax sbbl $0, %edx
@@ -91,47 +93,52 @@ BEGIN(sub_return) adcl 4(v), %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc) +BEGIN_IRQ_SAVE(inc) addl $1, (v) adcl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(inc_return) +BEGIN_IRQ_SAVE(inc_return) movl (v), %eax movl 4(v), %edx addl $1, %eax adcl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec) +BEGIN_IRQ_SAVE(dec) subl $1, (v) sbbl $0, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(dec_return) +BEGIN_IRQ_SAVE(dec_return) movl (v), %eax movl 4(v), %edx subl $1, %eax sbbl $0, %edx movl %eax, (v) movl %edx, 4(v) -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v #define v %esi -BEGIN(add_unless) +BEGIN_IRQ_SAVE(add_unless) addl %eax, %ecx adcl %edx, %edi addl (v), %eax
@@ -143,7 +150,7 @@ BEGIN(add_unless) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: cmpl %edx, %edi jne 1b
@@ -153,7 +160,7 @@ ENDP #undef v #define v %esi -BEGIN(inc_not_zero) +BEGIN_IRQ_SAVE(inc_not_zero) movl (v), %eax movl 4(v), %edx testl %eax, %eax
@@ -165,7 +172,7 @@ BEGIN(inc_not_zero) movl %edx, 4(v) movl $1, %eax 2: - RET + RET_IRQ_RESTORE 3: testl %edx, %edx jne 1b
@@ -174,7 +181,7 @@ ENDP #undef v #define v %esi -BEGIN(dec_if_positive) +BEGIN_IRQ_SAVE(dec_if_positive) movl (v), %eax movl 4(v), %edx subl $1, %eax
@@ -183,5 +190,6 @@ BEGIN(dec_if_positive) movl %eax, (v) movl %edx, 4(v) 1: -RET_ENDP + RET_IRQ_RESTORE +ENDP #undef v
Peter Zijlstra
maintainerPATCH v2 2/6 x86: Prepare asm files for straight-line-speculation
In reply to Peter ZijlstraReplace all ret/retq instructions with RET in preparation of making RET a macro. Since AS is case insensitive it's a big no-op without RET defined.
find arch/x86/ -name \*.S | while read file
do
sed -i 's/\<ret[q]*\>/RET/' $file
done
--- arch/x86/boot/compressed/efi_thunk_64.S | 2 arch/x86/boot/compressed/head_64.S | 8 +-- arch/x86/boot/compressed/mem_encrypt.S | 6 +- arch/x86/crypto/aegis128-aesni-asm.S | 48 +++++++++++------------ arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 2 arch/x86/crypto/aesni-intel_asm.S | 56 +++++++++++++-------------- arch/x86/crypto/aesni-intel_avx-x86_64.S | 40 +++++++++---------- arch/x86/crypto/blake2s-core.S | 4 - arch/x86/crypto/blowfish-x86_64-asm_64.S | 12 ++--- arch/x86/crypto/camellia-aesni-avx-asm_64.S | 14 +++--- arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 14 +++--- arch/x86/crypto/camellia-x86_64-asm_64.S | 12 ++--- arch/x86/crypto/cast5-avx-x86_64-asm_64.S | 12 ++--- arch/x86/crypto/cast6-avx-x86_64-asm_64.S | 10 ++-- arch/x86/crypto/chacha-avx2-x86_64.S | 6 +- arch/x86/crypto/chacha-avx512vl-x86_64.S | 6 +- arch/x86/crypto/chacha-ssse3-x86_64.S | 8 +-- arch/x86/crypto/crc32-pclmul_asm.S | 2 arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 2 arch/x86/crypto/crct10dif-pcl-asm_64.S | 2 arch/x86/crypto/des3_ede-asm_64.S | 4 - arch/x86/crypto/ghash-clmulni-intel_asm.S | 6 +- arch/x86/crypto/nh-avx2-x86_64.S | 2 arch/x86/crypto/nh-sse2-x86_64.S | 2 arch/x86/crypto/serpent-avx-x86_64-asm_64.S | 10 ++-- arch/x86/crypto/serpent-avx2-asm_64.S | 10 ++-- arch/x86/crypto/serpent-sse2-i586-asm_32.S | 6 +- arch/x86/crypto/serpent-sse2-x86_64-asm_64.S | 6 +- arch/x86/crypto/sha1_avx2_x86_64_asm.S | 2 arch/x86/crypto/sha1_ni_asm.S | 2 arch/x86/crypto/sha1_ssse3_asm.S | 2 arch/x86/crypto/sha256-avx-asm.S | 2 arch/x86/crypto/sha256-avx2-asm.S | 2 arch/x86/crypto/sha256-ssse3-asm.S | 2 arch/x86/crypto/sha256_ni_asm.S | 2 arch/x86/crypto/sha512-avx-asm.S | 2 arch/x86/crypto/sha512-avx2-asm.S | 2 arch/x86/crypto/sha512-ssse3-asm.S | 2 arch/x86/crypto/sm4-aesni-avx-asm_64.S | 12 ++--- arch/x86/crypto/sm4-aesni-avx2-asm_64.S | 8 +-- arch/x86/crypto/twofish-avx-x86_64-asm_64.S | 10 ++-- arch/x86/crypto/twofish-i586-asm_32.S | 4 - arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 6 +- arch/x86/crypto/twofish-x86_64-asm_64.S | 4 - arch/x86/entry/entry_32.S | 2 arch/x86/entry/entry_64.S | 12 ++--- arch/x86/entry/thunk_32.S | 2 arch/x86/entry/thunk_64.S | 2 arch/x86/entry/vdso/vdso32/system_call.S | 2 arch/x86/entry/vdso/vsgx.S | 2 arch/x86/entry/vsyscall/vsyscall_emu_64.S | 6 +- arch/x86/kernel/acpi/wakeup_32.S | 6 +- arch/x86/kernel/ftrace_32.S | 6 +- arch/x86/kernel/ftrace_64.S | 8 +-- arch/x86/kernel/head_32.S | 2 arch/x86/kernel/irqflags.S | 2 arch/x86/kernel/relocate_kernel_32.S | 10 ++-- arch/x86/kernel/relocate_kernel_64.S | 10 ++-- arch/x86/kernel/sev_verify_cbit.S | 2 arch/x86/kernel/verify_cpu.S | 4 - arch/x86/kvm/svm/vmenter.S | 4 - arch/x86/kvm/vmx/vmenter.S | 14 +++--- arch/x86/lib/atomic64_386_32.S | 2 arch/x86/lib/atomic64_cx8_32.S | 16 +++---- arch/x86/lib/checksum_32.S | 8 +-- arch/x86/lib/clear_page_64.S | 6 +- arch/x86/lib/cmpxchg16b_emu.S | 4 - arch/x86/lib/cmpxchg8b_emu.S | 4 - arch/x86/lib/copy_mc_64.S | 6 +- arch/x86/lib/copy_page_64.S | 4 - arch/x86/lib/copy_user_64.S | 10 ++-- arch/x86/lib/csum-copy_64.S | 2 arch/x86/lib/getuser.S | 22 +++++----- arch/x86/lib/hweight.S | 6 +- arch/x86/lib/iomap_copy_64.S | 2 arch/x86/lib/memcpy_64.S | 12 ++--- arch/x86/lib/memmove_64.S | 4 - arch/x86/lib/memset_64.S | 6 +- arch/x86/lib/msr-reg.S | 4 - arch/x86/lib/putuser.S | 6 +- arch/x86/lib/retpoline.S | 2 arch/x86/math-emu/div_Xsig.S | 2 arch/x86/math-emu/div_small.S | 2 arch/x86/math-emu/mul_Xsig.S | 6 +- arch/x86/math-emu/polynom_Xsig.S | 2 arch/x86/math-emu/reg_norm.S | 6 +- arch/x86/math-emu/reg_round.S | 2 arch/x86/math-emu/reg_u_add.S | 2 arch/x86/math-emu/reg_u_div.S | 2 arch/x86/math-emu/reg_u_mul.S | 2 arch/x86/math-emu/reg_u_sub.S | 2 arch/x86/math-emu/round_Xsig.S | 4 - arch/x86/math-emu/shr_Xsig.S | 8 +-- arch/x86/math-emu/wm_shrx.S | 16 +++---- arch/x86/mm/mem_encrypt_boot.S | 4 - arch/x86/platform/efi/efi_stub_32.S | 2 arch/x86/platform/efi/efi_stub_64.S | 2 arch/x86/platform/efi/efi_thunk_64.S | 2 arch/x86/platform/olpc/xo1-wakeup.S | 6 +- arch/x86/power/hibernate_asm_32.S | 4 - arch/x86/power/hibernate_asm_64.S | 4 - arch/x86/um/checksum_32.S | 4 - arch/x86/um/setjmp_32.S | 2 arch/x86/um/setjmp_64.S | 2 arch/x86/xen/xen-asm.S | 12 ++--- arch/x86/xen/xen-head.S | 2 106 files changed, 350 insertions(+), 350 deletions(-)
arch/x86/boot/compressed/efi_thunk_64.S +1 -1
@@ -93,7 +93,7 @@ SYM_FUNC_START(__efi64_thunk) pop %rbx pop %rbp - ret + RET SYM_FUNC_END(__efi64_thunk) .code32
arch/x86/boot/compressed/head_64.S +4 -4
@@ -813,7 +813,7 @@ SYM_FUNC_START(efi32_pe_entry) 2: popl %edi // restore callee-save registers popl %ebx leave - ret + RET SYM_FUNC_END(efi32_pe_entry) .section ".rodata"
@@ -868,7 +868,7 @@ SYM_FUNC_START(startup32_set_idt_entry) pop %ecx pop %ebx - ret + RET SYM_FUNC_END(startup32_set_idt_entry) #endif
@@ -884,7 +884,7 @@ SYM_FUNC_START(startup32_load_idt) movl %eax, rva(boot32_idt_desc+2)(%ebp) lidt rva(boot32_idt_desc)(%ebp) #endif - ret + RET SYM_FUNC_END(startup32_load_idt) /*
@@ -954,7 +954,7 @@ SYM_FUNC_START(startup32_check_sev_cbit) popl %ebx popl %eax #endif - ret + RET SYM_FUNC_END(startup32_check_sev_cbit) /*
arch/x86/boot/compressed/mem_encrypt.S +3 -3
@@ -58,7 +58,7 @@ SYM_FUNC_START(get_sev_encryption_bit) #endif /* CONFIG_AMD_MEM_ENCRYPT */ - ret + RET SYM_FUNC_END(get_sev_encryption_bit) /**
@@ -92,7 +92,7 @@ SYM_CODE_START_LOCAL(sev_es_req_cpuid) /* All good - return success */ xorl %eax, %eax 1: - ret + RET 2: movl $-1, %eax jmp 1b
@@ -221,7 +221,7 @@ SYM_FUNC_START(set_sev_encryption_mask) #endif xor %rax, %rax - ret + RET SYM_FUNC_END(set_sev_encryption_mask) .data
arch/x86/crypto/aegis128-aesni-asm.S +24 -24
181 lines not on this page · raw message
arch/x86/crypto/aes_ctrby8_avx-x86_64.S +1 -1
@@ -525,7 +525,7 @@ /* return updated IV */ vpshufb xbyteswap, xcounter, xcounter vmovdqu xcounter, (p_iv) - ret + RET .endm /*
arch/x86/crypto/aesni-intel_asm.S +28 -28
224 lines not on this page · raw message
arch/x86/crypto/aesni-intel_avx-x86_64.S +20 -20
@@ -1767,7 +1767,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen2) FUNC_SAVE INIT GHASH_MUL_AVX, PRECOMPUTE_AVX FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_init_avx_gen2) ###############################################################################
@@ -1788,15 +1788,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_ # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11 FUNC_RESTORE - ret + RET key_128_enc_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9 FUNC_RESTORE - ret + RET key_256_enc_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2) ###############################################################################
@@ -1817,15 +1817,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_ # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11 FUNC_RESTORE - ret + RET key_128_dec_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9 FUNC_RESTORE - ret + RET key_256_dec_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2) ###############################################################################
@@ -1846,15 +1846,15 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_ge # must be 192 GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4 FUNC_RESTORE - ret + RET key_128_finalize: GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4 FUNC_RESTORE - ret + RET key_256_finalize: GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_finalize_avx_gen2) ###############################################################################
@@ -2735,7 +2735,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen4) FUNC_SAVE INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_init_avx_gen4) ###############################################################################
@@ -2756,15 +2756,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_ # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11 FUNC_RESTORE - ret + RET key_128_enc_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9 FUNC_RESTORE - ret + RET key_256_enc_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4) ###############################################################################
@@ -2785,15 +2785,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_ # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11 FUNC_RESTORE - ret + RET key_128_dec_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9 FUNC_RESTORE - ret + RET key_256_dec_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4) ###############################################################################
@@ -2814,13 +2814,13 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_ge # must be 192 GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4 FUNC_RESTORE - ret + RET key_128_finalize4: GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4 FUNC_RESTORE - ret + RET key_256_finalize4: GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
arch/x86/crypto/blake2s-core.S +2 -2
@@ -171,7 +171,7 @@ SYM_FUNC_START(blake2s_compress_ssse3) movdqu %xmm1,0x10(%rdi) movdqu %xmm14,0x20(%rdi) .Lendofloop: - ret + RET SYM_FUNC_END(blake2s_compress_ssse3) #ifdef CONFIG_AS_AVX512
@@ -251,6 +251,6 @@ SYM_FUNC_START(blake2s_compress_avx512) vmovdqu %xmm1,0x10(%rdi) vmovdqu %xmm4,0x20(%rdi) vzeroupper - retq + RET SYM_FUNC_END(blake2s_compress_avx512) #endif /* CONFIG_AS_AVX512 */
arch/x86/crypto/blowfish-x86_64-asm_64.S +6 -6
@@ -135,10 +135,10 @@ SYM_FUNC_START(__blowfish_enc_blk) jnz .L__enc_xor; write_block(); - ret; + RET; .L__enc_xor: xor_block(); - ret; + RET; SYM_FUNC_END(__blowfish_enc_blk) SYM_FUNC_START(blowfish_dec_blk)
@@ -170,7 +170,7 @@ SYM_FUNC_START(blowfish_dec_blk) movq %r11, %r12; - ret; + RET; SYM_FUNC_END(blowfish_dec_blk) /**********************************************************************
@@ -322,14 +322,14 @@ SYM_FUNC_START(__blowfish_enc_blk_4way) popq %rbx; popq %r12; - ret; + RET; .L__enc_xor4: xor_block4(); popq %rbx; popq %r12; - ret; + RET; SYM_FUNC_END(__blowfish_enc_blk_4way) SYM_FUNC_START(blowfish_dec_blk_4way)
@@ -364,5 +364,5 @@ SYM_FUNC_START(blowfish_dec_blk_4way) popq %rbx; popq %r12; - ret; + RET; SYM_FUNC_END(blowfish_dec_blk_4way)
arch/x86/crypto/camellia-aesni-avx-asm_64.S +7 -7
@@ -192,7 +192,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x0_x1_x2_ roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %rcx, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8
@@ -200,7 +200,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x4_x5_x6_ roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, %rax, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /*
@@ -778,7 +778,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk1 %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32:
@@ -865,7 +865,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk1 %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32:
@@ -906,7 +906,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_enc_16way) SYM_FUNC_START(camellia_ecb_dec_16way)
@@ -936,7 +936,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_dec_16way) SYM_FUNC_START(camellia_cbc_dec_16way)
@@ -987,5 +987,5 @@ SYM_FUNC_START(camellia_cbc_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_cbc_dec_16way)
arch/x86/crypto/camellia-aesni-avx2-asm_64.S +7 -7
@@ -226,7 +226,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_ roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rcx, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8
@@ -234,7 +234,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_ roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, %rax, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /*
@@ -814,7 +814,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk3 %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32:
@@ -901,7 +901,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk3 %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32:
@@ -946,7 +946,7 @@ SYM_FUNC_START(camellia_ecb_enc_32way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_enc_32way) SYM_FUNC_START(camellia_ecb_dec_32way)
@@ -980,7 +980,7 @@ SYM_FUNC_START(camellia_ecb_dec_32way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_dec_32way) SYM_FUNC_START(camellia_cbc_dec_32way)
@@ -1047,5 +1047,5 @@ SYM_FUNC_START(camellia_cbc_dec_32way) addq $(16 * 32), %rsp; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_cbc_dec_32way)
arch/x86/crypto/camellia-x86_64-asm_64.S +6 -6
@@ -213,13 +213,13 @@ SYM_FUNC_START(__camellia_enc_blk) enc_outunpack(mov, RT1); movq RR12, %r12; - ret; + RET; .L__enc_xor: enc_outunpack(xor, RT1); movq RR12, %r12; - ret; + RET; SYM_FUNC_END(__camellia_enc_blk) SYM_FUNC_START(camellia_dec_blk)
@@ -257,7 +257,7 @@ SYM_FUNC_START(camellia_dec_blk) dec_outunpack(); movq RR12, %r12; - ret; + RET; SYM_FUNC_END(camellia_dec_blk) /**********************************************************************
@@ -448,14 +448,14 @@ SYM_FUNC_START(__camellia_enc_blk_2way) movq RR12, %r12; popq %rbx; - ret; + RET; .L__enc2_xor: enc_outunpack2(xor, RT2); movq RR12, %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(__camellia_enc_blk_2way) SYM_FUNC_START(camellia_dec_blk_2way)
@@ -495,5 +495,5 @@ SYM_FUNC_START(camellia_dec_blk_2way) movq RR12, %r12; movq RXOR, %rbx; - ret; + RET; SYM_FUNC_END(camellia_dec_blk_2way)
arch/x86/crypto/cast5-avx-x86_64-asm_64.S +6 -6
@@ -279,7 +279,7 @@ SYM_FUNC_START_LOCAL(__cast5_enc_blk16) outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; SYM_FUNC_END(__cast5_enc_blk16) .align 16
@@ -352,7 +352,7 @@ SYM_FUNC_START_LOCAL(__cast5_dec_blk16) outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; .L__skip_dec: vpsrldq $4, RKR, RKR;
@@ -393,7 +393,7 @@ SYM_FUNC_START(cast5_ecb_enc_16way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ecb_enc_16way) SYM_FUNC_START(cast5_ecb_dec_16way)
@@ -431,7 +431,7 @@ SYM_FUNC_START(cast5_ecb_dec_16way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ecb_dec_16way) SYM_FUNC_START(cast5_cbc_dec_16way)
@@ -483,7 +483,7 @@ SYM_FUNC_START(cast5_cbc_dec_16way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_cbc_dec_16way) SYM_FUNC_START(cast5_ctr_16way)
@@ -559,5 +559,5 @@ SYM_FUNC_START(cast5_ctr_16way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ctr_16way)
arch/x86/crypto/cast6-avx-x86_64-asm_64.S +5 -5
@@ -289,7 +289,7 @@ SYM_FUNC_START_LOCAL(__cast6_enc_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; SYM_FUNC_END(__cast6_enc_blk8) .align 8
@@ -336,7 +336,7 @@ SYM_FUNC_START_LOCAL(__cast6_dec_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; SYM_FUNC_END(__cast6_dec_blk8) SYM_FUNC_START(cast6_ecb_enc_8way)
@@ -359,7 +359,7 @@ SYM_FUNC_START(cast6_ecb_enc_8way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_ecb_enc_8way) SYM_FUNC_START(cast6_ecb_dec_8way)
@@ -382,7 +382,7 @@ SYM_FUNC_START(cast6_ecb_dec_8way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_ecb_dec_8way) SYM_FUNC_START(cast6_cbc_dec_8way)
@@ -408,5 +408,5 @@ SYM_FUNC_START(cast6_cbc_dec_8way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_cbc_dec_8way)
arch/x86/crypto/chacha-avx2-x86_64.S +3 -3
@@ -193,7 +193,7 @@ SYM_FUNC_START(chacha_2block_xor_avx2) .Ldone2: vzeroupper - ret + RET .Lxorpart2: # xor remaining bytes from partial register into output
@@ -498,7 +498,7 @@ SYM_FUNC_START(chacha_4block_xor_avx2) .Ldone4: vzeroupper - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
@@ -992,7 +992,7 @@ SYM_FUNC_START(chacha_8block_xor_avx2) .Ldone8: vzeroupper lea -8(%r10),%rsp - ret + RET .Lxorpart8: # xor remaining bytes from partial register into output
arch/x86/crypto/chacha-avx512vl-x86_64.S +3 -3
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_2block_xor_avx512v .Ldone2: vzeroupper - ret + RET .Lxorpart2: # xor remaining bytes from partial register into output
@@ -432,7 +432,7 @@ SYM_FUNC_START(chacha_4block_xor_avx512v .Ldone4: vzeroupper - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
@@ -812,7 +812,7 @@ SYM_FUNC_START(chacha_8block_xor_avx512v .Ldone8: vzeroupper - ret + RET .Lxorpart8: # xor remaining bytes from partial register into output
arch/x86/crypto/chacha-ssse3-x86_64.S +4 -4
@@ -108,7 +108,7 @@ SYM_FUNC_START_LOCAL(chacha_permute) sub $2,%r8d jnz .Ldoubleround - ret + RET SYM_FUNC_END(chacha_permute) SYM_FUNC_START(chacha_block_xor_ssse3)
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_block_xor_ssse3) .Ldone: FRAME_END - ret + RET .Lxorpart: # xor remaining bytes from partial register into output
@@ -217,7 +217,7 @@ SYM_FUNC_START(hchacha_block_ssse3) movdqu %xmm3,0x10(%rsi) FRAME_END - ret + RET SYM_FUNC_END(hchacha_block_ssse3) SYM_FUNC_START(chacha_4block_xor_ssse3)
@@ -762,7 +762,7 @@ SYM_FUNC_START(chacha_4block_xor_ssse3) .Ldone4: lea -8(%r10),%rsp - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
arch/x86/crypto/crc32-pclmul_asm.S +1 -1
@@ -236,5 +236,5 @@ loop_16:/* Folding rest buffer into 128b pxor %xmm2, %xmm1 pextrd $0x01, %xmm1, %eax - ret + RET SYM_FUNC_END(crc32_pclmul_le_16)
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +1 -1
@@ -306,7 +306,7 @@ less_than_1: # Length should be zero popq %rsi popq %rdi popq %rbx - ret + RET SYM_FUNC_END(crc_pcl) .section .rodata, "a", @progbits
arch/x86/crypto/crct10dif-pcl-asm_64.S +1 -1
@@ -257,7 +257,7 @@ SYM_FUNC_START(crc_t10dif_pcl) # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0. pextrw $0, %xmm0, %eax - ret + RET .align 16 .Lless_than_256_bytes:
arch/x86/crypto/des3_ede-asm_64.S +2 -2
@@ -243,7 +243,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk popq %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(des3_ede_x86_64_crypt_blk) /***********************************************************************
@@ -528,7 +528,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk popq %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way) .section .rodata, "a", @progbits
arch/x86/crypto/ghash-clmulni-intel_asm.S +3 -3
@@ -85,7 +85,7 @@ SYM_FUNC_START_LOCAL(__clmul_gf128mul_bl psrlq $1, T2 pxor T2, T1 pxor T1, DATA - ret + RET SYM_FUNC_END(__clmul_gf128mul_ble) /* void clmul_ghash_mul(char *dst, const u128 *shash) */
@@ -99,7 +99,7 @@ SYM_FUNC_START(clmul_ghash_mul) pshufb BSWAP, DATA movups DATA, (%rdi) FRAME_END - ret + RET SYM_FUNC_END(clmul_ghash_mul) /*
@@ -128,5 +128,5 @@ SYM_FUNC_START(clmul_ghash_update) movups DATA, (%rdi) .Lupdate_just_ret: FRAME_END - ret + RET SYM_FUNC_END(clmul_ghash_update)
arch/x86/crypto/nh-avx2-x86_64.S +1 -1
@@ -153,5 +153,5 @@ SYM_FUNC_START(nh_avx2) vpaddq T1, T0, T0 vpaddq T4, T0, T0 vmovdqu T0, (HASH) - ret + RET SYM_FUNC_END(nh_avx2)
arch/x86/crypto/nh-sse2-x86_64.S +1 -1
@@ -119,5 +119,5 @@ SYM_FUNC_START(nh_sse2) paddq PASS2_SUMS, T1 movdqu T0, 0x00(HASH) movdqu T1, 0x10(HASH) - ret + RET SYM_FUNC_END(nh_sse2)
arch/x86/crypto/serpent-avx-x86_64-asm_64.S +5 -5
@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk8_ write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk8_avx) .align 8
@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_ write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_dec_blk8_avx) SYM_FUNC_START(serpent_ecb_enc_8way_avx)
@@ -673,7 +673,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx) store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_enc_8way_avx) SYM_FUNC_START(serpent_ecb_dec_8way_avx)
@@ -691,7 +691,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx) store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_dec_8way_avx) SYM_FUNC_START(serpent_cbc_dec_8way_avx)
@@ -709,5 +709,5 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx) store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_cbc_dec_8way_avx)
arch/x86/crypto/serpent-avx2-asm_64.S +5 -5
@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk16 write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk16) .align 8
@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk16 write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_dec_blk16) SYM_FUNC_START(serpent_ecb_enc_16way)
@@ -677,7 +677,7 @@ SYM_FUNC_START(serpent_ecb_enc_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_enc_16way) SYM_FUNC_START(serpent_ecb_dec_16way)
@@ -699,7 +699,7 @@ SYM_FUNC_START(serpent_ecb_dec_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_dec_16way) SYM_FUNC_START(serpent_cbc_dec_16way)
@@ -722,5 +722,5 @@ SYM_FUNC_START(serpent_cbc_dec_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_cbc_dec_16way)
arch/x86/crypto/serpent-sse2-i586-asm_32.S +3 -3
@@ -553,12 +553,12 @@ SYM_FUNC_START(__serpent_enc_blk_4way) write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; .L__enc_xor4: xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk_4way) SYM_FUNC_START(serpent_dec_blk_4way)
@@ -612,5 +612,5 @@ SYM_FUNC_START(serpent_dec_blk_4way) movl arg_dst(%esp), %eax; write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA); - ret; + RET; SYM_FUNC_END(serpent_dec_blk_4way)
arch/x86/crypto/serpent-sse2-x86_64-asm_64.S +3 -3
@@ -675,13 +675,13 @@ SYM_FUNC_START(__serpent_enc_blk_8way) write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; .L__enc_xor8: xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk_8way) SYM_FUNC_START(serpent_dec_blk_8way)
@@ -735,5 +735,5 @@ SYM_FUNC_START(serpent_dec_blk_8way) write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(serpent_dec_blk_8way)
arch/x86/crypto/sha1_avx2_x86_64_asm.S +1 -1
@@ -674,7 +674,7 @@ pop %r12 pop %rbx - ret + RET SYM_FUNC_END(\name) .endm
arch/x86/crypto/sha1_ni_asm.S +1 -1
@@ -290,7 +290,7 @@ SYM_FUNC_START(sha1_ni_transform) mov %rbp, %rsp pop %rbp - ret + RET SYM_FUNC_END(sha1_ni_transform) .section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
arch/x86/crypto/sha1_ssse3_asm.S +1 -1
@@ -99,7 +99,7 @@ pop %rbp pop %r12 pop %rbx - ret + RET SYM_FUNC_END(\name) .endm
arch/x86/crypto/sha256-avx-asm.S +1 -1
@@ -458,7 +458,7 @@ SYM_FUNC_START(sha256_transform_avx) popq %r13 popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_avx) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha256-avx2-asm.S +1 -1
@@ -710,7 +710,7 @@ SYM_FUNC_START(sha256_transform_rorx) popq %r13 popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_rorx) .section .rodata.cst512.K256, "aM", @progbits, 512
arch/x86/crypto/sha256-ssse3-asm.S +1 -1
@@ -472,7 +472,7 @@ SYM_FUNC_START(sha256_transform_ssse3) popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_ssse3) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha256_ni_asm.S +1 -1
@@ -326,7 +326,7 @@ SYM_FUNC_START(sha256_ni_transform) .Ldone_hash: - ret + RET SYM_FUNC_END(sha256_ni_transform) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha512-avx-asm.S +1 -1
@@ -361,7 +361,7 @@ SYM_FUNC_START(sha512_transform_avx) pop %rbx nowork: - ret + RET SYM_FUNC_END(sha512_transform_avx) ########################################################################
arch/x86/crypto/sha512-avx2-asm.S +1 -1
@@ -679,7 +679,7 @@ SYM_FUNC_START(sha512_transform_rorx) pop %r12 pop %rbx - ret + RET SYM_FUNC_END(sha512_transform_rorx) ########################################################################
arch/x86/crypto/sha512-ssse3-asm.S +1 -1
@@ -363,7 +363,7 @@ SYM_FUNC_START(sha512_transform_ssse3) pop %rbx nowork: - ret + RET SYM_FUNC_END(sha512_transform_ssse3) ########################################################################
arch/x86/crypto/sm4-aesni-avx-asm_64.S +6 -6
@@ -246,7 +246,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt4) .Lblk4_store_output_done: vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_crypt4) .align 8
@@ -356,7 +356,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk8) vpshufb RTMP2, RB3, RB3; FRAME_END - ret; + RET; SYM_FUNC_END(__sm4_crypt_blk8) /*
@@ -412,7 +412,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt8) .Lblk8_store_output_done: vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_crypt8) /*
@@ -487,7 +487,7 @@ SYM_FUNC_START(sm4_aesni_avx_ctr_enc_blk vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8) /*
@@ -537,7 +537,7 @@ SYM_FUNC_START(sm4_aesni_avx_cbc_dec_blk vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8) /*
@@ -590,5 +590,5 @@ SYM_FUNC_START(sm4_aesni_avx_cfb_dec_blk vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8)
arch/x86/crypto/sm4-aesni-avx2-asm_64.S +4 -4
@@ -268,7 +268,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk16) vpshufb RTMP2, RB3, RB3; FRAME_END - ret; + RET; SYM_FUNC_END(__sm4_crypt_blk16) #define inc_le128(x, minus_one, tmp) \
@@ -387,7 +387,7 @@ SYM_FUNC_START(sm4_aesni_avx2_ctr_enc_bl vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16) /*
@@ -441,7 +441,7 @@ SYM_FUNC_START(sm4_aesni_avx2_cbc_dec_bl vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16) /*
@@ -497,5 +497,5 @@ SYM_FUNC_START(sm4_aesni_avx2_cfb_dec_bl vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
arch/x86/crypto/twofish-avx-x86_64-asm_64.S +5 -5
@@ -267,7 +267,7 @@ SYM_FUNC_START_LOCAL(__twofish_enc_blk8) outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); - ret; + RET; SYM_FUNC_END(__twofish_enc_blk8) .align 8
@@ -307,7 +307,7 @@ SYM_FUNC_START_LOCAL(__twofish_dec_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); - ret; + RET; SYM_FUNC_END(__twofish_dec_blk8) SYM_FUNC_START(twofish_ecb_enc_8way)
@@ -327,7 +327,7 @@ SYM_FUNC_START(twofish_ecb_enc_8way) store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); FRAME_END - ret; + RET; SYM_FUNC_END(twofish_ecb_enc_8way) SYM_FUNC_START(twofish_ecb_dec_8way)
@@ -347,7 +347,7 @@ SYM_FUNC_START(twofish_ecb_dec_8way) store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; SYM_FUNC_END(twofish_ecb_dec_8way) SYM_FUNC_START(twofish_cbc_dec_8way)
@@ -372,5 +372,5 @@ SYM_FUNC_START(twofish_cbc_dec_8way) popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(twofish_cbc_dec_8way)
arch/x86/crypto/twofish-i586-asm_32.S +2 -2
@@ -260,7 +260,7 @@ SYM_FUNC_START(twofish_enc_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET SYM_FUNC_END(twofish_enc_blk) SYM_FUNC_START(twofish_dec_blk)
@@ -317,5 +317,5 @@ SYM_FUNC_START(twofish_dec_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET SYM_FUNC_END(twofish_dec_blk)
arch/x86/crypto/twofish-x86_64-asm_64-3way.S +3 -3
@@ -258,7 +258,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; .L__enc_xor3: outunpack_enc3(xor);
@@ -266,7 +266,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; SYM_FUNC_END(__twofish_enc_blk_3way) SYM_FUNC_START(twofish_dec_blk_3way)
@@ -301,5 +301,5 @@ SYM_FUNC_START(twofish_dec_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; SYM_FUNC_END(twofish_dec_blk_3way)
arch/x86/crypto/twofish-x86_64-asm_64.S +2 -2
@@ -252,7 +252,7 @@ SYM_FUNC_START(twofish_enc_blk) popq R1 movl $1,%eax - ret + RET SYM_FUNC_END(twofish_enc_blk) SYM_FUNC_START(twofish_dec_blk)
@@ -304,5 +304,5 @@ SYM_FUNC_START(twofish_dec_blk) popq R1 movl $1,%eax - ret + RET SYM_FUNC_END(twofish_dec_blk)
arch/x86/entry/entry_32.S +1 -1
@@ -740,7 +740,7 @@ SYM_FUNC_START(schedule_tail_wrapper) popl %eax FRAME_END - ret + RET SYM_FUNC_END(schedule_tail_wrapper) .popsection
arch/x86/entry/entry_64.S +6 -6
@@ -734,7 +734,7 @@ SYM_FUNC_START(asm_load_gs_index) 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs FRAME_END - ret + RET SYM_FUNC_END(asm_load_gs_index) EXPORT_SYMBOL(asm_load_gs_index)
@@ -885,7 +885,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) * is needed here. */ SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx - ret + RET .Lparanoid_entry_checkgs: /* EBX = 1 -> kernel GSBASE active, no restore required */
@@ -898,7 +898,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) rdmsr testl %edx, %edx jns .Lparanoid_entry_swapgs - ret + RET .Lparanoid_entry_swapgs: swapgs
@@ -912,7 +912,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) /* EBX = 0 -> SWAPGS required on exit */ xorl %ebx, %ebx - ret + RET SYM_CODE_END(paranoid_entry) /*
@@ -991,12 +991,12 @@ SYM_CODE_START_LOCAL(error_entry) movq %rax, %rsp /* switch stack */ ENCODE_FRAME_POINTER pushq %r12 - ret + RET .Lerror_entry_done_lfence: FENCE_SWAPGS_KERNEL_ENTRY .Lerror_entry_done: - ret + RET /* * There are two places in the kernel that can potentially fault with
arch/x86/entry/thunk_32.S +1 -1
@@ -24,7 +24,7 @@ SYM_CODE_START_NOALIGN(\name) popl %edx popl %ecx popl %eax - ret + RET _ASM_NOKPROBE(\name) SYM_CODE_END(\name) .endm
arch/x86/entry/thunk_64.S +1 -1
@@ -50,7 +50,7 @@ SYM_CODE_START_LOCAL_NOALIGN(__thunk_res popq %rsi popq %rdi popq %rbp - ret + RET _ASM_NOKPROBE(__thunk_restore) SYM_CODE_END(__thunk_restore) #endif
arch/x86/entry/vdso/vdso32/system_call.S +1 -1
@@ -78,7 +78,7 @@ SYM_INNER_LABEL(int80_landing_pad, SYM_L popl %ecx CFI_RESTORE ecx CFI_ADJUST_CFA_OFFSET -4 - ret + RET CFI_ENDPROC .size __kernel_vsyscall,.-__kernel_vsyscall
arch/x86/entry/vdso/vsgx.S +1 -1
@@ -81,7 +81,7 @@ SYM_FUNC_START(__vdso_sgx_enter_enclave) pop %rbx leave .cfi_def_cfa %rsp, 8 - ret + RET /* The out-of-line code runs with the pre-leave stack frame. */ .cfi_def_cfa %rbp, 16
arch/x86/entry/vsyscall/vsyscall_emu_64.S +3 -3
@@ -19,17 +19,17 @@ __PAGE_ALIGNED_DATA mov $__NR_gettimeofday, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_time, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_getcpu, %rax syscall - ret + RET .balign 4096, 0xcc
arch/x86/kernel/acpi/wakeup_32.S +3 -3
@@ -60,7 +60,7 @@ SYM_CODE_END(wakeup_pmode_return) popl saved_context_eflags movl $ret_point, saved_eip - ret + RET restore_registers:
@@ -70,7 +70,7 @@ SYM_CODE_END(wakeup_pmode_return) movl saved_context_edi, %edi pushl saved_context_eflags popfl - ret + RET SYM_CODE_START(do_suspend_lowlevel) call save_processor_state
@@ -86,7 +86,7 @@ SYM_CODE_START(do_suspend_lowlevel) ret_point: call restore_registers call restore_processor_state - ret + RET SYM_CODE_END(do_suspend_lowlevel) .data
arch/x86/kernel/ftrace_32.S +3 -3
@@ -19,7 +19,7 @@ #endif SYM_FUNC_START(__fentry__) - ret + RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__)
@@ -84,7 +84,7 @@ SYM_CODE_START(ftrace_caller) /* This is weak to keep gas from relaxing the jumps */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) - ret + RET SYM_CODE_END(ftrace_caller) SYM_CODE_START(ftrace_regs_caller)
@@ -177,7 +177,7 @@ SYM_CODE_START(ftrace_graph_caller) popl %edx popl %ecx popl %eax - ret + RET SYM_CODE_END(ftrace_graph_caller) .globl return_to_handler
arch/x86/kernel/ftrace_64.S +4 -4
@@ -132,7 +132,7 @@ #ifdef CONFIG_DYNAMIC_FTRACE SYM_FUNC_START(__fentry__) - retq + RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__)
@@ -176,11 +176,11 @@ SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the retq for trampolines. + * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC - retq + RET SYM_FUNC_END(ftrace_epilogue) SYM_FUNC_START(ftrace_regs_caller)
@@ -284,7 +284,7 @@ SYM_FUNC_START(__fentry__) jnz trace SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) - retq + RET trace: /* save_mcount_regs fills in first two parameters */
arch/x86/kernel/head_32.S +1 -1
@@ -340,7 +340,7 @@ SYM_FUNC_END(startup_32_smp) __INIT setup_once: andl $0,setup_once_ref /* Once is enough, thanks */ - ret + RET SYM_FUNC_START(early_idt_handler_array) # 36(%esp) %eflags
arch/x86/kernel/irqflags.S +1 -1
@@ -11,7 +11,7 @@ SYM_FUNC_START(native_save_fl) pushf pop %_ASM_AX - ret + RET SYM_FUNC_END(native_save_fl) .popsection EXPORT_SYMBOL(native_save_fl)
arch/x86/kernel/relocate_kernel_32.S +5 -5
@@ -91,7 +91,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) movl %edi, %eax addl $(identity_mapped - relocate_kernel), %eax pushl %eax - ret + RET SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
@@ -159,7 +159,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_ma xorl %edx, %edx xorl %esi, %esi xorl %ebp, %ebp - ret + RET 1: popl %edx movl CP_PA_SWAP_PAGE(%edi), %esp
@@ -190,7 +190,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_ma movl %edi, %eax addl $(virtual_mapped - relocate_kernel), %eax pushl %eax - ret + RET SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
@@ -208,7 +208,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_map popl %edi popl %esi popl %ebx - ret + RET SYM_CODE_END(virtual_mapped) /* Do the copies */
@@ -271,7 +271,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) popl %edi popl %ebx popl %ebp - ret + RET SYM_CODE_END(swap_pages) .globl kexec_control_code_size
arch/x86/kernel/relocate_kernel_64.S +5 -5
@@ -104,7 +104,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) /* jump to identity mapped page */ addq $(identity_mapped - relocate_kernel), %r8 pushq %r8 - ret + RET SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
@@ -191,7 +191,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_ma xorl %r14d, %r14d xorl %r15d, %r15d - ret + RET 1: popq %rdx
@@ -210,7 +210,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_ma call swap_pages movq $virtual_mapped, %rax pushq %rax - ret + RET SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
@@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_map popq %r12 popq %rbp popq %rbx - ret + RET SYM_CODE_END(virtual_mapped) /* Do the copies */
@@ -288,7 +288,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) lea PAGE_SIZE(%rax), %rsi jmp 0b 3: - ret + RET SYM_CODE_END(swap_pages) .globl kexec_control_code_size
arch/x86/kernel/sev_verify_cbit.S +1 -1
@@ -85,5 +85,5 @@ SYM_FUNC_START(sev_verify_cbit) #endif /* Return page-table pointer */ movq %rdi, %rax - ret + RET SYM_FUNC_END(sev_verify_cbit)
arch/x86/kernel/verify_cpu.S +2 -2
@@ -132,9 +132,9 @@ SYM_FUNC_START_LOCAL(verify_cpu) .Lverify_cpu_no_longmode: popf # Restore caller passed flags movl $1,%eax - ret + RET .Lverify_cpu_sse_ok: popf # Restore caller passed flags xorl %eax, %eax - ret + RET SYM_FUNC_END(verify_cpu)
arch/x86/kvm/svm/vmenter.S +2 -2
@@ -148,7 +148,7 @@ SYM_FUNC_START(__svm_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET 3: cmpb $0, kvm_rebooting jne 2b
@@ -202,7 +202,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET 3: cmpb $0, kvm_rebooting jne 2b
arch/x86/kvm/vmx/vmenter.S +7 -7
@@ -49,14 +49,14 @@ SYM_FUNC_START_LOCAL(vmx_vmenter) je 2f 1: vmresume - ret + RET 2: vmlaunch - ret + RET 3: cmpb $0, kvm_rebooting je 4f - ret + RET 4: ud2 _ASM_EXTABLE(1b, 3b)
@@ -89,7 +89,7 @@ SYM_FUNC_START(vmx_vmexit) pop %_ASM_AX .Lvmexit_skip_rsb: #endif - ret + RET SYM_FUNC_END(vmx_vmexit) /**
@@ -228,7 +228,7 @@ SYM_FUNC_START(__vmx_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */ 2: mov $1, %eax
@@ -293,7 +293,7 @@ SYM_FUNC_START(vmread_error_trampoline) pop %_ASM_AX pop %_ASM_BP - ret + RET SYM_FUNC_END(vmread_error_trampoline) SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
@@ -326,5 +326,5 @@ SYM_FUNC_START(vmx_do_interrupt_nmi_irqo */ mov %_ASM_BP, %_ASM_SP pop %_ASM_BP - ret + RET SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
arch/x86/lib/atomic64_386_32.S +1 -1
@@ -30,7 +30,7 @@ SYM_FUNC_START(atomic64_##op##_386); \ #define RET_IRQ_RESTORE \ IRQ_RESTORE v; \ - ret + RET #define v %ecx BEGIN_IRQ_SAVE(read)
arch/x86/lib/atomic64_cx8_32.S +8 -8
@@ -18,7 +18,7 @@ SYM_FUNC_START(atomic64_read_cx8) read64 %ecx - ret + RET SYM_FUNC_END(atomic64_read_cx8) SYM_FUNC_START(atomic64_set_cx8)
@@ -28,7 +28,7 @@ SYM_FUNC_START(atomic64_set_cx8) cmpxchg8b (%esi) jne 1b - ret + RET SYM_FUNC_END(atomic64_set_cx8) SYM_FUNC_START(atomic64_xchg_cx8)
@@ -37,7 +37,7 @@ SYM_FUNC_START(atomic64_xchg_cx8) cmpxchg8b (%esi) jne 1b - ret + RET SYM_FUNC_END(atomic64_xchg_cx8) .macro addsub_return func ins insc
@@ -68,7 +68,7 @@ SYM_FUNC_START(atomic64_\func\()_return_ popl %esi popl %ebx popl %ebp - ret + RET SYM_FUNC_END(atomic64_\func\()_return_cx8) .endm
@@ -93,7 +93,7 @@ SYM_FUNC_START(atomic64_\func\()_return_ movl %ebx, %eax movl %ecx, %edx popl %ebx - ret + RET SYM_FUNC_END(atomic64_\func\()_return_cx8) .endm
@@ -118,7 +118,7 @@ SYM_FUNC_START(atomic64_dec_if_positive_ movl %ebx, %eax movl %ecx, %edx popl %ebx - ret + RET SYM_FUNC_END(atomic64_dec_if_positive_cx8) SYM_FUNC_START(atomic64_add_unless_cx8)
@@ -149,7 +149,7 @@ SYM_FUNC_START(atomic64_add_unless_cx8) addl $8, %esp popl %ebx popl %ebp - ret + RET 4: cmpl %edx, 4(%esp) jne 2b
@@ -176,5 +176,5 @@ SYM_FUNC_START(atomic64_inc_not_zero_cx8 movl $1, %eax 3: popl %ebx - ret + RET SYM_FUNC_END(atomic64_inc_not_zero_cx8)
arch/x86/lib/checksum_32.S +4 -4
@@ -127,7 +127,7 @@ SYM_FUNC_START(csum_partial) 8: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #else
@@ -245,7 +245,7 @@ SYM_FUNC_START(csum_partial) 90: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #endif
@@ -371,7 +371,7 @@ EXC( movb %cl, (%edi) ) popl %esi popl %edi popl %ecx # equivalent to addl $4,%esp - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #else
@@ -447,7 +447,7 @@ EXC( movb %dl, (%edi) ) popl %esi popl %edi popl %ebx - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #undef ROUND
arch/x86/lib/clear_page_64.S +3 -3
@@ -17,7 +17,7 @@ SYM_FUNC_START(clear_page_rep) movl $4096/8,%ecx xorl %eax,%eax rep stosq - ret + RET SYM_FUNC_END(clear_page_rep) EXPORT_SYMBOL_GPL(clear_page_rep)
@@ -39,7 +39,7 @@ SYM_FUNC_START(clear_page_orig) leaq 64(%rdi),%rdi jnz .Lloop nop - ret + RET SYM_FUNC_END(clear_page_orig) EXPORT_SYMBOL_GPL(clear_page_orig)
@@ -47,6 +47,6 @@ SYM_FUNC_START(clear_page_erms) movl $4096,%ecx xorl %eax,%eax rep stosb - ret + RET SYM_FUNC_END(clear_page_erms) EXPORT_SYMBOL_GPL(clear_page_erms)
arch/x86/lib/cmpxchg16b_emu.S +2 -2
@@ -37,11 +37,11 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu) popfq mov $1, %al - ret + RET .Lnot_same: popfq xor %al,%al - ret + RET SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
arch/x86/lib/cmpxchg8b_emu.S +2 -2
@@ -32,7 +32,7 @@ SYM_FUNC_START(cmpxchg8b_emu) movl %ecx, 4(%esi) popfl - ret + RET .Lnot_same: movl (%esi), %eax
@@ -40,7 +40,7 @@ SYM_FUNC_START(cmpxchg8b_emu) movl 4(%esi), %edx popfl - ret + RET SYM_FUNC_END(cmpxchg8b_emu) EXPORT_SYMBOL(cmpxchg8b_emu)
arch/x86/lib/copy_mc_64.S +3 -3
@@ -77,7 +77,7 @@ SYM_FUNC_START(copy_mc_fragile) .L_done_memcpy_trap: xorl %eax, %eax .L_done: - ret + RET SYM_FUNC_END(copy_mc_fragile) .section .fixup, "ax"
@@ -132,7 +132,7 @@ SYM_FUNC_START(copy_mc_enhanced_fast_str rep movsb /* Copy successful. Return zero */ xorl %eax, %eax - ret + RET SYM_FUNC_END(copy_mc_enhanced_fast_string) .section .fixup, "ax"
@@ -145,7 +145,7 @@ SYM_FUNC_END(copy_mc_enhanced_fast_strin * user-copy routines. */ movq %rcx, %rax - ret + RET .previous
arch/x86/lib/copy_page_64.S +2 -2
@@ -17,7 +17,7 @@ SYM_FUNC_START(copy_page) ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD movl $4096/8, %ecx rep movsq - ret + RET SYM_FUNC_END(copy_page) EXPORT_SYMBOL(copy_page)
@@ -85,5 +85,5 @@ SYM_FUNC_START_LOCAL(copy_page_regs) movq (%rsp), %rbx movq 1*8(%rsp), %r12 addq $2*8, %rsp - ret + RET SYM_FUNC_END(copy_page_regs)
arch/x86/lib/copy_user_64.S +5 -5
@@ -105,7 +105,7 @@ SYM_FUNC_START(copy_user_generic_unrolle jnz 21b 23: xor %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 30: shll $6,%ecx
@@ -173,7 +173,7 @@ SYM_FUNC_START(copy_user_generic_string) movsb xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 11: leal (%rdx,%rcx,8),%ecx
@@ -207,7 +207,7 @@ SYM_FUNC_START(copy_user_enhanced_fast_s movsb xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 12: movl %ecx,%edx /* ecx is zerorest also */
@@ -237,7 +237,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_ 1: rep movsb 2: mov %ecx,%eax ASM_CLAC - ret + RET _ASM_EXTABLE_CPY(1b, 2b) SYM_CODE_END(.Lcopy_user_handle_tail)
@@ -348,7 +348,7 @@ SYM_FUNC_START(__copy_user_nocache) xorl %eax,%eax ASM_CLAC sfence - ret + RET .section .fixup,"ax" .L_fixup_4x8b_copy:
arch/x86/lib/csum-copy_64.S +1 -1
@@ -201,7 +201,7 @@ SYM_FUNC_START(csum_partial_copy_generic movq 3*8(%rsp), %r13 movq 4*8(%rsp), %r15 addq $5*8, %rsp - ret + RET .Lshort: movl %ecx, %r10d jmp .L1
arch/x86/lib/getuser.S +11 -11
@@ -57,7 +57,7 @@ SYM_FUNC_START(__get_user_1) 1: movzbl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_1) EXPORT_SYMBOL(__get_user_1)
@@ -71,7 +71,7 @@ SYM_FUNC_START(__get_user_2) 2: movzwl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_2) EXPORT_SYMBOL(__get_user_2)
@@ -85,7 +85,7 @@ SYM_FUNC_START(__get_user_4) 3: movl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_4) EXPORT_SYMBOL(__get_user_4)
@@ -100,7 +100,7 @@ SYM_FUNC_START(__get_user_8) 4: movq (%_ASM_AX),%rdx xor %eax,%eax ASM_CLAC - ret + RET #else LOAD_TASK_SIZE_MINUS_N(7) cmp %_ASM_DX,%_ASM_AX
@@ -112,7 +112,7 @@ SYM_FUNC_START(__get_user_8) 5: movl 4(%_ASM_AX),%ecx xor %eax,%eax ASM_CLAC - ret + RET #endif SYM_FUNC_END(__get_user_8) EXPORT_SYMBOL(__get_user_8)
@@ -124,7 +124,7 @@ SYM_FUNC_START(__get_user_nocheck_1) 6: movzbl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_1) EXPORT_SYMBOL(__get_user_nocheck_1)
@@ -134,7 +134,7 @@ SYM_FUNC_START(__get_user_nocheck_2) 7: movzwl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_2) EXPORT_SYMBOL(__get_user_nocheck_2)
@@ -144,7 +144,7 @@ SYM_FUNC_START(__get_user_nocheck_4) 8: movl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_4) EXPORT_SYMBOL(__get_user_nocheck_4)
@@ -159,7 +159,7 @@ SYM_FUNC_START(__get_user_nocheck_8) #endif xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_8) EXPORT_SYMBOL(__get_user_nocheck_8)
@@ -169,7 +169,7 @@ SYM_CODE_START_LOCAL(.Lbad_get_user_clac bad_get_user: xor %edx,%edx mov $(-EFAULT),%_ASM_AX - ret + RET SYM_CODE_END(.Lbad_get_user_clac) #ifdef CONFIG_X86_32
@@ -179,7 +179,7 @@ SYM_CODE_START_LOCAL(.Lbad_get_user_8_cl xor %edx,%edx xor %ecx,%ecx mov $(-EFAULT),%_ASM_AX - ret + RET SYM_CODE_END(.Lbad_get_user_8_clac) #endif
arch/x86/lib/hweight.S +3 -3
@@ -32,7 +32,7 @@ SYM_FUNC_START(__sw_hweight32) imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101 shrl $24, %eax # w = w_tmp >> 24 __ASM_SIZE(pop,) %__ASM_REG(dx) - ret + RET SYM_FUNC_END(__sw_hweight32) EXPORT_SYMBOL(__sw_hweight32)
@@ -65,7 +65,7 @@ SYM_FUNC_START(__sw_hweight64) popq %rdx popq %rdi - ret + RET #else /* CONFIG_X86_32 */ /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */ pushl %ecx
@@ -77,7 +77,7 @@ SYM_FUNC_START(__sw_hweight64) addl %ecx, %eax # result popl %ecx - ret + RET #endif SYM_FUNC_END(__sw_hweight64) EXPORT_SYMBOL(__sw_hweight64)
arch/x86/lib/iomap_copy_64.S +1 -1
@@ -11,5 +11,5 @@ SYM_FUNC_START(__iowrite32_copy) movl %edx,%ecx rep movsd - ret + RET SYM_FUNC_END(__iowrite32_copy)
arch/x86/lib/memcpy_64.S +6 -6
@@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy) rep movsq movl %edx, %ecx rep movsb - ret + RET SYM_FUNC_END(memcpy) SYM_FUNC_END_ALIAS(__memcpy) EXPORT_SYMBOL(memcpy)
@@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms) movq %rdi, %rax movq %rdx, %rcx rep movsb - ret + RET SYM_FUNC_END(memcpy_erms) SYM_FUNC_START_LOCAL(memcpy_orig)
@@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq %r9, 1*8(%rdi) movq %r10, -2*8(%rdi, %rdx) movq %r11, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_16bytes: cmpl $8, %edx
@@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq -1*8(%rsi, %rdx), %r9 movq %r8, 0*8(%rdi) movq %r9, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_8bytes: cmpl $4, %edx
@@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movl -4(%rsi, %rdx), %r8d movl %ecx, (%rdi) movl %r8d, -4(%rdi, %rdx) - retq + RET .p2align 4 .Lless_3bytes: subl $1, %edx
@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movb %cl, (%rdi) .Lend: - retq + RET SYM_FUNC_END(memcpy_orig) .popsection
arch/x86/lib/memmove_64.S +2 -2
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) /* FSRM implies ERMS => no length checks, do the copy directly */ .Lmemmove_begin_forward: ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS + ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS /* * movsq instruction have many startup latency
@@ -205,7 +205,7 @@ SYM_FUNC_START(__memmove) movb (%rsi), %r11b movb %r11b, (%rdi) 13: - retq + RET SYM_FUNC_END(__memmove) SYM_FUNC_END_ALIAS(memmove) EXPORT_SYMBOL(__memmove)
arch/x86/lib/memset_64.S +3 -3
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memset) movl %edx,%ecx rep stosb movq %r9,%rax - ret + RET SYM_FUNC_END(__memset) SYM_FUNC_END_ALIAS(memset) EXPORT_SYMBOL(memset)
@@ -63,7 +63,7 @@ SYM_FUNC_START_LOCAL(memset_erms) movq %rdx,%rcx rep stosb movq %r9,%rax - ret + RET SYM_FUNC_END(memset_erms) SYM_FUNC_START_LOCAL(memset_orig)
@@ -125,7 +125,7 @@ SYM_FUNC_START_LOCAL(memset_orig) .Lende: movq %r10,%rax - ret + RET .Lbad_alignment: cmpq $7,%rdx
arch/x86/lib/msr-reg.S +2 -2
@@ -35,7 +35,7 @@ SYM_FUNC_START(\op\()_safe_regs) movl %edi, 28(%r10) popq %r12 popq %rbx - ret + RET 3: movl $-EIO, %r11d jmp 2b
@@ -77,7 +77,7 @@ SYM_FUNC_START(\op\()_safe_regs) popl %esi popl %ebp popl %ebx - ret + RET 3: movl $-EIO, 4(%esp) jmp 2b
arch/x86/lib/putuser.S +3 -3
@@ -52,7 +52,7 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SY 1: movb %al,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_1) EXPORT_SYMBOL(__put_user_1) EXPORT_SYMBOL(__put_user_nocheck_1)
@@ -66,7 +66,7 @@ SYM_INNER_LABEL(__put_user_nocheck_2, SY 2: movw %ax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2) EXPORT_SYMBOL(__put_user_nocheck_2)
@@ -80,7 +80,7 @@ SYM_INNER_LABEL(__put_user_nocheck_4, SY 3: movl %eax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_4) EXPORT_SYMBOL(__put_user_4) EXPORT_SYMBOL(__put_user_nocheck_4)
arch/x86/lib/retpoline.S +1 -1
@@ -23,7 +23,7 @@ .Ldo_rop_\@: mov %\reg, (%_ASM_SP) UNWIND_HINT_FUNC - ret + RET .endm .macro THUNK reg
arch/x86/math-emu/div_Xsig.S +1 -1
@@ -341,7 +341,7 @@ SYM_FUNC_START(div_Xsig) popl %esi leave - ret + RET #ifdef PARANOID
arch/x86/math-emu/div_small.S +1 -1
@@ -44,5 +44,5 @@ SYM_FUNC_START(FPU_div_small) popl %esi leave - ret + RET SYM_FUNC_END(FPU_div_small)
arch/x86/math-emu/mul_Xsig.S +3 -3
@@ -62,7 +62,7 @@ SYM_FUNC_START(mul32_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul32_Xsig)
@@ -115,7 +115,7 @@ SYM_FUNC_START(mul64_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul64_Xsig)
@@ -175,5 +175,5 @@ SYM_FUNC_START(mul_Xsig_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul_Xsig_Xsig)
arch/x86/math-emu/polynom_Xsig.S +1 -1
@@ -133,5 +133,5 @@ SYM_FUNC_START(polynomial_Xsig) popl %edi popl %esi leave - ret + RET SYM_FUNC_END(polynomial_Xsig)
arch/x86/math-emu/reg_norm.S +3 -3
@@ -72,7 +72,7 @@ SYM_FUNC_START(FPU_normalize) L_exit: popl %ebx leave - ret + RET L_zero:
@@ -138,7 +138,7 @@ SYM_FUNC_START(FPU_normalize_nuo) popl %ebx leave - ret + RET L_exit_nuo_zero: movl TAG_Zero,%eax
@@ -146,5 +146,5 @@ SYM_FUNC_START(FPU_normalize_nuo) popl %ebx leave - ret + RET SYM_FUNC_END(FPU_normalize_nuo)
arch/x86/math-emu/reg_round.S +1 -1
@@ -437,7 +437,7 @@ LGreater_Half_53: /* Rounding: increme popl %edi popl %esi leave - ret + RET /*
arch/x86/math-emu/reg_u_add.S +1 -1
@@ -164,6 +164,6 @@ SYM_FUNC_START(FPU_u_add) popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_add)
arch/x86/math-emu/reg_u_div.S +1 -1
@@ -468,7 +468,7 @@ SYM_FUNC_START(FPU_u_div) popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_div)
arch/x86/math-emu/reg_u_mul.S +1 -1
@@ -144,7 +144,7 @@ SYM_FUNC_START(FPU_u_mul) popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_mul)
arch/x86/math-emu/reg_u_sub.S +1 -1
@@ -270,5 +270,5 @@ SYM_FUNC_START(FPU_u_sub) popl %edi popl %esi leave - ret + RET SYM_FUNC_END(FPU_u_sub)
arch/x86/math-emu/round_Xsig.S +2 -2
@@ -78,7 +78,7 @@ SYM_FUNC_START(round_Xsig) popl %esi popl %ebx leave - ret + RET SYM_FUNC_END(round_Xsig)
@@ -138,5 +138,5 @@ SYM_FUNC_START(norm_Xsig) popl %esi popl %ebx leave - ret + RET SYM_FUNC_END(norm_Xsig)
arch/x86/math-emu/shr_Xsig.S +4 -4
@@ -45,7 +45,7 @@ SYM_FUNC_START(shr_Xsig) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx
@@ -61,7 +61,7 @@ SYM_FUNC_START(shr_Xsig) movl $0,8(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx
@@ -76,7 +76,7 @@ SYM_FUNC_START(shr_Xsig) movl %edx,8(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax
@@ -85,5 +85,5 @@ SYM_FUNC_START(shr_Xsig) movl %eax,8(%esi) popl %esi leave - ret + RET SYM_FUNC_END(shr_Xsig)
arch/x86/math-emu/wm_shrx.S +8 -8
@@ -55,7 +55,7 @@ SYM_FUNC_START(FPU_shrx) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx
@@ -70,7 +70,7 @@ SYM_FUNC_START(FPU_shrx) movl $0,4(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx
@@ -84,7 +84,7 @@ SYM_FUNC_START(FPU_shrx) movl %edx,4(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax
@@ -92,7 +92,7 @@ SYM_FUNC_START(FPU_shrx) movl %eax,4(%esi) popl %esi leave - ret + RET SYM_FUNC_END(FPU_shrx)
@@ -146,7 +146,7 @@ SYM_FUNC_START(FPU_shrxs) popl %ebx popl %esi leave - ret + RET /* Shift by [0..31] bits */ Ls_less_than_32:
@@ -163,7 +163,7 @@ SYM_FUNC_START(FPU_shrxs) popl %ebx popl %esi leave - ret + RET /* Shift by [64..95] bits */ Ls_more_than_63:
@@ -189,7 +189,7 @@ SYM_FUNC_START(FPU_shrxs) popl %ebx popl %esi leave - ret + RET Ls_more_than_95: /* Shift by [96..inf) bits */
@@ -203,5 +203,5 @@ SYM_FUNC_START(FPU_shrxs) popl %ebx popl %esi leave - ret + RET SYM_FUNC_END(FPU_shrxs)
arch/x86/mm/mem_encrypt_boot.S +2 -2
@@ -65,7 +65,7 @@ SYM_FUNC_START(sme_encrypt_execute) movq %rbp, %rsp /* Restore original stack pointer */ pop %rbp - ret + RET SYM_FUNC_END(sme_encrypt_execute) SYM_FUNC_START(__enc_copy)
@@ -151,6 +151,6 @@ SYM_FUNC_START(__enc_copy) pop %r12 pop %r15 - ret + RET .L__enc_copy_end: SYM_FUNC_END(__enc_copy)
arch/x86/platform/efi/efi_stub_32.S +1 -1
@@ -56,5 +56,5 @@ SYM_FUNC_START(efi_call_svam) movl 16(%esp), %ebx leave - ret + RET SYM_FUNC_END(efi_call_svam)
arch/x86/platform/efi/efi_stub_64.S +1 -1
@@ -23,5 +23,5 @@ SYM_FUNC_START(__efi_call) mov %rsi, %rcx CALL_NOSPEC rdi leave - ret + RET SYM_FUNC_END(__efi_call)
arch/x86/platform/efi/efi_thunk_64.S +1 -1
@@ -63,7 +63,7 @@ SYM_CODE_START(__efi64_thunk) 1: movq 24(%rsp), %rsp pop %rbx pop %rbp - retq + RET .code32 2: pushl $__KERNEL_CS
arch/x86/platform/olpc/xo1-wakeup.S +3 -3
@@ -77,7 +77,7 @@ pushfl popl saved_context_eflags - ret + RET restore_registers: movl saved_context_ebp, %ebp
@@ -88,7 +88,7 @@ pushl saved_context_eflags popfl - ret + RET SYM_CODE_START(do_olpc_suspend_lowlevel) call save_processor_state
@@ -109,7 +109,7 @@ SYM_CODE_START(do_olpc_suspend_lowlevel) call restore_registers call restore_processor_state - ret + RET SYM_CODE_END(do_olpc_suspend_lowlevel) .data
arch/x86/power/hibernate_asm_32.S +2 -2
@@ -32,7 +32,7 @@ SYM_FUNC_START(swsusp_arch_suspend) FRAME_BEGIN call swsusp_save FRAME_END - ret + RET SYM_FUNC_END(swsusp_arch_suspend) SYM_CODE_START(restore_image)
@@ -108,5 +108,5 @@ SYM_FUNC_START(restore_registers) /* tell the hibernation core that we've just restored the memory */ movl %eax, in_suspend - ret + RET SYM_FUNC_END(restore_registers)
arch/x86/power/hibernate_asm_64.S +2 -2
@@ -66,7 +66,7 @@ SYM_FUNC_START(restore_registers) /* tell the hibernation core that we've just restored the memory */ movq %rax, in_suspend(%rip) - ret + RET SYM_FUNC_END(restore_registers) SYM_FUNC_START(swsusp_arch_suspend)
@@ -96,7 +96,7 @@ SYM_FUNC_START(swsusp_arch_suspend) FRAME_BEGIN call swsusp_save FRAME_END - ret + RET SYM_FUNC_END(swsusp_arch_suspend) SYM_FUNC_START(restore_image)
arch/x86/um/checksum_32.S +2 -2
@@ -110,7 +110,7 @@ unsigned int csum_partial(const unsigned 7: popl %ebx popl %esi - ret + RET #else
@@ -208,7 +208,7 @@ unsigned int csum_partial(const unsigned 80: popl %ebx popl %esi - ret + RET #endif EXPORT_SYMBOL(csum_partial)
arch/x86/um/setjmp_32.S +1 -1
@@ -34,7 +34,7 @@ movl %esi,12(%edx) movl %edi,16(%edx) movl %ecx,20(%edx) # Return address - ret + RET .size kernel_setjmp,.-kernel_setjmp
arch/x86/um/setjmp_64.S +1 -1
@@ -33,7 +33,7 @@ movq %r14,40(%rdi) movq %r15,48(%rdi) movq %rsi,56(%rdi) # Return address - ret + RET .size kernel_setjmp,.-kernel_setjmp
arch/x86/xen/xen-asm.S +6 -6
@@ -28,7 +28,7 @@ */ SYM_FUNC_START(xen_irq_disable_direct) movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask - ret + RET SYM_FUNC_END(xen_irq_disable_direct) /*
@@ -57,7 +57,7 @@ SYM_FUNC_START(check_events) pop %rcx pop %rax FRAME_END - ret + RET SYM_FUNC_END(check_events) /*
@@ -83,7 +83,7 @@ SYM_FUNC_START(xen_irq_enable_direct) call check_events 1: FRAME_END - ret + RET SYM_FUNC_END(xen_irq_enable_direct) /*
@@ -99,7 +99,7 @@ SYM_FUNC_START(xen_save_fl_direct) testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask setz %ah addb %ah, %ah - ret + RET SYM_FUNC_END(xen_save_fl_direct) SYM_FUNC_START(xen_read_cr2)
@@ -107,14 +107,14 @@ SYM_FUNC_START(xen_read_cr2) _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2); SYM_FUNC_START(xen_read_cr2_direct) FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2_direct); .popsection
arch/x86/xen/xen-head.S +1 -1
@@ -26,7 +26,7 @@ SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) UNWIND_HINT_FUNC .skip 31, 0x90 - ret + RET .endr #define HYPERCALL(n) \
[tip: x86/core] x86: Prepare asm files for straight-line-speculation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f94909ceb1ed4bfdb2ada72f93236305e6d6951f Gitweb: https://git.kernel.org/tip/f94909ceb1ed4bfdb2ada72f93236305e6d6951f Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:40 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 08 Dec 2021 12:25:37 +01:00
x86: Prepare asm files for straight-line-speculation
Replace all ret/retq instructions with RET in preparation of making RET a macro. Since AS is case insensitive it's a big no-op without RET defined.
find arch/x86/ -name \*.S | while read file
do
sed -i 's/\<ret[q]*\>/RET/' $file
done
--- arch/x86/boot/compressed/efi_thunk_64.S | 2 +- arch/x86/boot/compressed/head_64.S | 8 +-- arch/x86/boot/compressed/mem_encrypt.S | 6 +- arch/x86/crypto/aegis128-aesni-asm.S | 48 ++++++++-------- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 2 +- arch/x86/crypto/aesni-intel_asm.S | 56 +++++++++---------- arch/x86/crypto/aesni-intel_avx-x86_64.S | 40 +++++++------- arch/x86/crypto/blake2s-core.S | 4 +- arch/x86/crypto/blowfish-x86_64-asm_64.S | 12 ++-- arch/x86/crypto/camellia-aesni-avx-asm_64.S | 14 ++--- arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 14 ++--- arch/x86/crypto/camellia-x86_64-asm_64.S | 12 ++-- arch/x86/crypto/cast5-avx-x86_64-asm_64.S | 12 ++-- arch/x86/crypto/cast6-avx-x86_64-asm_64.S | 10 +-- arch/x86/crypto/chacha-avx2-x86_64.S | 6 +- arch/x86/crypto/chacha-avx512vl-x86_64.S | 6 +- arch/x86/crypto/chacha-ssse3-x86_64.S | 8 +-- arch/x86/crypto/crc32-pclmul_asm.S | 2 +- arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 2 +- arch/x86/crypto/crct10dif-pcl-asm_64.S | 2 +- arch/x86/crypto/des3_ede-asm_64.S | 4 +- arch/x86/crypto/ghash-clmulni-intel_asm.S | 6 +- arch/x86/crypto/nh-avx2-x86_64.S | 2 +- arch/x86/crypto/nh-sse2-x86_64.S | 2 +- arch/x86/crypto/serpent-avx-x86_64-asm_64.S | 10 +-- arch/x86/crypto/serpent-avx2-asm_64.S | 10 +-- arch/x86/crypto/serpent-sse2-i586-asm_32.S | 6 +- arch/x86/crypto/serpent-sse2-x86_64-asm_64.S | 6 +- arch/x86/crypto/sha1_avx2_x86_64_asm.S | 2 +- arch/x86/crypto/sha1_ni_asm.S | 2 +- arch/x86/crypto/sha1_ssse3_asm.S | 2 +- arch/x86/crypto/sha256-avx-asm.S | 2 +- arch/x86/crypto/sha256-avx2-asm.S | 2 +- arch/x86/crypto/sha256-ssse3-asm.S | 2 +- arch/x86/crypto/sha256_ni_asm.S | 2 +- arch/x86/crypto/sha512-avx-asm.S | 2 +- arch/x86/crypto/sha512-avx2-asm.S | 2 +- arch/x86/crypto/sha512-ssse3-asm.S | 2 +- arch/x86/crypto/sm4-aesni-avx-asm_64.S | 12 ++-- arch/x86/crypto/sm4-aesni-avx2-asm_64.S | 8 +-- arch/x86/crypto/twofish-avx-x86_64-asm_64.S | 10 +-- arch/x86/crypto/twofish-i586-asm_32.S | 4 +- arch/x86/crypto/twofish-x86_64-asm_64-3way.S | 6 +- arch/x86/crypto/twofish-x86_64-asm_64.S | 4 +- arch/x86/entry/entry_32.S | 2 +- arch/x86/entry/entry_64.S | 10 +-- arch/x86/entry/thunk_32.S | 2 +- arch/x86/entry/thunk_64.S | 2 +- arch/x86/entry/vdso/vdso32/system_call.S | 2 +- arch/x86/entry/vdso/vsgx.S | 2 +- arch/x86/entry/vsyscall/vsyscall_emu_64.S | 6 +- arch/x86/kernel/acpi/wakeup_32.S | 6 +- arch/x86/kernel/ftrace_32.S | 6 +- arch/x86/kernel/ftrace_64.S | 8 +-- arch/x86/kernel/head_32.S | 2 +- arch/x86/kernel/irqflags.S | 2 +- arch/x86/kernel/relocate_kernel_32.S | 10 +-- arch/x86/kernel/relocate_kernel_64.S | 10 +-- arch/x86/kernel/sev_verify_cbit.S | 2 +- arch/x86/kernel/verify_cpu.S | 4 +- arch/x86/kvm/svm/vmenter.S | 4 +- arch/x86/kvm/vmx/vmenter.S | 14 ++--- arch/x86/lib/atomic64_386_32.S | 2 +- arch/x86/lib/atomic64_cx8_32.S | 16 ++--- arch/x86/lib/checksum_32.S | 8 +-- arch/x86/lib/clear_page_64.S | 6 +- arch/x86/lib/cmpxchg16b_emu.S | 4 +- arch/x86/lib/cmpxchg8b_emu.S | 4 +- arch/x86/lib/copy_mc_64.S | 6 +- arch/x86/lib/copy_page_64.S | 4 +- arch/x86/lib/copy_user_64.S | 10 +-- arch/x86/lib/csum-copy_64.S | 2 +- arch/x86/lib/getuser.S | 22 +++---- arch/x86/lib/hweight.S | 6 +- arch/x86/lib/iomap_copy_64.S | 2 +- arch/x86/lib/memcpy_64.S | 12 ++-- arch/x86/lib/memmove_64.S | 4 +- arch/x86/lib/memset_64.S | 6 +- arch/x86/lib/msr-reg.S | 4 +- arch/x86/lib/putuser.S | 6 +- arch/x86/lib/retpoline.S | 2 +- arch/x86/math-emu/div_Xsig.S | 2 +- arch/x86/math-emu/div_small.S | 2 +- arch/x86/math-emu/mul_Xsig.S | 6 +- arch/x86/math-emu/polynom_Xsig.S | 2 +- arch/x86/math-emu/reg_norm.S | 6 +- arch/x86/math-emu/reg_round.S | 2 +- arch/x86/math-emu/reg_u_add.S | 2 +- arch/x86/math-emu/reg_u_div.S | 2 +- arch/x86/math-emu/reg_u_mul.S | 2 +- arch/x86/math-emu/reg_u_sub.S | 2 +- arch/x86/math-emu/round_Xsig.S | 4 +- arch/x86/math-emu/shr_Xsig.S | 8 +-- arch/x86/math-emu/wm_shrx.S | 16 ++--- arch/x86/mm/mem_encrypt_boot.S | 4 +- arch/x86/platform/efi/efi_stub_32.S | 2 +- arch/x86/platform/efi/efi_stub_64.S | 2 +- arch/x86/platform/efi/efi_thunk_64.S | 2 +- arch/x86/platform/olpc/xo1-wakeup.S | 6 +- arch/x86/power/hibernate_asm_32.S | 4 +- arch/x86/power/hibernate_asm_64.S | 4 +- arch/x86/um/checksum_32.S | 4 +- arch/x86/um/setjmp_32.S | 2 +- arch/x86/um/setjmp_64.S | 2 +- arch/x86/xen/xen-asm.S | 12 ++-- arch/x86/xen/xen-head.S | 2 +- 106 files changed, 349 insertions(+), 349 deletions(-)
arch/x86/boot/compressed/efi_thunk_64.S +1 -1
@@ -93,7 +93,7 @@ SYM_FUNC_START(__efi64_thunk) pop %rbx pop %rbp - ret + RET SYM_FUNC_END(__efi64_thunk) .code32
arch/x86/boot/compressed/head_64.S +4 -4
@@ -813,7 +813,7 @@ SYM_FUNC_START(efi32_pe_entry) 2: popl %edi // restore callee-save registers popl %ebx leave - ret + RET SYM_FUNC_END(efi32_pe_entry) .section ".rodata"
@@ -868,7 +868,7 @@ SYM_FUNC_START(startup32_set_idt_entry) pop %ecx pop %ebx - ret + RET SYM_FUNC_END(startup32_set_idt_entry) #endif
@@ -884,7 +884,7 @@ SYM_FUNC_START(startup32_load_idt) movl %eax, rva(boot32_idt_desc+2)(%ebp) lidt rva(boot32_idt_desc)(%ebp) #endif - ret + RET SYM_FUNC_END(startup32_load_idt) /*
@@ -954,7 +954,7 @@ SYM_FUNC_START(startup32_check_sev_cbit) popl %ebx popl %eax #endif - ret + RET SYM_FUNC_END(startup32_check_sev_cbit) /*
arch/x86/boot/compressed/mem_encrypt.S +3 -3
@@ -58,7 +58,7 @@ SYM_FUNC_START(get_sev_encryption_bit) #endif /* CONFIG_AMD_MEM_ENCRYPT */ - ret + RET SYM_FUNC_END(get_sev_encryption_bit) /**
@@ -92,7 +92,7 @@ SYM_CODE_START_LOCAL(sev_es_req_cpuid) /* All good - return success */ xorl %eax, %eax 1: - ret + RET 2: movl $-1, %eax jmp 1b
@@ -221,7 +221,7 @@ SYM_FUNC_START(set_sev_encryption_mask) #endif xor %rax, %rax - ret + RET SYM_FUNC_END(set_sev_encryption_mask) .data
arch/x86/crypto/aegis128-aesni-asm.S +24 -24
181 lines not on this page · raw message
arch/x86/crypto/aes_ctrby8_avx-x86_64.S +1 -1
@@ -525,7 +525,7 @@ ddq_add_8: /* return updated IV */ vpshufb xbyteswap, xcounter, xcounter vmovdqu xcounter, (p_iv) - ret + RET .endm /*
arch/x86/crypto/aesni-intel_asm.S +28 -28
224 lines not on this page · raw message
arch/x86/crypto/aesni-intel_avx-x86_64.S +20 -20
@@ -1767,7 +1767,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen2) FUNC_SAVE INIT GHASH_MUL_AVX, PRECOMPUTE_AVX FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_init_avx_gen2) ###############################################################################
@@ -1788,15 +1788,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen2) # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 11 FUNC_RESTORE - ret + RET key_128_enc_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 9 FUNC_RESTORE - ret + RET key_256_enc_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, ENC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_enc_update_avx_gen2) ###############################################################################
@@ -1817,15 +1817,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen2) # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 11 FUNC_RESTORE - ret + RET key_128_dec_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 9 FUNC_RESTORE - ret + RET key_256_dec_update: GCM_ENC_DEC INITIAL_BLOCKS_AVX, GHASH_8_ENCRYPT_8_PARALLEL_AVX, GHASH_LAST_8_AVX, GHASH_MUL_AVX, DEC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_dec_update_avx_gen2) ###############################################################################
@@ -1846,15 +1846,15 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen2) # must be 192 GCM_COMPLETE GHASH_MUL_AVX, 11, arg3, arg4 FUNC_RESTORE - ret + RET key_128_finalize: GCM_COMPLETE GHASH_MUL_AVX, 9, arg3, arg4 FUNC_RESTORE - ret + RET key_256_finalize: GCM_COMPLETE GHASH_MUL_AVX, 13, arg3, arg4 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_finalize_avx_gen2) ###############################################################################
@@ -2735,7 +2735,7 @@ SYM_FUNC_START(aesni_gcm_init_avx_gen4) FUNC_SAVE INIT GHASH_MUL_AVX2, PRECOMPUTE_AVX2 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_init_avx_gen4) ###############################################################################
@@ -2756,15 +2756,15 @@ SYM_FUNC_START(aesni_gcm_enc_update_avx_gen4) # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 11 FUNC_RESTORE - ret + RET key_128_enc_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 9 FUNC_RESTORE - ret + RET key_256_enc_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, ENC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_enc_update_avx_gen4) ###############################################################################
@@ -2785,15 +2785,15 @@ SYM_FUNC_START(aesni_gcm_dec_update_avx_gen4) # must be 192 GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 11 FUNC_RESTORE - ret + RET key_128_dec_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 9 FUNC_RESTORE - ret + RET key_256_dec_update4: GCM_ENC_DEC INITIAL_BLOCKS_AVX2, GHASH_8_ENCRYPT_8_PARALLEL_AVX2, GHASH_LAST_8_AVX2, GHASH_MUL_AVX2, DEC, 13 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_dec_update_avx_gen4) ###############################################################################
@@ -2814,13 +2814,13 @@ SYM_FUNC_START(aesni_gcm_finalize_avx_gen4) # must be 192 GCM_COMPLETE GHASH_MUL_AVX2, 11, arg3, arg4 FUNC_RESTORE - ret + RET key_128_finalize4: GCM_COMPLETE GHASH_MUL_AVX2, 9, arg3, arg4 FUNC_RESTORE - ret + RET key_256_finalize4: GCM_COMPLETE GHASH_MUL_AVX2, 13, arg3, arg4 FUNC_RESTORE - ret + RET SYM_FUNC_END(aesni_gcm_finalize_avx_gen4)
arch/x86/crypto/blake2s-core.S +2 -2
@@ -171,7 +171,7 @@ SYM_FUNC_START(blake2s_compress_ssse3) movdqu %xmm1,0x10(%rdi) movdqu %xmm14,0x20(%rdi) .Lendofloop: - ret + RET SYM_FUNC_END(blake2s_compress_ssse3) #ifdef CONFIG_AS_AVX512
@@ -251,6 +251,6 @@ SYM_FUNC_START(blake2s_compress_avx512) vmovdqu %xmm1,0x10(%rdi) vmovdqu %xmm4,0x20(%rdi) vzeroupper - retq + RET SYM_FUNC_END(blake2s_compress_avx512) #endif /* CONFIG_AS_AVX512 */
arch/x86/crypto/blowfish-x86_64-asm_64.S +6 -6
@@ -135,10 +135,10 @@ SYM_FUNC_START(__blowfish_enc_blk) jnz .L__enc_xor; write_block(); - ret; + RET; .L__enc_xor: xor_block(); - ret; + RET; SYM_FUNC_END(__blowfish_enc_blk) SYM_FUNC_START(blowfish_dec_blk)
@@ -170,7 +170,7 @@ SYM_FUNC_START(blowfish_dec_blk) movq %r11, %r12; - ret; + RET; SYM_FUNC_END(blowfish_dec_blk) /**********************************************************************
@@ -322,14 +322,14 @@ SYM_FUNC_START(__blowfish_enc_blk_4way) popq %rbx; popq %r12; - ret; + RET; .L__enc_xor4: xor_block4(); popq %rbx; popq %r12; - ret; + RET; SYM_FUNC_END(__blowfish_enc_blk_4way) SYM_FUNC_START(blowfish_dec_blk_4way)
@@ -364,5 +364,5 @@ SYM_FUNC_START(blowfish_dec_blk_4way) popq %rbx; popq %r12; - ret; + RET; SYM_FUNC_END(blowfish_dec_blk_4way)
arch/x86/crypto/camellia-aesni-avx-asm_64.S +7 -7
@@ -192,7 +192,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %rcx, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8
@@ -200,7 +200,7 @@ SYM_FUNC_START_LOCAL(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, %rax, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /*
@@ -778,7 +778,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk16) %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32:
@@ -865,7 +865,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk16) %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32:
@@ -906,7 +906,7 @@ SYM_FUNC_START(camellia_ecb_enc_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_enc_16way) SYM_FUNC_START(camellia_ecb_dec_16way)
@@ -936,7 +936,7 @@ SYM_FUNC_START(camellia_ecb_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_dec_16way) SYM_FUNC_START(camellia_cbc_dec_16way)
@@ -987,5 +987,5 @@ SYM_FUNC_START(camellia_cbc_dec_16way) %xmm8, %rsi); FRAME_END - ret; + RET; SYM_FUNC_END(camellia_cbc_dec_16way)
arch/x86/crypto/camellia-aesni-avx2-asm_64.S +7 -7
@@ -226,7 +226,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rcx, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) .align 8
@@ -234,7 +234,7 @@ SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_a roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, %rax, (%r9)); - ret; + RET; SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) /*
@@ -814,7 +814,7 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk32) %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Lenc_max32:
@@ -901,7 +901,7 @@ SYM_FUNC_START_LOCAL(__camellia_dec_blk32) %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); FRAME_END - ret; + RET; .align 8 .Ldec_max32:
@@ -946,7 +946,7 @@ SYM_FUNC_START(camellia_ecb_enc_32way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_enc_32way) SYM_FUNC_START(camellia_ecb_dec_32way)
@@ -980,7 +980,7 @@ SYM_FUNC_START(camellia_ecb_dec_32way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_ecb_dec_32way) SYM_FUNC_START(camellia_cbc_dec_32way)
@@ -1047,5 +1047,5 @@ SYM_FUNC_START(camellia_cbc_dec_32way) addq $(16 * 32), %rsp; FRAME_END - ret; + RET; SYM_FUNC_END(camellia_cbc_dec_32way)
arch/x86/crypto/camellia-x86_64-asm_64.S +6 -6
@@ -213,13 +213,13 @@ SYM_FUNC_START(__camellia_enc_blk) enc_outunpack(mov, RT1); movq RR12, %r12; - ret; + RET; .L__enc_xor: enc_outunpack(xor, RT1); movq RR12, %r12; - ret; + RET; SYM_FUNC_END(__camellia_enc_blk) SYM_FUNC_START(camellia_dec_blk)
@@ -257,7 +257,7 @@ SYM_FUNC_START(camellia_dec_blk) dec_outunpack(); movq RR12, %r12; - ret; + RET; SYM_FUNC_END(camellia_dec_blk) /**********************************************************************
@@ -448,14 +448,14 @@ SYM_FUNC_START(__camellia_enc_blk_2way) movq RR12, %r12; popq %rbx; - ret; + RET; .L__enc2_xor: enc_outunpack2(xor, RT2); movq RR12, %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(__camellia_enc_blk_2way) SYM_FUNC_START(camellia_dec_blk_2way)
@@ -495,5 +495,5 @@ SYM_FUNC_START(camellia_dec_blk_2way) movq RR12, %r12; movq RXOR, %rbx; - ret; + RET; SYM_FUNC_END(camellia_dec_blk_2way)
arch/x86/crypto/cast5-avx-x86_64-asm_64.S +6 -6
@@ -279,7 +279,7 @@ SYM_FUNC_START_LOCAL(__cast5_enc_blk16) outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; SYM_FUNC_END(__cast5_enc_blk16) .align 16
@@ -352,7 +352,7 @@ SYM_FUNC_START_LOCAL(__cast5_dec_blk16) outunpack_blocks(RR3, RL3, RTMP, RX, RKM); outunpack_blocks(RR4, RL4, RTMP, RX, RKM); - ret; + RET; .L__skip_dec: vpsrldq $4, RKR, RKR;
@@ -393,7 +393,7 @@ SYM_FUNC_START(cast5_ecb_enc_16way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ecb_enc_16way) SYM_FUNC_START(cast5_ecb_dec_16way)
@@ -431,7 +431,7 @@ SYM_FUNC_START(cast5_ecb_dec_16way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ecb_dec_16way) SYM_FUNC_START(cast5_cbc_dec_16way)
@@ -483,7 +483,7 @@ SYM_FUNC_START(cast5_cbc_dec_16way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_cbc_dec_16way) SYM_FUNC_START(cast5_ctr_16way)
@@ -559,5 +559,5 @@ SYM_FUNC_START(cast5_ctr_16way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast5_ctr_16way)
arch/x86/crypto/cast6-avx-x86_64-asm_64.S +5 -5
@@ -289,7 +289,7 @@ SYM_FUNC_START_LOCAL(__cast6_enc_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; SYM_FUNC_END(__cast6_enc_blk8) .align 8
@@ -336,7 +336,7 @@ SYM_FUNC_START_LOCAL(__cast6_dec_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM); outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); - ret; + RET; SYM_FUNC_END(__cast6_dec_blk8) SYM_FUNC_START(cast6_ecb_enc_8way)
@@ -359,7 +359,7 @@ SYM_FUNC_START(cast6_ecb_enc_8way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_ecb_enc_8way) SYM_FUNC_START(cast6_ecb_dec_8way)
@@ -382,7 +382,7 @@ SYM_FUNC_START(cast6_ecb_dec_8way) popq %r15; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_ecb_dec_8way) SYM_FUNC_START(cast6_cbc_dec_8way)
@@ -408,5 +408,5 @@ SYM_FUNC_START(cast6_cbc_dec_8way) popq %r15; popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(cast6_cbc_dec_8way)
arch/x86/crypto/chacha-avx2-x86_64.S +3 -3
@@ -193,7 +193,7 @@ SYM_FUNC_START(chacha_2block_xor_avx2) .Ldone2: vzeroupper - ret + RET .Lxorpart2: # xor remaining bytes from partial register into output
@@ -498,7 +498,7 @@ SYM_FUNC_START(chacha_4block_xor_avx2) .Ldone4: vzeroupper - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
@@ -992,7 +992,7 @@ SYM_FUNC_START(chacha_8block_xor_avx2) .Ldone8: vzeroupper lea -8(%r10),%rsp - ret + RET .Lxorpart8: # xor remaining bytes from partial register into output
arch/x86/crypto/chacha-avx512vl-x86_64.S +3 -3
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl) .Ldone2: vzeroupper - ret + RET .Lxorpart2: # xor remaining bytes from partial register into output
@@ -432,7 +432,7 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl) .Ldone4: vzeroupper - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
@@ -812,7 +812,7 @@ SYM_FUNC_START(chacha_8block_xor_avx512vl) .Ldone8: vzeroupper - ret + RET .Lxorpart8: # xor remaining bytes from partial register into output
arch/x86/crypto/chacha-ssse3-x86_64.S +4 -4
@@ -108,7 +108,7 @@ SYM_FUNC_START_LOCAL(chacha_permute) sub $2,%r8d jnz .Ldoubleround - ret + RET SYM_FUNC_END(chacha_permute) SYM_FUNC_START(chacha_block_xor_ssse3)
@@ -166,7 +166,7 @@ SYM_FUNC_START(chacha_block_xor_ssse3) .Ldone: FRAME_END - ret + RET .Lxorpart: # xor remaining bytes from partial register into output
@@ -217,7 +217,7 @@ SYM_FUNC_START(hchacha_block_ssse3) movdqu %xmm3,0x10(%rsi) FRAME_END - ret + RET SYM_FUNC_END(hchacha_block_ssse3) SYM_FUNC_START(chacha_4block_xor_ssse3)
@@ -762,7 +762,7 @@ SYM_FUNC_START(chacha_4block_xor_ssse3) .Ldone4: lea -8(%r10),%rsp - ret + RET .Lxorpart4: # xor remaining bytes from partial register into output
arch/x86/crypto/crc32-pclmul_asm.S +1 -1
@@ -236,5 +236,5 @@ fold_64: pxor %xmm2, %xmm1 pextrd $0x01, %xmm1, %eax - ret + RET SYM_FUNC_END(crc32_pclmul_le_16)
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +1 -1
@@ -306,7 +306,7 @@ do_return: popq %rsi popq %rdi popq %rbx - ret + RET SYM_FUNC_END(crc_pcl) .section .rodata, "a", @progbits
arch/x86/crypto/crct10dif-pcl-asm_64.S +1 -1
@@ -257,7 +257,7 @@ SYM_FUNC_START(crc_t10dif_pcl) # Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of xmm0. pextrw $0, %xmm0, %eax - ret + RET .align 16 .Lless_than_256_bytes:
arch/x86/crypto/des3_ede-asm_64.S +2 -2
@@ -243,7 +243,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk) popq %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(des3_ede_x86_64_crypt_blk) /***********************************************************************
@@ -528,7 +528,7 @@ SYM_FUNC_START(des3_ede_x86_64_crypt_blk_3way) popq %r12; popq %rbx; - ret; + RET; SYM_FUNC_END(des3_ede_x86_64_crypt_blk_3way) .section .rodata, "a", @progbits
arch/x86/crypto/ghash-clmulni-intel_asm.S +3 -3
@@ -85,7 +85,7 @@ SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) psrlq $1, T2 pxor T2, T1 pxor T1, DATA - ret + RET SYM_FUNC_END(__clmul_gf128mul_ble) /* void clmul_ghash_mul(char *dst, const u128 *shash) */
@@ -99,7 +99,7 @@ SYM_FUNC_START(clmul_ghash_mul) pshufb BSWAP, DATA movups DATA, (%rdi) FRAME_END - ret + RET SYM_FUNC_END(clmul_ghash_mul) /*
@@ -128,5 +128,5 @@ SYM_FUNC_START(clmul_ghash_update) movups DATA, (%rdi) .Lupdate_just_ret: FRAME_END - ret + RET SYM_FUNC_END(clmul_ghash_update)
arch/x86/crypto/nh-avx2-x86_64.S +1 -1
@@ -153,5 +153,5 @@ SYM_FUNC_START(nh_avx2) vpaddq T1, T0, T0 vpaddq T4, T0, T0 vmovdqu T0, (HASH) - ret + RET SYM_FUNC_END(nh_avx2)
arch/x86/crypto/nh-sse2-x86_64.S +1 -1
@@ -119,5 +119,5 @@ SYM_FUNC_START(nh_sse2) paddq PASS2_SUMS, T1 movdqu T0, 0x00(HASH) movdqu T1, 0x10(HASH) - ret + RET SYM_FUNC_END(nh_sse2)
arch/x86/crypto/serpent-avx-x86_64-asm_64.S +5 -5
@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx) write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk8_avx) .align 8
@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx) write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_dec_blk8_avx) SYM_FUNC_START(serpent_ecb_enc_8way_avx)
@@ -673,7 +673,7 @@ SYM_FUNC_START(serpent_ecb_enc_8way_avx) store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_enc_8way_avx) SYM_FUNC_START(serpent_ecb_dec_8way_avx)
@@ -691,7 +691,7 @@ SYM_FUNC_START(serpent_ecb_dec_8way_avx) store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_dec_8way_avx) SYM_FUNC_START(serpent_cbc_dec_8way_avx)
@@ -709,5 +709,5 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx) store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); FRAME_END - ret; + RET; SYM_FUNC_END(serpent_cbc_dec_8way_avx)
arch/x86/crypto/serpent-avx2-asm_64.S +5 -5
@@ -601,7 +601,7 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk16) write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk16) .align 8
@@ -655,7 +655,7 @@ SYM_FUNC_START_LOCAL(__serpent_dec_blk16) write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_dec_blk16) SYM_FUNC_START(serpent_ecb_enc_16way)
@@ -677,7 +677,7 @@ SYM_FUNC_START(serpent_ecb_enc_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_enc_16way) SYM_FUNC_START(serpent_ecb_dec_16way)
@@ -699,7 +699,7 @@ SYM_FUNC_START(serpent_ecb_dec_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_ecb_dec_16way) SYM_FUNC_START(serpent_cbc_dec_16way)
@@ -722,5 +722,5 @@ SYM_FUNC_START(serpent_cbc_dec_16way) vzeroupper; FRAME_END - ret; + RET; SYM_FUNC_END(serpent_cbc_dec_16way)
arch/x86/crypto/serpent-sse2-i586-asm_32.S +3 -3
@@ -553,12 +553,12 @@ SYM_FUNC_START(__serpent_enc_blk_4way) write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; .L__enc_xor4: xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk_4way) SYM_FUNC_START(serpent_dec_blk_4way)
@@ -612,5 +612,5 @@ SYM_FUNC_START(serpent_dec_blk_4way) movl arg_dst(%esp), %eax; write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA); - ret; + RET; SYM_FUNC_END(serpent_dec_blk_4way)
arch/x86/crypto/serpent-sse2-x86_64-asm_64.S +3 -3
@@ -675,13 +675,13 @@ SYM_FUNC_START(__serpent_enc_blk_8way) write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; .L__enc_xor8: xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(__serpent_enc_blk_8way) SYM_FUNC_START(serpent_dec_blk_8way)
@@ -735,5 +735,5 @@ SYM_FUNC_START(serpent_dec_blk_8way) write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2); write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); - ret; + RET; SYM_FUNC_END(serpent_dec_blk_8way)
arch/x86/crypto/sha1_avx2_x86_64_asm.S +1 -1
@@ -674,7 +674,7 @@ _loop3: pop %r12 pop %rbx - ret + RET SYM_FUNC_END(\name) .endm
arch/x86/crypto/sha1_ni_asm.S +1 -1
@@ -290,7 +290,7 @@ SYM_FUNC_START(sha1_ni_transform) mov %rbp, %rsp pop %rbp - ret + RET SYM_FUNC_END(sha1_ni_transform) .section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
arch/x86/crypto/sha1_ssse3_asm.S +1 -1
@@ -99,7 +99,7 @@ pop %rbp pop %r12 pop %rbx - ret + RET SYM_FUNC_END(\name) .endm
arch/x86/crypto/sha256-avx-asm.S +1 -1
@@ -458,7 +458,7 @@ done_hash: popq %r13 popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_avx) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha256-avx2-asm.S +1 -1
@@ -710,7 +710,7 @@ done_hash: popq %r13 popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_rorx) .section .rodata.cst512.K256, "aM", @progbits, 512
arch/x86/crypto/sha256-ssse3-asm.S +1 -1
@@ -472,7 +472,7 @@ done_hash: popq %r12 popq %rbx - ret + RET SYM_FUNC_END(sha256_transform_ssse3) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha256_ni_asm.S +1 -1
@@ -326,7 +326,7 @@ SYM_FUNC_START(sha256_ni_transform) .Ldone_hash: - ret + RET SYM_FUNC_END(sha256_ni_transform) .section .rodata.cst256.K256, "aM", @progbits, 256
arch/x86/crypto/sha512-avx-asm.S +1 -1
@@ -361,7 +361,7 @@ updateblock: pop %rbx nowork: - ret + RET SYM_FUNC_END(sha512_transform_avx) ########################################################################
arch/x86/crypto/sha512-avx2-asm.S +1 -1
@@ -679,7 +679,7 @@ done_hash: pop %r12 pop %rbx - ret + RET SYM_FUNC_END(sha512_transform_rorx) ########################################################################
arch/x86/crypto/sha512-ssse3-asm.S +1 -1
@@ -363,7 +363,7 @@ updateblock: pop %rbx nowork: - ret + RET SYM_FUNC_END(sha512_transform_ssse3) ########################################################################
arch/x86/crypto/sm4-aesni-avx-asm_64.S +6 -6
@@ -246,7 +246,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt4) .Lblk4_store_output_done: vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_crypt4) .align 8
@@ -356,7 +356,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk8) vpshufb RTMP2, RB3, RB3; FRAME_END - ret; + RET; SYM_FUNC_END(__sm4_crypt_blk8) /*
@@ -412,7 +412,7 @@ SYM_FUNC_START(sm4_aesni_avx_crypt8) .Lblk8_store_output_done: vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_crypt8) /*
@@ -487,7 +487,7 @@ SYM_FUNC_START(sm4_aesni_avx_ctr_enc_blk8) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8) /*
@@ -537,7 +537,7 @@ SYM_FUNC_START(sm4_aesni_avx_cbc_dec_blk8) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8) /*
@@ -590,5 +590,5 @@ SYM_FUNC_START(sm4_aesni_avx_cfb_dec_blk8) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8)
arch/x86/crypto/sm4-aesni-avx2-asm_64.S +4 -4
@@ -268,7 +268,7 @@ SYM_FUNC_START_LOCAL(__sm4_crypt_blk16) vpshufb RTMP2, RB3, RB3; FRAME_END - ret; + RET; SYM_FUNC_END(__sm4_crypt_blk16) #define inc_le128(x, minus_one, tmp) \
@@ -387,7 +387,7 @@ SYM_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16) /*
@@ -441,7 +441,7 @@ SYM_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16) /*
@@ -497,5 +497,5 @@ SYM_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16) vzeroall; FRAME_END - ret; + RET; SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16)
arch/x86/crypto/twofish-avx-x86_64-asm_64.S +5 -5
@@ -267,7 +267,7 @@ SYM_FUNC_START_LOCAL(__twofish_enc_blk8) outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); - ret; + RET; SYM_FUNC_END(__twofish_enc_blk8) .align 8
@@ -307,7 +307,7 @@ SYM_FUNC_START_LOCAL(__twofish_dec_blk8) outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); - ret; + RET; SYM_FUNC_END(__twofish_dec_blk8) SYM_FUNC_START(twofish_ecb_enc_8way)
@@ -327,7 +327,7 @@ SYM_FUNC_START(twofish_ecb_enc_8way) store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); FRAME_END - ret; + RET; SYM_FUNC_END(twofish_ecb_enc_8way) SYM_FUNC_START(twofish_ecb_dec_8way)
@@ -347,7 +347,7 @@ SYM_FUNC_START(twofish_ecb_dec_8way) store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); FRAME_END - ret; + RET; SYM_FUNC_END(twofish_ecb_dec_8way) SYM_FUNC_START(twofish_cbc_dec_8way)
@@ -372,5 +372,5 @@ SYM_FUNC_START(twofish_cbc_dec_8way) popq %r12; FRAME_END - ret; + RET; SYM_FUNC_END(twofish_cbc_dec_8way)
arch/x86/crypto/twofish-i586-asm_32.S +2 -2
@@ -260,7 +260,7 @@ SYM_FUNC_START(twofish_enc_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET SYM_FUNC_END(twofish_enc_blk) SYM_FUNC_START(twofish_dec_blk)
@@ -317,5 +317,5 @@ SYM_FUNC_START(twofish_dec_blk) pop %ebx pop %ebp mov $1, %eax - ret + RET SYM_FUNC_END(twofish_dec_blk)
arch/x86/crypto/twofish-x86_64-asm_64-3way.S +3 -3
@@ -258,7 +258,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; .L__enc_xor3: outunpack_enc3(xor);
@@ -266,7 +266,7 @@ SYM_FUNC_START(__twofish_enc_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; SYM_FUNC_END(__twofish_enc_blk_3way) SYM_FUNC_START(twofish_dec_blk_3way)
@@ -301,5 +301,5 @@ SYM_FUNC_START(twofish_dec_blk_3way) popq %rbx; popq %r12; popq %r13; - ret; + RET; SYM_FUNC_END(twofish_dec_blk_3way)
arch/x86/crypto/twofish-x86_64-asm_64.S +2 -2
@@ -252,7 +252,7 @@ SYM_FUNC_START(twofish_enc_blk) popq R1 movl $1,%eax - ret + RET SYM_FUNC_END(twofish_enc_blk) SYM_FUNC_START(twofish_dec_blk)
@@ -304,5 +304,5 @@ SYM_FUNC_START(twofish_dec_blk) popq R1 movl $1,%eax - ret + RET SYM_FUNC_END(twofish_dec_blk)
arch/x86/entry/entry_32.S +1 -1
@@ -740,7 +740,7 @@ SYM_FUNC_START(schedule_tail_wrapper) popl %eax FRAME_END - ret + RET SYM_FUNC_END(schedule_tail_wrapper) .popsection
arch/x86/entry/entry_64.S +5 -5
@@ -738,7 +738,7 @@ SYM_FUNC_START(asm_load_gs_index) 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs FRAME_END - ret + RET SYM_FUNC_END(asm_load_gs_index) EXPORT_SYMBOL(asm_load_gs_index)
@@ -889,7 +889,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) * is needed here. */ SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx - ret + RET .Lparanoid_entry_checkgs: /* EBX = 1 -> kernel GSBASE active, no restore required */
@@ -910,7 +910,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) .Lparanoid_kernel_gsbase: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET SYM_CODE_END(paranoid_entry) /*
@@ -989,7 +989,7 @@ SYM_CODE_START_LOCAL(error_entry) movq %rax, %rsp /* switch stack */ ENCODE_FRAME_POINTER pushq %r12 - ret + RET /* * There are two places in the kernel that can potentially fault with
@@ -1020,7 +1020,7 @@ SYM_CODE_START_LOCAL(error_entry) */ .Lerror_entry_done_lfence: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET .Lbstep_iret: /* Fix truncated RIP */
arch/x86/entry/thunk_32.S +1 -1
@@ -24,7 +24,7 @@ SYM_CODE_START_NOALIGN(\name) popl %edx popl %ecx popl %eax - ret + RET _ASM_NOKPROBE(\name) SYM_CODE_END(\name) .endm
arch/x86/entry/thunk_64.S +1 -1
@@ -50,7 +50,7 @@ SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore) popq %rsi popq %rdi popq %rbp - ret + RET _ASM_NOKPROBE(__thunk_restore) SYM_CODE_END(__thunk_restore) #endif
arch/x86/entry/vdso/vdso32/system_call.S +1 -1
@@ -78,7 +78,7 @@ SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL) popl %ecx CFI_RESTORE ecx CFI_ADJUST_CFA_OFFSET -4 - ret + RET CFI_ENDPROC .size __kernel_vsyscall,.-__kernel_vsyscall
arch/x86/entry/vdso/vsgx.S +1 -1
@@ -81,7 +81,7 @@ SYM_FUNC_START(__vdso_sgx_enter_enclave) pop %rbx leave .cfi_def_cfa %rsp, 8 - ret + RET /* The out-of-line code runs with the pre-leave stack frame. */ .cfi_def_cfa %rbp, 16
arch/x86/entry/vsyscall/vsyscall_emu_64.S +3 -3
@@ -19,17 +19,17 @@ __vsyscall_page: mov $__NR_gettimeofday, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_time, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_getcpu, %rax syscall - ret + RET .balign 4096, 0xcc
arch/x86/kernel/acpi/wakeup_32.S +3 -3
@@ -60,7 +60,7 @@ save_registers: popl saved_context_eflags movl $ret_point, saved_eip - ret + RET restore_registers:
@@ -70,7 +70,7 @@ restore_registers: movl saved_context_edi, %edi pushl saved_context_eflags popfl - ret + RET SYM_CODE_START(do_suspend_lowlevel) call save_processor_state
@@ -86,7 +86,7 @@ SYM_CODE_START(do_suspend_lowlevel) ret_point: call restore_registers call restore_processor_state - ret + RET SYM_CODE_END(do_suspend_lowlevel) .data
arch/x86/kernel/ftrace_32.S +3 -3
@@ -19,7 +19,7 @@ #endif SYM_FUNC_START(__fentry__) - ret + RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__)
@@ -84,7 +84,7 @@ ftrace_graph_call: /* This is weak to keep gas from relaxing the jumps */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) - ret + RET SYM_CODE_END(ftrace_caller) SYM_CODE_START(ftrace_regs_caller)
@@ -177,7 +177,7 @@ SYM_CODE_START(ftrace_graph_caller) popl %edx popl %ecx popl %eax - ret + RET SYM_CODE_END(ftrace_graph_caller) .globl return_to_handler
arch/x86/kernel/ftrace_64.S +4 -4
@@ -132,7 +132,7 @@ #ifdef CONFIG_DYNAMIC_FTRACE SYM_FUNC_START(__fentry__) - retq + RET SYM_FUNC_END(__fentry__) EXPORT_SYMBOL(__fentry__)
@@ -176,11 +176,11 @@ SYM_FUNC_END(ftrace_caller); SYM_FUNC_START(ftrace_epilogue) /* * This is weak to keep gas from relaxing the jumps. - * It is also used to copy the retq for trampolines. + * It is also used to copy the RET for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) UNWIND_HINT_FUNC - retq + RET SYM_FUNC_END(ftrace_epilogue) SYM_FUNC_START(ftrace_regs_caller)
@@ -284,7 +284,7 @@ SYM_FUNC_START(__fentry__) jnz trace SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL) - retq + RET trace: /* save_mcount_regs fills in first two parameters */
arch/x86/kernel/head_32.S +1 -1
@@ -340,7 +340,7 @@ SYM_FUNC_END(startup_32_smp) __INIT setup_once: andl $0,setup_once_ref /* Once is enough, thanks */ - ret + RET SYM_FUNC_START(early_idt_handler_array) # 36(%esp) %eflags
arch/x86/kernel/irqflags.S +1 -1
@@ -11,7 +11,7 @@ SYM_FUNC_START(native_save_fl) pushf pop %_ASM_AX - ret + RET SYM_FUNC_END(native_save_fl) .popsection EXPORT_SYMBOL(native_save_fl)
arch/x86/kernel/relocate_kernel_32.S +5 -5
@@ -91,7 +91,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) movl %edi, %eax addl $(identity_mapped - relocate_kernel), %eax pushl %eax - ret + RET SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
@@ -159,7 +159,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) xorl %edx, %edx xorl %esi, %esi xorl %ebp, %ebp - ret + RET 1: popl %edx movl CP_PA_SWAP_PAGE(%edi), %esp
@@ -190,7 +190,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) movl %edi, %eax addl $(virtual_mapped - relocate_kernel), %eax pushl %eax - ret + RET SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
@@ -208,7 +208,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) popl %edi popl %esi popl %ebx - ret + RET SYM_CODE_END(virtual_mapped) /* Do the copies */
@@ -271,7 +271,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) popl %edi popl %ebx popl %ebp - ret + RET SYM_CODE_END(swap_pages) .globl kexec_control_code_size
arch/x86/kernel/relocate_kernel_64.S +5 -5
@@ -104,7 +104,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) /* jump to identity mapped page */ addq $(identity_mapped - relocate_kernel), %r8 pushq %r8 - ret + RET SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
@@ -191,7 +191,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) xorl %r14d, %r14d xorl %r15d, %r15d - ret + RET 1: popq %rdx
@@ -210,7 +210,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) call swap_pages movq $virtual_mapped, %rax pushq %rax - ret + RET SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
@@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) popq %r12 popq %rbp popq %rbx - ret + RET SYM_CODE_END(virtual_mapped) /* Do the copies */
@@ -288,7 +288,7 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) lea PAGE_SIZE(%rax), %rsi jmp 0b 3: - ret + RET SYM_CODE_END(swap_pages) .globl kexec_control_code_size
arch/x86/kernel/sev_verify_cbit.S +1 -1
@@ -85,5 +85,5 @@ SYM_FUNC_START(sev_verify_cbit) #endif /* Return page-table pointer */ movq %rdi, %rax - ret + RET SYM_FUNC_END(sev_verify_cbit)
arch/x86/kernel/verify_cpu.S +2 -2
@@ -132,9 +132,9 @@ SYM_FUNC_START_LOCAL(verify_cpu) .Lverify_cpu_no_longmode: popf # Restore caller passed flags movl $1,%eax - ret + RET .Lverify_cpu_sse_ok: popf # Restore caller passed flags xorl %eax, %eax - ret + RET SYM_FUNC_END(verify_cpu)
arch/x86/kvm/svm/vmenter.S +2 -2
@@ -148,7 +148,7 @@ SYM_FUNC_START(__svm_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET 3: cmpb $0, kvm_rebooting jne 2b
@@ -202,7 +202,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET 3: cmpb $0, kvm_rebooting jne 2b
arch/x86/kvm/vmx/vmenter.S +7 -7
@@ -49,14 +49,14 @@ SYM_FUNC_START_LOCAL(vmx_vmenter) je 2f 1: vmresume - ret + RET 2: vmlaunch - ret + RET 3: cmpb $0, kvm_rebooting je 4f - ret + RET 4: ud2 _ASM_EXTABLE(1b, 3b)
@@ -89,7 +89,7 @@ SYM_FUNC_START(vmx_vmexit) pop %_ASM_AX .Lvmexit_skip_rsb: #endif - ret + RET SYM_FUNC_END(vmx_vmexit) /**
@@ -228,7 +228,7 @@ SYM_FUNC_START(__vmx_vcpu_run) pop %edi #endif pop %_ASM_BP - ret + RET /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */ 2: mov $1, %eax
@@ -293,7 +293,7 @@ SYM_FUNC_START(vmread_error_trampoline) pop %_ASM_AX pop %_ASM_BP - ret + RET SYM_FUNC_END(vmread_error_trampoline) SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff)
@@ -326,5 +326,5 @@ SYM_FUNC_START(vmx_do_interrupt_nmi_irqoff) */ mov %_ASM_BP, %_ASM_SP pop %_ASM_BP - ret + RET SYM_FUNC_END(vmx_do_interrupt_nmi_irqoff)
arch/x86/lib/atomic64_386_32.S +1 -1
@@ -30,7 +30,7 @@ SYM_FUNC_START(atomic64_##op##_386); \ #define RET_IRQ_RESTORE \ IRQ_RESTORE v; \ - ret + RET #define v %ecx BEGIN_IRQ_SAVE(read)
arch/x86/lib/atomic64_cx8_32.S +8 -8
@@ -18,7 +18,7 @@ SYM_FUNC_START(atomic64_read_cx8) read64 %ecx - ret + RET SYM_FUNC_END(atomic64_read_cx8) SYM_FUNC_START(atomic64_set_cx8)
@@ -28,7 +28,7 @@ SYM_FUNC_START(atomic64_set_cx8) cmpxchg8b (%esi) jne 1b - ret + RET SYM_FUNC_END(atomic64_set_cx8) SYM_FUNC_START(atomic64_xchg_cx8)
@@ -37,7 +37,7 @@ SYM_FUNC_START(atomic64_xchg_cx8) cmpxchg8b (%esi) jne 1b - ret + RET SYM_FUNC_END(atomic64_xchg_cx8) .macro addsub_return func ins insc
@@ -68,7 +68,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8) popl %esi popl %ebx popl %ebp - ret + RET SYM_FUNC_END(atomic64_\func\()_return_cx8) .endm
@@ -93,7 +93,7 @@ SYM_FUNC_START(atomic64_\func\()_return_cx8) movl %ebx, %eax movl %ecx, %edx popl %ebx - ret + RET SYM_FUNC_END(atomic64_\func\()_return_cx8) .endm
@@ -118,7 +118,7 @@ SYM_FUNC_START(atomic64_dec_if_positive_cx8) movl %ebx, %eax movl %ecx, %edx popl %ebx - ret + RET SYM_FUNC_END(atomic64_dec_if_positive_cx8) SYM_FUNC_START(atomic64_add_unless_cx8)
@@ -149,7 +149,7 @@ SYM_FUNC_START(atomic64_add_unless_cx8) addl $8, %esp popl %ebx popl %ebp - ret + RET 4: cmpl %edx, 4(%esp) jne 2b
@@ -176,5 +176,5 @@ SYM_FUNC_START(atomic64_inc_not_zero_cx8) movl $1, %eax 3: popl %ebx - ret + RET SYM_FUNC_END(atomic64_inc_not_zero_cx8)
arch/x86/lib/checksum_32.S +4 -4
@@ -127,7 +127,7 @@ SYM_FUNC_START(csum_partial) 8: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #else
@@ -245,7 +245,7 @@ SYM_FUNC_START(csum_partial) 90: popl %ebx popl %esi - ret + RET SYM_FUNC_END(csum_partial) #endif
@@ -371,7 +371,7 @@ EXC( movb %cl, (%edi) ) popl %esi popl %edi popl %ecx # equivalent to addl $4,%esp - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #else
@@ -447,7 +447,7 @@ EXC( movb %dl, (%edi) ) popl %esi popl %edi popl %ebx - ret + RET SYM_FUNC_END(csum_partial_copy_generic) #undef ROUND
arch/x86/lib/clear_page_64.S +3 -3
@@ -17,7 +17,7 @@ SYM_FUNC_START(clear_page_rep) movl $4096/8,%ecx xorl %eax,%eax rep stosq - ret + RET SYM_FUNC_END(clear_page_rep) EXPORT_SYMBOL_GPL(clear_page_rep)
@@ -39,7 +39,7 @@ SYM_FUNC_START(clear_page_orig) leaq 64(%rdi),%rdi jnz .Lloop nop - ret + RET SYM_FUNC_END(clear_page_orig) EXPORT_SYMBOL_GPL(clear_page_orig)
@@ -47,6 +47,6 @@ SYM_FUNC_START(clear_page_erms) movl $4096,%ecx xorl %eax,%eax rep stosb - ret + RET SYM_FUNC_END(clear_page_erms) EXPORT_SYMBOL_GPL(clear_page_erms)
arch/x86/lib/cmpxchg16b_emu.S +2 -2
@@ -37,11 +37,11 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu) popfq mov $1, %al - ret + RET .Lnot_same: popfq xor %al,%al - ret + RET SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
arch/x86/lib/cmpxchg8b_emu.S +2 -2
@@ -32,7 +32,7 @@ SYM_FUNC_START(cmpxchg8b_emu) movl %ecx, 4(%esi) popfl - ret + RET .Lnot_same: movl (%esi), %eax
@@ -40,7 +40,7 @@ SYM_FUNC_START(cmpxchg8b_emu) movl 4(%esi), %edx popfl - ret + RET SYM_FUNC_END(cmpxchg8b_emu) EXPORT_SYMBOL(cmpxchg8b_emu)
arch/x86/lib/copy_mc_64.S +3 -3
@@ -77,7 +77,7 @@ SYM_FUNC_START(copy_mc_fragile) .L_done_memcpy_trap: xorl %eax, %eax .L_done: - ret + RET SYM_FUNC_END(copy_mc_fragile) .section .fixup, "ax"
@@ -132,7 +132,7 @@ SYM_FUNC_START(copy_mc_enhanced_fast_string) rep movsb /* Copy successful. Return zero */ xorl %eax, %eax - ret + RET SYM_FUNC_END(copy_mc_enhanced_fast_string) .section .fixup, "ax"
@@ -145,7 +145,7 @@ SYM_FUNC_END(copy_mc_enhanced_fast_string) * user-copy routines. */ movq %rcx, %rax - ret + RET .previous
arch/x86/lib/copy_page_64.S +2 -2
@@ -17,7 +17,7 @@ SYM_FUNC_START(copy_page) ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD movl $4096/8, %ecx rep movsq - ret + RET SYM_FUNC_END(copy_page) EXPORT_SYMBOL(copy_page)
@@ -85,5 +85,5 @@ SYM_FUNC_START_LOCAL(copy_page_regs) movq (%rsp), %rbx movq 1*8(%rsp), %r12 addq $2*8, %rsp - ret + RET SYM_FUNC_END(copy_page_regs)
arch/x86/lib/copy_user_64.S +5 -5
@@ -105,7 +105,7 @@ SYM_FUNC_START(copy_user_generic_unrolled) jnz 21b 23: xor %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 30: shll $6,%ecx
@@ -173,7 +173,7 @@ SYM_FUNC_START(copy_user_generic_string) movsb xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 11: leal (%rdx,%rcx,8),%ecx
@@ -207,7 +207,7 @@ SYM_FUNC_START(copy_user_enhanced_fast_string) movsb xorl %eax,%eax ASM_CLAC - ret + RET .section .fixup,"ax" 12: movl %ecx,%edx /* ecx is zerorest also */
@@ -237,7 +237,7 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail) 1: rep movsb 2: mov %ecx,%eax ASM_CLAC - ret + RET _ASM_EXTABLE_CPY(1b, 2b) SYM_CODE_END(.Lcopy_user_handle_tail)
@@ -348,7 +348,7 @@ SYM_FUNC_START(__copy_user_nocache) xorl %eax,%eax ASM_CLAC sfence - ret + RET .section .fixup,"ax" .L_fixup_4x8b_copy:
arch/x86/lib/csum-copy_64.S +1 -1
@@ -201,7 +201,7 @@ SYM_FUNC_START(csum_partial_copy_generic) movq 3*8(%rsp), %r13 movq 4*8(%rsp), %r15 addq $5*8, %rsp - ret + RET .Lshort: movl %ecx, %r10d jmp .L1
arch/x86/lib/getuser.S +11 -11
@@ -57,7 +57,7 @@ SYM_FUNC_START(__get_user_1) 1: movzbl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_1) EXPORT_SYMBOL(__get_user_1)
@@ -71,7 +71,7 @@ SYM_FUNC_START(__get_user_2) 2: movzwl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_2) EXPORT_SYMBOL(__get_user_2)
@@ -85,7 +85,7 @@ SYM_FUNC_START(__get_user_4) 3: movl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_4) EXPORT_SYMBOL(__get_user_4)
@@ -100,7 +100,7 @@ SYM_FUNC_START(__get_user_8) 4: movq (%_ASM_AX),%rdx xor %eax,%eax ASM_CLAC - ret + RET #else LOAD_TASK_SIZE_MINUS_N(7) cmp %_ASM_DX,%_ASM_AX
@@ -112,7 +112,7 @@ SYM_FUNC_START(__get_user_8) 5: movl 4(%_ASM_AX),%ecx xor %eax,%eax ASM_CLAC - ret + RET #endif SYM_FUNC_END(__get_user_8) EXPORT_SYMBOL(__get_user_8)
@@ -124,7 +124,7 @@ SYM_FUNC_START(__get_user_nocheck_1) 6: movzbl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_1) EXPORT_SYMBOL(__get_user_nocheck_1)
@@ -134,7 +134,7 @@ SYM_FUNC_START(__get_user_nocheck_2) 7: movzwl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_2) EXPORT_SYMBOL(__get_user_nocheck_2)
@@ -144,7 +144,7 @@ SYM_FUNC_START(__get_user_nocheck_4) 8: movl (%_ASM_AX),%edx xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_4) EXPORT_SYMBOL(__get_user_nocheck_4)
@@ -159,7 +159,7 @@ SYM_FUNC_START(__get_user_nocheck_8) #endif xor %eax,%eax ASM_CLAC - ret + RET SYM_FUNC_END(__get_user_nocheck_8) EXPORT_SYMBOL(__get_user_nocheck_8)
@@ -169,7 +169,7 @@ SYM_CODE_START_LOCAL(.Lbad_get_user_clac) bad_get_user: xor %edx,%edx mov $(-EFAULT),%_ASM_AX - ret + RET SYM_CODE_END(.Lbad_get_user_clac) #ifdef CONFIG_X86_32
@@ -179,7 +179,7 @@ bad_get_user_8: xor %edx,%edx xor %ecx,%ecx mov $(-EFAULT),%_ASM_AX - ret + RET SYM_CODE_END(.Lbad_get_user_8_clac) #endif
arch/x86/lib/hweight.S +3 -3
@@ -32,7 +32,7 @@ SYM_FUNC_START(__sw_hweight32) imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101 shrl $24, %eax # w = w_tmp >> 24 __ASM_SIZE(pop,) %__ASM_REG(dx) - ret + RET SYM_FUNC_END(__sw_hweight32) EXPORT_SYMBOL(__sw_hweight32)
@@ -65,7 +65,7 @@ SYM_FUNC_START(__sw_hweight64) popq %rdx popq %rdi - ret + RET #else /* CONFIG_X86_32 */ /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */ pushl %ecx
@@ -77,7 +77,7 @@ SYM_FUNC_START(__sw_hweight64) addl %ecx, %eax # result popl %ecx - ret + RET #endif SYM_FUNC_END(__sw_hweight64) EXPORT_SYMBOL(__sw_hweight64)
arch/x86/lib/iomap_copy_64.S +1 -1
@@ -11,5 +11,5 @@ SYM_FUNC_START(__iowrite32_copy) movl %edx,%ecx rep movsd - ret + RET SYM_FUNC_END(__iowrite32_copy)
arch/x86/lib/memcpy_64.S +6 -6
@@ -39,7 +39,7 @@ SYM_FUNC_START_WEAK(memcpy) rep movsq movl %edx, %ecx rep movsb - ret + RET SYM_FUNC_END(memcpy) SYM_FUNC_END_ALIAS(__memcpy) EXPORT_SYMBOL(memcpy)
@@ -53,7 +53,7 @@ SYM_FUNC_START_LOCAL(memcpy_erms) movq %rdi, %rax movq %rdx, %rcx rep movsb - ret + RET SYM_FUNC_END(memcpy_erms) SYM_FUNC_START_LOCAL(memcpy_orig)
@@ -137,7 +137,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq %r9, 1*8(%rdi) movq %r10, -2*8(%rdi, %rdx) movq %r11, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_16bytes: cmpl $8, %edx
@@ -149,7 +149,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movq -1*8(%rsi, %rdx), %r9 movq %r8, 0*8(%rdi) movq %r9, -1*8(%rdi, %rdx) - retq + RET .p2align 4 .Lless_8bytes: cmpl $4, %edx
@@ -162,7 +162,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movl -4(%rsi, %rdx), %r8d movl %ecx, (%rdi) movl %r8d, -4(%rdi, %rdx) - retq + RET .p2align 4 .Lless_3bytes: subl $1, %edx
@@ -180,7 +180,7 @@ SYM_FUNC_START_LOCAL(memcpy_orig) movb %cl, (%rdi) .Lend: - retq + RET SYM_FUNC_END(memcpy_orig) .popsection
arch/x86/lib/memmove_64.S +2 -2
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) /* FSRM implies ERMS => no length checks, do the copy directly */ .Lmemmove_begin_forward: ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS + ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS /* * movsq instruction have many startup latency
@@ -205,7 +205,7 @@ SYM_FUNC_START(__memmove) movb (%rsi), %r11b movb %r11b, (%rdi) 13: - retq + RET SYM_FUNC_END(__memmove) SYM_FUNC_END_ALIAS(memmove) EXPORT_SYMBOL(__memmove)
arch/x86/lib/memset_64.S +3 -3
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memset) movl %edx,%ecx rep stosb movq %r9,%rax - ret + RET SYM_FUNC_END(__memset) SYM_FUNC_END_ALIAS(memset) EXPORT_SYMBOL(memset)
@@ -63,7 +63,7 @@ SYM_FUNC_START_LOCAL(memset_erms) movq %rdx,%rcx rep stosb movq %r9,%rax - ret + RET SYM_FUNC_END(memset_erms) SYM_FUNC_START_LOCAL(memset_orig)
@@ -125,7 +125,7 @@ SYM_FUNC_START_LOCAL(memset_orig) .Lende: movq %r10,%rax - ret + RET .Lbad_alignment: cmpq $7,%rdx
arch/x86/lib/msr-reg.S +2 -2
@@ -35,7 +35,7 @@ SYM_FUNC_START(\op\()_safe_regs) movl %edi, 28(%r10) popq %r12 popq %rbx - ret + RET 3: movl $-EIO, %r11d jmp 2b
@@ -77,7 +77,7 @@ SYM_FUNC_START(\op\()_safe_regs) popl %esi popl %ebp popl %ebx - ret + RET 3: movl $-EIO, 4(%esp) jmp 2b
arch/x86/lib/putuser.S +3 -3
@@ -52,7 +52,7 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL) 1: movb %al,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_1) EXPORT_SYMBOL(__put_user_1) EXPORT_SYMBOL(__put_user_nocheck_1)
@@ -66,7 +66,7 @@ SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL) 2: movw %ax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_2) EXPORT_SYMBOL(__put_user_2) EXPORT_SYMBOL(__put_user_nocheck_2)
@@ -80,7 +80,7 @@ SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL) 3: movl %eax,(%_ASM_CX) xor %ecx,%ecx ASM_CLAC - ret + RET SYM_FUNC_END(__put_user_4) EXPORT_SYMBOL(__put_user_4) EXPORT_SYMBOL(__put_user_nocheck_4)
arch/x86/lib/retpoline.S +1 -1
@@ -23,7 +23,7 @@ .Ldo_rop_\@: mov %\reg, (%_ASM_SP) UNWIND_HINT_FUNC - ret + RET .endm .macro THUNK reg
arch/x86/math-emu/div_Xsig.S +1 -1
@@ -341,7 +341,7 @@ L_exit: popl %esi leave - ret + RET #ifdef PARANOID
arch/x86/math-emu/div_small.S +1 -1
@@ -44,5 +44,5 @@ SYM_FUNC_START(FPU_div_small) popl %esi leave - ret + RET SYM_FUNC_END(FPU_div_small)
arch/x86/math-emu/mul_Xsig.S +3 -3
@@ -62,7 +62,7 @@ SYM_FUNC_START(mul32_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul32_Xsig)
@@ -115,7 +115,7 @@ SYM_FUNC_START(mul64_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul64_Xsig)
@@ -175,5 +175,5 @@ SYM_FUNC_START(mul_Xsig_Xsig) popl %esi leave - ret + RET SYM_FUNC_END(mul_Xsig_Xsig)
arch/x86/math-emu/polynom_Xsig.S +1 -1
@@ -133,5 +133,5 @@ L_accum_done: popl %edi popl %esi leave - ret + RET SYM_FUNC_END(polynomial_Xsig)
arch/x86/math-emu/reg_norm.S +3 -3
@@ -72,7 +72,7 @@ L_exit_valid: L_exit: popl %ebx leave - ret + RET L_zero:
@@ -138,7 +138,7 @@ L_exit_nuo_valid: popl %ebx leave - ret + RET L_exit_nuo_zero: movl TAG_Zero,%eax
@@ -146,5 +146,5 @@ L_exit_nuo_zero: popl %ebx leave - ret + RET SYM_FUNC_END(FPU_normalize_nuo)
arch/x86/math-emu/reg_round.S +1 -1
@@ -437,7 +437,7 @@ fpu_Arith_exit: popl %edi popl %esi leave - ret + RET /*
arch/x86/math-emu/reg_u_add.S +1 -1
@@ -164,6 +164,6 @@ L_exit: popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_add)
arch/x86/math-emu/reg_u_div.S +1 -1
@@ -468,7 +468,7 @@ L_exit: popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_div)
arch/x86/math-emu/reg_u_mul.S +1 -1
@@ -144,7 +144,7 @@ L_exit: popl %edi popl %esi leave - ret + RET #endif /* PARANOID */ SYM_FUNC_END(FPU_u_mul)
arch/x86/math-emu/reg_u_sub.S +1 -1
@@ -270,5 +270,5 @@ L_exit: popl %edi popl %esi leave - ret + RET SYM_FUNC_END(FPU_u_sub)
arch/x86/math-emu/round_Xsig.S +2 -2
@@ -78,7 +78,7 @@ L_exit: popl %esi popl %ebx leave - ret + RET SYM_FUNC_END(round_Xsig)
@@ -138,5 +138,5 @@ L_n_exit: popl %esi popl %ebx leave - ret + RET SYM_FUNC_END(norm_Xsig)
arch/x86/math-emu/shr_Xsig.S +4 -4
@@ -45,7 +45,7 @@ SYM_FUNC_START(shr_Xsig) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx
@@ -61,7 +61,7 @@ L_more_than_31: movl $0,8(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx
@@ -76,7 +76,7 @@ L_more_than_63: movl %edx,8(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax
@@ -85,5 +85,5 @@ L_more_than_95: movl %eax,8(%esi) popl %esi leave - ret + RET SYM_FUNC_END(shr_Xsig)
arch/x86/math-emu/wm_shrx.S +8 -8
@@ -55,7 +55,7 @@ SYM_FUNC_START(FPU_shrx) popl %ebx popl %esi leave - ret + RET L_more_than_31: cmpl $64,%ecx
@@ -70,7 +70,7 @@ L_more_than_31: movl $0,4(%esi) popl %esi leave - ret + RET L_more_than_63: cmpl $96,%ecx
@@ -84,7 +84,7 @@ L_more_than_63: movl %edx,4(%esi) popl %esi leave - ret + RET L_more_than_95: xorl %eax,%eax
@@ -92,7 +92,7 @@ L_more_than_95: movl %eax,4(%esi) popl %esi leave - ret + RET SYM_FUNC_END(FPU_shrx)
@@ -146,7 +146,7 @@ SYM_FUNC_START(FPU_shrxs) popl %ebx popl %esi leave - ret + RET /* Shift by [0..31] bits */ Ls_less_than_32:
@@ -163,7 +163,7 @@ Ls_less_than_32: popl %ebx popl %esi leave - ret + RET /* Shift by [64..95] bits */ Ls_more_than_63:
@@ -189,7 +189,7 @@ Ls_more_than_63: popl %ebx popl %esi leave - ret + RET Ls_more_than_95: /* Shift by [96..inf) bits */
@@ -203,5 +203,5 @@ Ls_more_than_95: popl %ebx popl %esi leave - ret + RET SYM_FUNC_END(FPU_shrxs)
arch/x86/mm/mem_encrypt_boot.S +2 -2
@@ -65,7 +65,7 @@ SYM_FUNC_START(sme_encrypt_execute) movq %rbp, %rsp /* Restore original stack pointer */ pop %rbp - ret + RET SYM_FUNC_END(sme_encrypt_execute) SYM_FUNC_START(__enc_copy)
@@ -151,6 +151,6 @@ SYM_FUNC_START(__enc_copy) pop %r12 pop %r15 - ret + RET .L__enc_copy_end: SYM_FUNC_END(__enc_copy)
arch/x86/platform/efi/efi_stub_32.S +1 -1
@@ -56,5 +56,5 @@ SYM_FUNC_START(efi_call_svam) movl 16(%esp), %ebx leave - ret + RET SYM_FUNC_END(efi_call_svam)
arch/x86/platform/efi/efi_stub_64.S +1 -1
@@ -23,5 +23,5 @@ SYM_FUNC_START(__efi_call) mov %rsi, %rcx CALL_NOSPEC rdi leave - ret + RET SYM_FUNC_END(__efi_call)
arch/x86/platform/efi/efi_thunk_64.S +1 -1
@@ -63,7 +63,7 @@ SYM_CODE_START(__efi64_thunk) 1: movq 24(%rsp), %rsp pop %rbx pop %rbp - retq + RET .code32 2: pushl $__KERNEL_CS
arch/x86/platform/olpc/xo1-wakeup.S +3 -3
@@ -77,7 +77,7 @@ save_registers: pushfl popl saved_context_eflags - ret + RET restore_registers: movl saved_context_ebp, %ebp
@@ -88,7 +88,7 @@ restore_registers: pushl saved_context_eflags popfl - ret + RET SYM_CODE_START(do_olpc_suspend_lowlevel) call save_processor_state
@@ -109,7 +109,7 @@ ret_point: call restore_registers call restore_processor_state - ret + RET SYM_CODE_END(do_olpc_suspend_lowlevel) .data
arch/x86/power/hibernate_asm_32.S +2 -2
@@ -32,7 +32,7 @@ SYM_FUNC_START(swsusp_arch_suspend) FRAME_BEGIN call swsusp_save FRAME_END - ret + RET SYM_FUNC_END(swsusp_arch_suspend) SYM_CODE_START(restore_image)
@@ -108,5 +108,5 @@ SYM_FUNC_START(restore_registers) /* tell the hibernation core that we've just restored the memory */ movl %eax, in_suspend - ret + RET SYM_FUNC_END(restore_registers)
arch/x86/power/hibernate_asm_64.S +2 -2
@@ -66,7 +66,7 @@ SYM_FUNC_START(restore_registers) /* tell the hibernation core that we've just restored the memory */ movq %rax, in_suspend(%rip) - ret + RET SYM_FUNC_END(restore_registers) SYM_FUNC_START(swsusp_arch_suspend)
@@ -96,7 +96,7 @@ SYM_FUNC_START(swsusp_arch_suspend) FRAME_BEGIN call swsusp_save FRAME_END - ret + RET SYM_FUNC_END(swsusp_arch_suspend) SYM_FUNC_START(restore_image)
arch/x86/um/checksum_32.S +2 -2
@@ -110,7 +110,7 @@ csum_partial: 7: popl %ebx popl %esi - ret + RET #else
@@ -208,7 +208,7 @@ csum_partial: 80: popl %ebx popl %esi - ret + RET #endif EXPORT_SYMBOL(csum_partial)
arch/x86/um/setjmp_32.S +1 -1
@@ -34,7 +34,7 @@ kernel_setjmp: movl %esi,12(%edx) movl %edi,16(%edx) movl %ecx,20(%edx) # Return address - ret + RET .size kernel_setjmp,.-kernel_setjmp
arch/x86/um/setjmp_64.S +1 -1
@@ -33,7 +33,7 @@ kernel_setjmp: movq %r14,40(%rdi) movq %r15,48(%rdi) movq %rsi,56(%rdi) # Return address - ret + RET .size kernel_setjmp,.-kernel_setjmp
arch/x86/xen/xen-asm.S +6 -6
@@ -29,7 +29,7 @@ */ SYM_FUNC_START(xen_irq_disable_direct) movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask - ret + RET SYM_FUNC_END(xen_irq_disable_direct) /*
@@ -58,7 +58,7 @@ SYM_FUNC_START(check_events) pop %rcx pop %rax FRAME_END - ret + RET SYM_FUNC_END(check_events) /*
@@ -84,7 +84,7 @@ SYM_FUNC_START(xen_irq_enable_direct) call check_events 1: FRAME_END - ret + RET SYM_FUNC_END(xen_irq_enable_direct) /*
@@ -100,7 +100,7 @@ SYM_FUNC_START(xen_save_fl_direct) testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask setz %ah addb %ah, %ah - ret + RET SYM_FUNC_END(xen_save_fl_direct) SYM_FUNC_START(xen_read_cr2)
@@ -108,14 +108,14 @@ SYM_FUNC_START(xen_read_cr2) _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2); SYM_FUNC_START(xen_read_cr2_direct) FRAME_BEGIN _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX FRAME_END - ret + RET SYM_FUNC_END(xen_read_cr2_direct); .popsection
arch/x86/xen/xen-head.S +1 -1
@@ -26,7 +26,7 @@ SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) UNWIND_HINT_FUNC .skip 31, 0x90 - ret + RET .endr #define HYPERCALL(n) \
Peter Zijlstra
maintainerPATCH v2 3/6 x86: Prepare inline-asm for straight-line-speculation
In reply to Peter ZijlstraReplace all ret/retq instructions with ASM_RET in preparation of making it more than a single instruction.
--- arch/x86/include/asm/linkage.h | 4 ++++ arch/x86/include/asm/paravirt.h | 2 +- arch/x86/include/asm/qspinlock_paravirt.h | 4 ++-- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/kprobes/core.c | 2 +- arch/x86/kernel/paravirt.c | 4 ++-- arch/x86/kvm/emulate.c | 4 ++-- arch/x86/lib/error-inject.c | 3 ++- samples/ftrace/ftrace-direct-modify.c | 4 ++-- samples/ftrace/ftrace-direct-too.c | 2 +- samples/ftrace/ftrace-direct.c | 2 +- 11 files changed, 19 insertions(+), 14 deletions(-)
arch/x86/include/asm/linkage.h +4 -0
@@ -18,6 +18,10 @@#define __ALIGN_STR __stringify(__ALIGN)#endif +#else /* __ASSEMBLY__ */ + +#define ASM_RET "ret\n\t" +#endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_LINKAGE_H */
arch/x86/include/asm/paravirt.h +1 -1
@@ -671,7 +671,7 @@ bool __raw_callee_save___native_vcpu_is_ "call " #func ";" \ PV_RESTORE_ALL_CALLER_REGS \ FRAME_END \ - "ret;" \ + ASM_RET \ ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ ".popsection")
arch/x86/include/asm/qspinlock_paravirt.h +2 -2
@@ -48,7 +48,7 @@ asm (".pushsection .text;""jne .slowpath;" "pop %rdx;" FRAME_END -"ret;" +ASM_RET ".slowpath: " "push %rsi;" "movzbl %al,%esi;"
@@ -56,7 +56,7 @@ asm (".pushsection .text;""pop %rsi;" "pop %rdx;" FRAME_END -"ret;" +ASM_RET ".size " PV_UNLOCK ", .-" PV_UNLOCK ";" ".popsection");
arch/x86/kernel/alternative.c +1 -1
@@ -714,7 +714,7 @@ asm (" .type int3_magic, @function\n" "int3_magic:\n" " movl $1, (%" _ASM_ARG1")\n" -" ret\n" +ASM_RET " .size int3_magic, .-int3_magic\n" " .popsection\n" );
arch/x86/kernel/kprobes/core.c +1 -1
@@ -1051,7 +1051,7 @@ asm(" addl $4, %esp\n" " popfl\n"#endif -" ret\n" +ASM_RET ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n" );NOKPROBE_SYMBOL (__kretprobe_trampoline);
arch/x86/kernel/paravirt.c +2 -2
@@ -41,7 +41,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" -"ret\n\t" +ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t" ".popsection" );
@@ -51,7 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n".global paravirt_ret0\n" "paravirt_ret0:\n\t" "xor %" _ASM_AX", %" _ASM_AX";\n\t" -"ret\n\t" +ASM_RET ".size paravirt_ret0, . - paravirt_ret0\n\t" ".type paravirt_ret0, @function\n\t" ".popsection");
arch/x86/kvm/emulate.c +2 -2
@@ -315,7 +315,7 @@ static int fastop(struct x86_emulate_ctx __FOP_FUNC(#name)#define __FOP_RET (name) \ - "ret \n\t" \ + ASM_RET \ ".size " name ", .-" name "\n\t"#define FOP_RET (name) \
@@ -435,7 +435,7 @@ static int fastop(struct x86_emulate_ctx __FOP_RET(#op) asm(".pushsection .fixup, \"ax\"\n" -"kvm_fastop_exception: xor %esi, %esi; ret\n" +"kvm_fastop_exception: xor %esi, %esi; " ASM_RET ".popsection" );FOP_START (setcc)
arch/x86/lib/error-inject.c +2 -1
@@ -1,5 +1,6 @@// SPDX-License-Identifier: GPL-2.0 +#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h>
@@ -10,7 +11,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" -" ret\n" +ASM_RET ".size just_return_func, .-just_return_func\n" );
samples/ftrace/ftrace-direct-modify.c +2 -2
@@ -31,7 +31,7 @@ asm (" call my_direct_func1\n" " leave\n" " .size my_tramp1, .-my_tramp1\n" -" ret\n" +ASM_RET " .type my_tramp2, @function\n" " .globl my_tramp2\n" " my_tramp2:"
@@ -39,7 +39,7 @@ asm (" movq %rsp, %rbp\n" " call my_direct_func2\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp2, .-my_tramp2\n" " .popsection\n" );
samples/ftrace/ftrace-direct-too.c +1 -1
@@ -31,7 +31,7 @@ asm (" popq %rsi\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
samples/ftrace/ftrace-direct.c +1 -1
@@ -25,7 +25,7 @@ asm (" call my_direct_func\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
[tip: x86/core] x86: Prepare inline-asm for straight-line-speculation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: b17c2baa305cccbd16bafa289fd743cc2db77966 Gitweb: https://git.kernel.org/tip/b17c2baa305cccbd16bafa289fd743cc2db77966 Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:41 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 08 Dec 2021 19:23:12 +01:00
x86: Prepare inline-asm for straight-line-speculation
Replace all ret/retq instructions with ASM_RET in preparation of making it more than a single instruction.
--- arch/x86/include/asm/linkage.h | 4 ++++ arch/x86/include/asm/paravirt.h | 2 +- arch/x86/include/asm/qspinlock_paravirt.h | 4 ++-- arch/x86/kernel/alternative.c | 2 +- arch/x86/kernel/kprobes/core.c | 2 +- arch/x86/kernel/paravirt.c | 4 ++-- arch/x86/kvm/emulate.c | 4 ++-- arch/x86/lib/error-inject.c | 3 ++- samples/ftrace/ftrace-direct-modify.c | 4 ++-- samples/ftrace/ftrace-direct-too.c | 2 +- samples/ftrace/ftrace-direct.c | 2 +- 11 files changed, 19 insertions(+), 14 deletions(-)
arch/x86/include/asm/linkage.h +4 -0
@@ -18,6 +18,10 @@#define __ALIGN_STR __stringify(__ALIGN)#endif +#else /* __ASSEMBLY__ */ + +#define ASM_RET "ret\n\t" +#endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_LINKAGE_H */
arch/x86/include/asm/paravirt.h +1 -1
@@ -671,7 +671,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); "call " #func ";" \ PV_RESTORE_ALL_CALLER_REGS \ FRAME_END \ - "ret;" \ + ASM_RET \ ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ ".popsection")
arch/x86/include/asm/qspinlock_paravirt.h +2 -2
@@ -48,7 +48,7 @@ asm (".pushsection .text;""jne .slowpath;" "pop %rdx;" FRAME_END -"ret;" +ASM_RET ".slowpath: " "push %rsi;" "movzbl %al,%esi;"
@@ -56,7 +56,7 @@ asm (".pushsection .text;""pop %rsi;" "pop %rdx;" FRAME_END -"ret;" +ASM_RET ".size " PV_UNLOCK ", .-" PV_UNLOCK ";" ".popsection");
arch/x86/kernel/alternative.c +1 -1
@@ -714,7 +714,7 @@ asm (" .type int3_magic, @function\n" "int3_magic:\n" " movl $1, (%" _ASM_ARG1")\n" -" ret\n" +ASM_RET " .size int3_magic, .-int3_magic\n" " .popsection\n" );
arch/x86/kernel/kprobes/core.c +1 -1
@@ -1051,7 +1051,7 @@ asm(" addl $4, %esp\n" " popfl\n"#endif -" ret\n" +ASM_RET ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n" );NOKPROBE_SYMBOL (__kretprobe_trampoline);
arch/x86/kernel/paravirt.c +2 -2
@@ -41,7 +41,7 @@ extern void _paravirt_nop(void); asm (".pushsection .entry.text, \"ax\"\n" ".global _paravirt_nop\n" "_paravirt_nop:\n\t" -"ret\n\t" +ASM_RET ".size _paravirt_nop, . - _paravirt_nop\n\t" ".type _paravirt_nop, @function\n\t" ".popsection" );
@@ -51,7 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n"".global paravirt_ret0\n" "paravirt_ret0:\n\t" "xor %" _ASM_AX", %" _ASM_AX";\n\t" -"ret\n\t" +ASM_RET ".size paravirt_ret0, . - paravirt_ret0\n\t" ".type paravirt_ret0, @function\n\t" ".popsection");
arch/x86/kvm/emulate.c +2 -2
@@ -315,7 +315,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); __FOP_FUNC(#name)#define __FOP_RET (name) \ - "ret \n\t" \ + ASM_RET \ ".size " name ", .-" name "\n\t"#define FOP_RET (name) \
@@ -435,7 +435,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); __FOP_RET(#op) asm(".pushsection .fixup, \"ax\"\n" -"kvm_fastop_exception: xor %esi, %esi; ret\n" +"kvm_fastop_exception: xor %esi, %esi; " ASM_RET ".popsection" );FOP_START (setcc)
arch/x86/lib/error-inject.c +2 -1
@@ -1,5 +1,6 @@// SPDX-License-Identifier: GPL-2.0 +#include <linux/linkage.h> #include <linux/error-injection.h> #include <linux/kprobes.h>
@@ -10,7 +11,7 @@ asm(".type just_return_func, @function\n" ".globl just_return_func\n" "just_return_func:\n" -" ret\n" +ASM_RET ".size just_return_func, .-just_return_func\n" );
samples/ftrace/ftrace-direct-modify.c +2 -2
@@ -31,7 +31,7 @@ asm (" call my_direct_func1\n" " leave\n" " .size my_tramp1, .-my_tramp1\n" -" ret\n" +ASM_RET " .type my_tramp2, @function\n" " .globl my_tramp2\n" " my_tramp2:"
@@ -39,7 +39,7 @@ asm (" movq %rsp, %rbp\n" " call my_direct_func2\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp2, .-my_tramp2\n" " .popsection\n" );
samples/ftrace/ftrace-direct-too.c +1 -1
@@ -31,7 +31,7 @@ asm (" popq %rsi\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
samples/ftrace/ftrace-direct.c +1 -1
@@ -25,7 +25,7 @@ asm (" call my_direct_func\n" " popq %rdi\n" " leave\n" -" ret\n" +ASM_RET " .size my_tramp, .-my_tramp\n" " .popsection\n" );
Peter Zijlstra
maintainerPATCH v2 4/6 objtool: Add straight-line-speculation validation
In reply to Peter ZijlstraTeach objtool to validate the straight-line-speculation constraints:
- speculation trap after indirect calls
- speculation trap after ret
Notable: when an instruction is annotated RETPOLINE_SAFE, indicating
speculation isn't a problem, also don't care about sls for that instruction.
--- tools/objtool/arch/x86/decode.c | 13 +++++++++---- tools/objtool/builtin-check.c | 3 ++- tools/objtool/check.c | 14 ++++++++++++++ tools/objtool/include/objtool/arch.h | 1 + tools/objtool/include/objtool/builtin.h | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-)
tools/objtool/arch/x86/decode.c +9 -4
@@ -531,6 +531,11 @@ int arch_decode_instruction(struct objto }break ; +case 0xcc : +/* int3 */ + *type =INSN_TRAP ; +break ; +case 0xe3 :/* jecxz/jrcxz */ *type =INSN_JUMP_CONDITIONAL ;
@@ -697,10 +702,10 @@ const char *arch_ret_insn(int len) {static const char ret[5 ][5 ] = { {BYTE_RET }, - {BYTE_RET ,BYTES_NOP1 }, - {BYTE_RET ,BYTES_NOP2 }, - {BYTE_RET ,BYTES_NOP3 }, - {BYTE_RET ,BYTES_NOP4 }, + {BYTE_RET ,0xcc }, + {BYTE_RET ,0xcc ,BYTES_NOP1 }, + {BYTE_RET ,0xcc ,BYTES_NOP2 }, + {BYTE_RET ,0xcc ,BYTES_NOP3 }, };if (len <1 || len >5 ) {
tools/objtool/builtin-check.c +2 -1
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup; + validate_dup, vmlinux, mcount, noinstr, backup, sls;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -45,6 +45,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('l' ,"vmlinux" , &vmlinux,"vmlinux.o validation" ),OPT_BOOLEAN ('M' ,"mcount" , &mcount,"generate __mcount_loc" ),OPT_BOOLEAN ('B' ,"backup" , &backup,"create .orig files before modification" ), +OPT_BOOLEAN ('S' ,"sls" , &sls,"validate straight-line-speculation" ), OPT_END(), };
tools/objtool/check.c +14 -0
@@ -3084,6 +3084,12 @@ static int validate_branch(struct objtooswitch (insn->type) {case INSN_RETURN : +if (next_insn && next_insn->type ==INSN_TRAP ) { + next_insn->ignore = true; + }else if (sls && !insn->retpoline_safe) { +WARN_FUNC ("missing int3 after ret" , + insn->sec, insn->offset); + }return validate_return (func, insn, &state);case INSN_CALL :
@@ -3127,6 +3133,14 @@ static int validate_branch(struct objtoobreak ;case INSN_JUMP_DYNAMIC : +if (next_insn && next_insn->type ==INSN_TRAP ) { + next_insn->ignore = true; + }else if (sls && !insn->retpoline_safe) { +WARN_FUNC ("missing int3 after indirect jump" , + insn->sec, insn->offset); + } + +/* fallthrough */ case INSN_JUMP_DYNAMIC_CONDITIONAL :if (is_sibling_call (insn)) { ret =validate_sibling_call (file, insn, &state);
tools/objtool/include/objtool/arch.h +1 -0
@@ -26,6 +26,7 @@ enum insn_type {INSN_CLAC ,INSN_STD ,INSN_CLD , +INSN_TRAP , INSN_OTHER, };
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup; + validate_dup, vmlinux, mcount, noinstr, backup, sls;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
[tip: x86/core] objtool: Add straight-line-speculation validation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 1cc1e4c8aab4213bd4e6353dec2620476a233d6d Gitweb: https://git.kernel.org/tip/1cc1e4c8aab4213bd4e6353dec2620476a233d6d Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:42 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Wed, 08 Dec 2021 19:26:50 +01:00
objtool: Add straight-line-speculation validation
Teach objtool to validate the straight-line-speculation constraints:
- speculation trap after indirect calls
- speculation trap after RET
Notable: when an instruction is annotated RETPOLINE_SAFE, indicating
speculation isn't a problem, also don't care about sls for that instruction.
--- tools/objtool/arch/x86/decode.c | 13 +++++++++---- tools/objtool/builtin-check.c | 3 ++- tools/objtool/check.c | 14 ++++++++++++++ tools/objtool/include/objtool/arch.h | 1 + tools/objtool/include/objtool/builtin.h | 2 +- 5 files changed, 27 insertions(+), 6 deletions(-)
tools/objtool/arch/x86/decode.c +9 -4
@@ -531,6 +531,11 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec }break ; +case 0xcc : +/* int3 */ + *type =INSN_TRAP ; +break ; +case 0xe3 :/* jecxz/jrcxz */ *type =INSN_JUMP_CONDITIONAL ;
@@ -697,10 +702,10 @@ const char *arch_ret_insn(int len) {static const char ret[5 ][5 ] = { {BYTE_RET }, - {BYTE_RET ,BYTES_NOP1 }, - {BYTE_RET ,BYTES_NOP2 }, - {BYTE_RET ,BYTES_NOP3 }, - {BYTE_RET ,BYTES_NOP4 }, + {BYTE_RET ,0xcc }, + {BYTE_RET ,0xcc ,BYTES_NOP1 }, + {BYTE_RET ,0xcc ,BYTES_NOP2 }, + {BYTE_RET ,0xcc ,BYTES_NOP3 }, };if (len <1 || len >5 ) {
tools/objtool/builtin-check.c +2 -1
@@ -20,7 +20,7 @@#include <objtool/objtool.h> bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup; + validate_dup, vmlinux, mcount, noinstr, backup, sls;static const char *const check_usage[] = {"objtool check [<options>] file.o" ,
@@ -45,6 +45,7 @@ const struct option check_options[] = {OPT_BOOLEAN ('l' ,"vmlinux" , &vmlinux,"vmlinux.o validation" ),OPT_BOOLEAN ('M' ,"mcount" , &mcount,"generate __mcount_loc" ),OPT_BOOLEAN ('B' ,"backup" , &backup,"create .orig files before modification" ), +OPT_BOOLEAN ('S' ,"sls" , &sls,"validate straight-line-speculation" ), OPT_END(), };
tools/objtool/check.c +14 -0
@@ -3084,6 +3084,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,switch (insn->type) {case INSN_RETURN : +if (next_insn && next_insn->type ==INSN_TRAP ) { + next_insn->ignore = true; + }else if (sls && !insn->retpoline_safe) { +WARN_FUNC ("missing int3 after ret" , + insn->sec, insn->offset); + }return validate_return (func, insn, &state);case INSN_CALL :
@@ -3127,6 +3133,14 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,break ;case INSN_JUMP_DYNAMIC : +if (next_insn && next_insn->type ==INSN_TRAP ) { + next_insn->ignore = true; + }else if (sls && !insn->retpoline_safe) { +WARN_FUNC ("missing int3 after indirect jump" , + insn->sec, insn->offset); + } + +/* fallthrough */ case INSN_JUMP_DYNAMIC_CONDITIONAL :if (is_sibling_call (insn)) { ret =validate_sibling_call (file, insn, &state);
tools/objtool/include/objtool/arch.h +1 -0
@@ -26,6 +26,7 @@ enum insn_type {INSN_CLAC ,INSN_STD ,INSN_CLD , +INSN_TRAP , INSN_OTHER, };
tools/objtool/include/objtool/builtin.h +1 -1
@@ -9,7 +9,7 @@extern const struct option check_options[];extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - validate_dup, vmlinux, mcount, noinstr, backup; + validate_dup, vmlinux, mcount, noinstr, backup, sls;extern int cmd_parse_options (int argc,const char **argv,const char *const usage[]);
Peter Zijlstra
maintainerPATCH v2 5/6 x86/alternative: Relax text_poke_bp() constraint
In reply to Peter ZijlstraCurrently text_poke_bp() is very strict to only allow patching a single instruction; however with straight-line-speculation it will be required to patch: ret; int3, which is two instructions.
As such, relax the constraints a little to allow int3 padding for all instructions that do not imply the execution of the next instruction,
ie: RET, JMP.d8 and JMP.d32.
While there, rename the text_poke_loc::rel32 field to ::disp.
Note: this fills up the text_poke_loc structure which is now a round
16 bytes big.
--- arch/x86/kernel/alternative.c | 44 ++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-)
arch/x86/kernel/alternative.c +30 -14
@@ -1114,10 +1114,11 @@ void text_poke_sync(void)struct text_poke_loc {s32 rel_addr;/* addr := _stext + rel_addr */ -s32 rel32; +s32 disp; +u8 len;u8 opcode;const u8 text[POKE_MAX_OPCODE_SIZE ]; -u8 old; +u8 old;/* see text_poke_bp_batch() */ };struct bp_patching_desc {
@@ -1165,7 +1166,7 @@ noinstr int poke_int3_handler(struct pt_ {struct bp_patching_desc *desc;struct text_poke_loc *tp; -int len, ret =0 ; +int ret =0 ;void *ip;if (user_mode (regs))
@@ -1205,8 +1206,7 @@ noinstr int poke_int3_handler(struct pt_ goto out_put; } - len =text_opcode_size (tp->opcode); - ip += len; + ip += tp->len;switch (tp->opcode) {case INT3_INSN_OPCODE :
@@ -1221,12 +1221,12 @@ noinstr int poke_int3_handler(struct pt_break ;case CALL_INSN_OPCODE : -int3_emulate_call (regs, (long )ip + tp->rel32); +int3_emulate_call (regs, (long )ip + tp->disp);break ;case JMP32_INSN_OPCODE :case JMP8_INSN_OPCODE : -int3_emulate_jmp (regs, (long )ip + tp->rel32); +int3_emulate_jmp (regs, (long )ip + tp->disp);break ;default :
@@ -1301,7 +1301,7 @@ static void text_poke_bp_batch(struct te*/ for (do_sync =0 , i =0 ; i < nr_entries; i++) {u8 old[POKE_MAX_OPCODE_SIZE ] = { tp[i].old, }; -int len =text_opcode_size (tp[i].opcode); +int len = tp[i].len;if (len -INT3_INSN_SIZE >0 ) { memcpy(old + INT3_INSN_SIZE,
@@ -1378,20 +1378,36 @@ static void text_poke_loc_init(struct te const void *opcode, size_t len, const void *emulate) { struct insn insn; -int ret; +int ret, i;memcpy ((void *)tp->text, opcode, len);if (!emulate) emulate = opcode; ret =insn_decode_kernel (&insn, emulate); -BUG_ON (ret <0 ); -BUG_ON (len != insn.length); tp->rel_addr = addr - (void *)_stext; + tp->len = len; tp->opcode = insn.opcode.bytes[0 ]; +switch (tp->opcode) { +case RET_INSN_OPCODE : +case JMP32_INSN_OPCODE : +case JMP8_INSN_OPCODE : +/* +* Control flow instructions without implied execution of the +* next instruction can be padded with INT3. +*/ +for (i = insn.length; i < len; i++) +BUG_ON (tp->text[i] !=INT3_INSN_OPCODE ); +break ; + +default : +BUG_ON (len != insn.length); + }; + +switch (tp->opcode) {case INT3_INSN_OPCODE :case RET_INSN_OPCODE :
@@ -1400,7 +1416,7 @@ static void text_poke_loc_init(struct tecase CALL_INSN_OPCODE :case JMP32_INSN_OPCODE :case JMP8_INSN_OPCODE : - tp->rel32 = insn.immediate.value; + tp->disp = insn.immediate.value;break ;default :/* assume NOP */
@@ -1408,13 +1424,13 @@ static void text_poke_loc_init(struct tecase 2 :/* NOP2 -- emulate as JMP8+0 */ BUG_ON (memcmp (emulate, x86_nops[len], len)); tp->opcode =JMP8_INSN_OPCODE ; - tp->rel32 =0 ; + tp->disp =0 ;break ;case 5 :/* NOP5 -- emulate as JMP32+0 */ BUG_ON (memcmp (emulate, x86_nops[len], len)); tp->opcode =JMP32_INSN_OPCODE ; - tp->rel32 =0 ; + tp->disp =0 ;break ;default :/* unknown instruction */
[tip: x86/core] x86/alternative: Relax text_poke_bp() constraint
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 26c44b776dba4ac692a0bf5a3836feb8a63fea6b Gitweb: https://git.kernel.org/tip/26c44b776dba4ac692a0bf5a3836feb8a63fea6b Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:43 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Thu, 09 Dec 2021 11:04:50 +01:00
x86/alternative: Relax text_poke_bp() constraint
Currently, text_poke_bp() is very strict to only allow patching a single instruction; however with straight-line-speculation it will be required to patch: ret; int3, which is two instructions.
As such, relax the constraints a little to allow int3 padding for all instructions that do not imply the execution of the next instruction,
ie: RET, JMP.d8 and JMP.d32.
While there, rename the text_poke_loc::rel32 field to ::disp.
Note: this fills up the text_poke_loc structure which is now a round
16 bytes big.
[ bp: Put comments ontop instead of on the side. ]
--- arch/x86/kernel/alternative.c | 49 +++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-)
arch/x86/kernel/alternative.c +34 -15
@@ -1113,10 +1113,13 @@ void text_poke_sync(void) }struct text_poke_loc { -s32 rel_addr;/* addr := _stext + rel_addr */ -s32 rel32; +/* addr := _stext + rel_addr */ +s32 rel_addr; +s32 disp; +u8 len;u8 opcode;const u8 text[POKE_MAX_OPCODE_SIZE ]; +/* see text_poke_bp_batch() */ u8 old; };
@@ -1131,7 +1134,8 @@ static struct bp_patching_desc *bp_desc;static __always_inline struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) { -struct bp_patching_desc *desc =__READ_ONCE (*descp);/* rcu_dereference */ +/* rcu_dereference */ +struct bp_patching_desc *desc =__READ_ONCE (*descp);if (!desc || !arch_atomic_inc_not_zero (&desc->refs))return NULL ;
@@ -1165,7 +1169,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) {struct bp_patching_desc *desc;struct text_poke_loc *tp; -int len, ret =0 ; +int ret =0 ;void *ip;if (user_mode (regs))
@@ -1205,8 +1209,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) goto out_put; } - len =text_opcode_size (tp->opcode); - ip += len; + ip += tp->len;switch (tp->opcode) {case INT3_INSN_OPCODE :
@@ -1221,12 +1224,12 @@ noinstr int poke_int3_handler(struct pt_regs *regs)break ;case CALL_INSN_OPCODE : -int3_emulate_call (regs, (long )ip + tp->rel32); +int3_emulate_call (regs, (long )ip + tp->disp);break ;case JMP32_INSN_OPCODE :case JMP8_INSN_OPCODE : -int3_emulate_jmp (regs, (long )ip + tp->rel32); +int3_emulate_jmp (regs, (long )ip + tp->disp);break ;default :
@@ -1301,7 +1304,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries*/ for (do_sync =0 , i =0 ; i < nr_entries; i++) {u8 old[POKE_MAX_OPCODE_SIZE ] = { tp[i].old, }; -int len =text_opcode_size (tp[i].opcode); +int len = tp[i].len;if (len -INT3_INSN_SIZE >0 ) { memcpy(old + INT3_INSN_SIZE,
@@ -1378,21 +1381,37 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, const void *opcode, size_t len, const void *emulate) { struct insn insn; -int ret; +int ret, i;memcpy ((void *)tp->text, opcode, len);if (!emulate) emulate = opcode; ret =insn_decode_kernel (&insn, emulate); -BUG_ON (ret <0 ); -BUG_ON (len != insn.length); tp->rel_addr = addr - (void *)_stext; + tp->len = len; tp->opcode = insn.opcode.bytes[0 ];switch (tp->opcode) { +case RET_INSN_OPCODE : +case JMP32_INSN_OPCODE : +case JMP8_INSN_OPCODE : +/* +* Control flow instructions without implied execution of the +* next instruction can be padded with INT3. +*/ +for (i = insn.length; i < len; i++) +BUG_ON (tp->text[i] !=INT3_INSN_OPCODE ); +break ; + +default : +BUG_ON (len != insn.length); + }; + + +switch (tp->opcode) {case INT3_INSN_OPCODE :case RET_INSN_OPCODE :break ;
@@ -1400,7 +1419,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,case CALL_INSN_OPCODE :case JMP32_INSN_OPCODE :case JMP8_INSN_OPCODE : - tp->rel32 = insn.immediate.value; + tp->disp = insn.immediate.value;break ;default :/* assume NOP */
@@ -1408,13 +1427,13 @@ static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,case 2 :/* NOP2 -- emulate as JMP8+0 */ BUG_ON (memcmp (emulate, x86_nops[len], len)); tp->opcode =JMP8_INSN_OPCODE ; - tp->rel32 =0 ; + tp->disp =0 ;break ;case 5 :/* NOP5 -- emulate as JMP32+0 */ BUG_ON (memcmp (emulate, x86_nops[len], len)); tp->opcode =JMP32_INSN_OPCODE ; - tp->rel32 =0 ; + tp->disp =0 ;break ;default :/* unknown instruction */
Peter Zijlstra
maintainerPATCH v2 6/6 x86: Add straight-line-speculation mitigation
In reply to Peter ZijlstraMake use of an upcomming GCC feature to mitigate
straight-line-speculation for x86:
https://gcc.gnu.org/g:53a643f8568067d7700a9f2facc8ba39974973d3
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102952
https://bugs.llvm.org/show_bug.cgi?id=52323
It's built tested on x86_64-allyesconfig using GCC-12 and GCC-11.
Maintenace overhead of this should be fairly low due to objtool validation.
Size overhead of all these additional int3 instructions comes to:
text data bss dec hex filename
22267751 6933356 2011368 31212475 1dc43bb defconfig-build/vmlinux
22804126 6933356 1470696 31208178 1dc32f2 defconfig-build/vmlinux.sls
Or roughly 2.4% additional text.
--- arch/x86/Kconfig | 12 ++++++++++++ arch/x86/Makefile | 4 ++++ arch/x86/include/asm/linkage.h | 10 ++++++++++ arch/x86/include/asm/static_call.h | 2 +- arch/x86/kernel/ftrace.c | 2 +- arch/x86/kernel/static_call.c | 5 +++-- arch/x86/lib/memmove_64.S | 2 +- arch/x86/lib/retpoline.S | 2 +- scripts/Makefile.build | 3 ++- scripts/link-vmlinux.sh | 3 +++ 10 files changed, 38 insertions(+), 7 deletions(-)
arch/x86/Kconfig +12 -0
@@ -472,6 +472,18 @@ config RETPOLINE branches. Requires a compiler with -mindirect-branch=thunk-extern support for full protection. The kernel may run slower. +config CC_HAS_SLS + def_bool $(cc-option,-mharden-sls=all) + +config SLS + bool "Mitigate Straight-Line-Speculation" + depends on CC_HAS_SLS && X86_64 + default n + help + Compile the kernel with straight-line-speculation options to guard + against straight line speculation. The kernel image might be slightly + larger. + config X86_CPU_RESCTRL bool "x86 CPU resource control support" depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
arch/x86/Makefile +4 -0
@@ -179,6 +179,10 @@ ifdef CONFIG_RETPOLINE endif endif +ifdef CONFIG_SLS + KBUILD_CFLAGS += -mharden-sls=all +endif + KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE) ifdef CONFIG_LTO_CLANG
arch/x86/include/asm/linkage.h +10 -0
@@ -18,9 +18,19 @@#define __ALIGN_STR __stringify(__ALIGN)#endif +#ifdef CONFIG_SLS +#define RET ret; int3 +#else +#define RET ret +#endif +#else /* __ASSEMBLY__ */ +#ifdef CONFIG_SLS +#define ASM_RET "ret; int3\n\t" +#else #define ASM_RET "ret\n\t" +#endif #endif /* __ASSEMBLY__ */
arch/x86/include/asm/static_call.h +1 -1
@@ -36,7 +36,7 @@ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP (name) \ - __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop")#define ARCH_ADD_TRAMP_KEY (name) \
arch/x86/kernel/ftrace.c +1 -1
@@ -303,7 +303,7 @@ union ftrace_op_code_union { } __attribute__((packed)); }; -#define RET_SIZE 1 +#define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS)static unsigned long create_trampoline (struct ftrace_ops *ops,unsigned int *tramp_size)
arch/x86/kernel/static_call.c +3 -2
@@ -17,6 +17,8 @@ enum insn_type {*/ static const u8 xor5rax[] = {0x66 ,0x66 ,0x48 ,0x31 ,0xc0 }; +static const u8 retinsn[] = {RET_INSN_OPCODE ,0xcc ,0xcc ,0xcc ,0xcc }; +static void __ref __static_call_transform (void *insn,enum insn_type type,void *func) {const void *emulate =NULL ;
@@ -42,8 +44,7 @@ static void __ref __static_call_transforbreak ;case RET : - code =text_gen_insn (RET_INSN_OPCODE , insn, func); - size =RET_INSN_SIZE ; + code = &retinsn;break ; }
arch/x86/lib/memmove_64.S +1 -1
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) /* FSRM implies ERMS => no length checks, do the copy directly */ .Lmemmove_begin_forward: ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS + ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS /* * movsq instruction have many startup latency
arch/x86/lib/retpoline.S +1 -1
@@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\re ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD .endm
scripts/Makefile.build +2 -1
@@ -234,7 +234,8 @@ objtool_args = \ $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\ $(if $(CONFIG_RETPOLINE), --retpoline) \ $(if $(CONFIG_X86_SMAP), --uaccess) \ - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) + $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ + $(if $(CONFIG_SLS), --sls) cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
scripts/link-vmlinux.sh +3 -0
@@ -139,6 +139,9 @@ objtool_link() if [ -n "${CONFIG_X86_SMAP}" ]; then objtoolopt="${objtoolopt} --uaccess" fi + if [ -n "${CONFIG_SLS}" ]; then + objtoolopt="${objtoolopt} --sls" + fi info OBJTOOL ${1} tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1} fi
[tip: x86/core] x86: Add straight-line-speculation mitigation
The following commit has been merged into the x86/core branch of tip:
The following commit has been merged into the x86/core branch of tip:
Commit-ID: e463a09af2f0677b9485a7e8e4e70b396b2ffb6f Gitweb: https://git.kernel.org/tip/e463a09af2f0677b9485a7e8e4e70b396b2ffb6f Author: Peter Zijlstra <[email protected]> AuthorDate: Sat, 04 Dec 2021 14:43:44 +01:00 Committer: Borislav Petkov <[email protected]> CommitterDate: Thu, 09 Dec 2021 13:32:25 +01:00
x86: Add straight-line-speculation mitigation
Make use of an upcoming GCC feature to mitigate
straight-line-speculation for x86:
https://gcc.gnu.org/g:53a643f8568067d7700a9f2facc8ba39974973d3
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102952
https://bugs.llvm.org/show_bug.cgi?id=52323
It's built tested on x86_64-allyesconfig using GCC-12 and GCC-11.
Maintenance overhead of this should be fairly low due to objtool validation.
Size overhead of all these additional int3 instructions comes to:
text data bss dec hex filename
22267751 6933356 2011368 31212475 1dc43bb defconfig-build/vmlinux
22804126 6933356 1470696 31208178 1dc32f2 defconfig-build/vmlinux.sls
Or roughly 2.4% additional text.
--- arch/x86/Kconfig | 12 ++++++++++++ arch/x86/Makefile | 4 ++++ arch/x86/include/asm/linkage.h | 10 ++++++++++ arch/x86/include/asm/static_call.h | 2 +- arch/x86/kernel/ftrace.c | 2 +- arch/x86/kernel/static_call.c | 5 +++-- arch/x86/lib/memmove_64.S | 2 +- arch/x86/lib/retpoline.S | 2 +- scripts/Makefile.build | 3 ++- scripts/link-vmlinux.sh | 3 +++ 10 files changed, 38 insertions(+), 7 deletions(-)
arch/x86/Kconfig +12 -0
@@ -472,6 +472,18 @@ config RETPOLINE branches. Requires a compiler with -mindirect-branch=thunk-extern support for full protection. The kernel may run slower. +config CC_HAS_SLS + def_bool $(cc-option,-mharden-sls=all) + +config SLS + bool "Mitigate Straight-Line-Speculation" + depends on CC_HAS_SLS && X86_64 + default n + help + Compile the kernel with straight-line-speculation options to guard + against straight line speculation. The kernel image might be slightly + larger. + config X86_CPU_RESCTRL bool "x86 CPU resource control support" depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
arch/x86/Makefile +4 -0
@@ -191,6 +191,10 @@ ifdef CONFIG_RETPOLINE endif endif +ifdef CONFIG_SLS + KBUILD_CFLAGS += -mharden-sls=all +endif + KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE) ifdef CONFIG_LTO_CLANG
arch/x86/include/asm/linkage.h +10 -0
@@ -18,9 +18,19 @@#define __ALIGN_STR __stringify(__ALIGN)#endif +#ifdef CONFIG_SLS +#define RET ret; int3 +#else +#define RET ret +#endif +#else /* __ASSEMBLY__ */ +#ifdef CONFIG_SLS +#define ASM_RET "ret; int3\n\t" +#else #define ASM_RET "ret\n\t" +#endif #endif /* __ASSEMBLY__ */
arch/x86/include/asm/static_call.h +1 -1
@@ -36,7 +36,7 @@ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP (name) \ - __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") + __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop")#define ARCH_ADD_TRAMP_KEY (name) \
arch/x86/kernel/ftrace.c +1 -1
@@ -303,7 +303,7 @@ union ftrace_op_code_union { } __attribute__((packed)); }; -#define RET_SIZE 1 +#define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS)static unsigned long create_trampoline (struct ftrace_ops *ops,unsigned int *tramp_size)
arch/x86/kernel/static_call.c +3 -2
@@ -17,6 +17,8 @@ enum insn_type {*/ static const u8 xor5rax[] = {0x66 ,0x66 ,0x48 ,0x31 ,0xc0 }; +static const u8 retinsn[] = {RET_INSN_OPCODE ,0xcc ,0xcc ,0xcc ,0xcc }; +static void __ref __static_call_transform (void *insn,enum insn_type type,void *func) {const void *emulate =NULL ;
@@ -42,8 +44,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, voidbreak ;case RET : - code =text_gen_insn (RET_INSN_OPCODE , insn, func); - size =RET_INSN_SIZE ; + code = &retinsn;break ; }
arch/x86/lib/memmove_64.S +1 -1
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) /* FSRM implies ERMS => no length checks, do the copy directly */ .Lmemmove_begin_forward: ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM - ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS + ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS /* * movsq instruction have many startup latency
arch/x86/lib/retpoline.S +1 -1
@@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD .endm
scripts/Makefile.build +2 -1
@@ -234,7 +234,8 @@ objtool_args = \ $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\ $(if $(CONFIG_RETPOLINE), --retpoline) \ $(if $(CONFIG_X86_SMAP), --uaccess) \ - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) + $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ + $(if $(CONFIG_SLS), --sls) cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
scripts/link-vmlinux.sh +3 -0
@@ -139,6 +139,9 @@ objtool_link() if [ -n "${CONFIG_X86_SMAP}" ]; then objtoolopt="${objtoolopt} --uaccess" fi + if [ -n "${CONFIG_SLS}" ]; then + objtoolopt="${objtoolopt} --sls" + fi info OBJTOOL ${1} tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1} fi
Missing SLS int3 in JMP_NOSPEC? (Was: [PATCH v2 6/6] x86: Add straight-line-speculation mitigation)
In reply to Peter ZijlstraQuoted 22 lines · Peter Zijlstra
Make use of an upcomming GCC feature to mitigate
straight-line-speculation for x86:https://gcc.gnu.org/g:53a643f8568067d7700a9f2facc8ba39974973d3 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102952 https://bugs.llvm.org/show_bug.cgi?id=52323It's built tested on x86_64-allyesconfig using GCC-12 and GCC-11.
Maintenace overhead of this should be fairly low due to objtool validation.
Size overhead of all these additional int3 instructions comes to:
text data bss dec hex filename
22267751 6933356 2011368 31212475 1dc43bb defconfig-build/vmlinux
22804126 6933356 1470696 31208178 1dc32f2 defconfig-build/vmlinux.slsOr roughly 2.4% additional text.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
(..)
arch/x86/lib/retpoline.S SYM_INNER_LABEL(__x86_indirect_thunk_\re · line 40
Quoted 11 lines
arch/x86/lib/retpoline.S +1
-1@@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\re ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD .endm
Looking at this __x86_indirect_thunk_* change makes me wonder why there is no similar int3 SLS protection in the X86_FEATURE_RETPOLINE_LFENCE case of JMP_NOSPEC in arch/x86/include/asm/nospec-branch.h:
Quoted 6 lines
.macro
JMP_NOSPECreg:req#ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE#else
JMP_NOSPEC users seem to have no explicit trailing int3 instructions either.
Or am I missing something here?
Thanks,
Maciej
Peter Zijlstra
maintainerRFC PATCH 1/1 x86,nospec: Simplify {JMP,CALL}_NOSPEC
Quoted 43 lines · Maciej S. Szmigiero
Quoted 22 lines · Peter Zijlstra
Make use of an upcomming GCC feature to mitigate
straight-line-speculation for x86:https://gcc.gnu.org/g:53a643f8568067d7700a9f2facc8ba39974973d3 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102952 https://bugs.llvm.org/show_bug.cgi?id=52323It's built tested on x86_64-allyesconfig using GCC-12 and GCC-11.
Maintenace overhead of this should be fairly low due to objtool validation.
Size overhead of all these additional int3 instructions comes to:
text data bss dec hex filename
22267751 6933356 2011368 31212475 1dc43bb defconfig-build/vmlinux
22804126 6933356 1470696 31208178 1dc32f2 defconfig-build/vmlinux.slsOr roughly 2.4% additional text.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
(..)
arch/x86/lib/retpoline.S SYM_INNER_LABEL(__x86_indirect_thunk_\re · line 37
Quoted 8 lines
arch/x86/lib/retpoline.S +1
-1@@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\re ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD .endmLooking at this __x86_indirect_thunk_* change makes me wonder why there is no similar int3 SLS protection in the
X86_FEATURE_RETPOLINE_LFENCEcase ofJMP_NOSPECinarch/x86/include/asm/nospec-branch.h:Quoted 6 lines
.macro
JMP_NOSPECreg:req#ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE#else
Quoted 4 lines
JMP_NOSPECusers seem to have no explicit trailing int3 instructions either.Or am I missing something here?
Probably just forgot about those. I'm thinking we ought to do something like this...
---
Subject: x86,nospec: Simplify {JMP,CALL}_NOSPEC
Have {JMP,CALL}_NOSPEC generate the same code GCC does for indirect calls and rely on the objtool retpoline patching infrastructure.
There's no reason these should be alternatives while the vast bulk of compiler generated retpolines are not.
--- arch/x86/include/asm/nospec-branch.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)
arch/x86/include/asm/nospec-branch.h +18 -6
@@ -93,6 +93,19 @@#endif .endm +/* + * Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call + * to the retpoline thunk with a CS prefix when the register requires + * a RAX prefix byte to encode. Also see apply_alternatives(). + */ +.macro __CS_PREFIX reg:req + .irp rs,r8,r9,r10,r11,r12,r13,r14,r15 + .ifc \reg,\rs + .byte 0x2e + .endif + .endr +.endm +/* * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple * indirect jmp/call which may be susceptible to the Spectre variant 2
@@ -100,19 +113,18 @@*/ .macro JMP_NOSPEC reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ - __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE + __CS_PREFIX \reg + jmp __x86_indirect_thunk_\reg #else jmp *%\reg + int3 #endif .endm .macro CALL_NOSPEC reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ - __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE + __CS_PREFIX \reg + call __x86_indirect_thunk_\reg #else call *%\reg #endif
Peter Zijlstra
maintainerRe: [RFC][PATCH] x86,nospec: Simplify {JMP,CALL}_NOSPEC
Quoted 18 lines · Peter Zijlstra
Subject: x86,nospec: Simplify {JMP,CALL}_NOSPEC
Have {JMP,CALL}_NOSPEC generate the same code GCC does for indirect calls and rely on the objtool retpoline patching infrastructure.
There's no reason these should be alternatives while the vast bulk of compiler generated retpolines are not.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/include/asm/nospec-branch.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)arch/x86/include/asm/nospec-branch.h +0
-0
arch/x86/include/asm/nospec-branch.h line 99
Quoted 7 lines
arch/x86/include/asm/nospec-branch.h +4
-0@@ -93,6 +93,19 @@#endif .endm +/* +* Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call +* to the retpoline thunk with a CS prefix when the register requires +* a RAX prefix byte to encode. Also see apply_alternatives().
Obviously I meant: apply_retpolines() ...
Quoted 37 lines
arch/x86/include/asm/nospec-branch.h +14
-6+*/ +.macro __CS_PREFIX reg:req + .irp rs,r8,r9,r10,r11,r12,r13,r14,r15 + .ifc \reg,\rs + .byte 0x2e + .endif + .endr +.endm +/* * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple * indirect jmp/call which may be susceptible to the Spectre variant 2 @@ -100,19 +113,18 @@*/ .macro JMP_NOSPEC reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ - __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE + __CS_PREFIX \reg + jmp __x86_indirect_thunk_\reg #else jmp *%\reg + int3 #endif .endm .macro CALL_NOSPEC reg:req #ifdef CONFIG_RETPOLINE - ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ - __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE + __CS_PREFIX \reg + call __x86_indirect_thunk_\reg #else call *%\reg #endif
Re: [RFC][PATCH] x86,nospec: Simplify {JMP,CALL}_NOSPEC
Quoted 22 lines · Peter Zijlstra
Quoted 21 lines · Peter Zijlstra
Subject: x86,nospec: Simplify {JMP,CALL}_NOSPEC
Have {JMP,CALL}_NOSPEC generate the same code GCC does for indirect calls and rely on the objtool retpoline patching infrastructure.
There's no reason these should be alternatives while the vast bulk of compiler generated retpolines are not.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/include/asm/nospec-branch.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)arch/x86/include/asm/nospec-branch.h +0
-0@@ -93,6 +93,19 @@#endif .endm
Quoted 6 lines
arch/x86/include/asm/nospec-branch.h line 99
Quoted 4 lines
arch/x86/include/asm/nospec-branch.h +4
-0+/* +* Equivalent to -mindirect-branch-cs-prefix; emit the 5 byte jmp/call +* to the retpoline thunk with a CS prefix when the register requires +* a RAX prefix byte to encode. Also see apply_alternatives(). Obviously I meant:
apply_retpolines()...
Will apply_retpolines() actually patch in that trailing int3 in the X86_FEATURE_RETPOLINE_LFENCE case?
Looking at its code it uses just ordinary NOPs as fill:
Quoted 9 lines
/*
* For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
*/
if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
bytes[i++] = 0x0f;
bytes[i++] = 0xae;
bytes[i++] = 0xe8; /* LFENCE */
}Quoted 7 lines
ret = emit_indirect(op, reg, bytes + i); if (ret < 0) return ret; i += ret; for (; i < insn->length;) bytes[i++] = BYTES_NOP1;
BYTES_NOP1 is 0x90.
Quoted 17 lines
Quoted 17 lines
arch/x86/include/asm/nospec-branch.h +9
-1+*/ +.macro __CS_PREFIX reg:req + .irp rs,r8,r9,r10,r11,r12,r13,r14,r15 + .ifc \reg,\rs + .byte 0x2e + .endif + .endr +.endm +/* * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple * indirect jmp/call which may be susceptible to the Spectre variant 2 @@ -100,19 +113,18 @@*/ .macro JMP_NOSPEC reg:req#ifdef CONFIG_RETPOLINE - ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
Quoted 8 lines
arch/x86/include/asm/nospec-branch.h line 121
Quoted 8 lines
arch/x86/include/asm/nospec-branch.h +3
-2- __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE + __CS_PREFIX \reg + jmp __x86_indirect_thunk_\reg#else jmp *%\reg + int3#endif
Perhaps that int3 should be here to be common to both "#ifdef CONFIG_RETPOLINE" branches?
Quoted 2 lines
arch/x86/include/asm/nospec-branch.h line 123
Quoted 2 lines
arch/x86/include/asm/nospec-branch.h +0
-0.endm
Thanks,
Maciej
Peter Zijlstra
maintainerRe: [RFC][PATCH] x86,nospec: Simplify {JMP,CALL}_NOSPEC
Quoted 22 lines · Maciej S. Szmigiero
Quoted 1 line
Obviously I meant:
apply_retpolines()...Will
apply_retpolines()actually patch in that trailing int3 in theX86_FEATURE_RETPOLINE_LFENCEcase?Looking at its code it uses just ordinary NOPs as fill:
Quoted 16 lines
/* * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE. */ if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { bytes[i++] = 0x0f; bytes[i++] = 0xae; bytes[i++] = 0xe8; /* LFENCE */ } ret = emit_indirect(op, reg, bytes + i); if (ret < 0) return ret; i += ret; for (; i < insn->length;) bytes[i++] = BYTES_NOP1;
There is no space for int3 in that case. You get 3 bytes for LFENCE and {2,3} bytes for 'jmp *%reg', which fully consumes the {5,6} bytes available.
There will be no nops added.
But this is what all regular retpolines get to look like.
The plan was; and that's still pending, to get the INT3 from the AMD BTC mitigation that adds INT3 after regular JMPs but those compiler patches still need to happen I think.