objtool: Fix fallthrough detection for vmlinux
cover objtool: Fix fallthrough detection for vmlinux
Remove the 'c_file' hack and fix function fallthrough detection for vmlinux.
Josh Poimboeuf (4):
x86/uaccess: Don't jump between functions
objtool: Don't set 'jump_dest' for sibling calls
objtool: Fix sibling call detection in alternatives
objtool: Fix function fallthrough detection for vmlinux
arch/x86/lib/copy_user_64.S | 87 +++++++++++++++---------- tools/objtool/check.c | 73 +++++++++++---------- tools/objtool/include/objtool/objtool.h | 2 +- tools/objtool/objtool.c | 1 - 4 files changed, 93 insertions(+), 70 deletions(-)
Signature · 1 line
2.34.1
PATCH 1/4 x86/uaccess: Don't jump between functions
For unwinding sanity, a function shouldn't jump to the middle of another function. Move the short string user copy code out to a separate non-function code snippet.
--- arch/x86/lib/copy_user_64.S | 87 ++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 35 deletions(-)
arch/x86/lib/copy_user_64.S +52 -35
@@ -53,12 +53,12 @@ SYM_FUNC_START(copy_user_generic_unrolled) ASM_STAC cmpl $8,%edx - jb 20f /* less then 8 bytes, go to byte copy loop */ + jb .Lcopy_user_short_string_bytes ALIGN_DESTINATION movl %edx,%ecx andl $63,%edx shrl $6,%ecx - jz .L_copy_short_string + jz copy_user_short_string 1: movq (%rsi),%r8 2: movq 1*8(%rsi),%r9 3: movq 2*8(%rsi),%r10
@@ -79,37 +79,11 @@ SYM_FUNC_START(copy_user_generic_unrolled) leaq 64(%rdi),%rdi decl %ecx jnz 1b -.L_copy_short_string: - movl %edx,%ecx - andl $7,%edx - shrl $3,%ecx - jz 20f -18: movq (%rsi),%r8 -19: movq %r8,(%rdi) - leaq 8(%rsi),%rsi - leaq 8(%rdi),%rdi - decl %ecx - jnz 18b -20: andl %edx,%edx - jz 23f - movl %edx,%ecx -21: movb (%rsi),%al -22: movb %al,(%rdi) - incq %rsi - incq %rdi - decl %ecx - jnz 21b -23: xor %eax,%eax - ASM_CLAC - RET + jmp copy_user_short_string 30: shll $6,%ecx addl %ecx,%edx - jmp 60f -40: leal (%rdx,%rcx,8),%edx - jmp 60f -50: movl %ecx,%edx -60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */ + jmp .Lcopy_user_handle_tail _ASM_EXTABLE_CPY(1b, 30b) _ASM_EXTABLE_CPY(2b, 30b)
@@ -127,10 +101,6 @@ SYM_FUNC_START(copy_user_generic_unrolled) _ASM_EXTABLE_CPY(14b, 30b) _ASM_EXTABLE_CPY(15b, 30b) _ASM_EXTABLE_CPY(16b, 30b) - _ASM_EXTABLE_CPY(18b, 40b) - _ASM_EXTABLE_CPY(19b, 40b) - _ASM_EXTABLE_CPY(21b, 50b) - _ASM_EXTABLE_CPY(22b, 50b) SYM_FUNC_END(copy_user_generic_unrolled) EXPORT_SYMBOL(copy_user_generic_unrolled)
@@ -191,7 +161,7 @@ EXPORT_SYMBOL(copy_user_generic_string) SYM_FUNC_START(copy_user_enhanced_fast_string) ASM_STAC /* CPUs without FSRM should avoid rep movsb for short copies */ - ALTERNATIVE "cmpl $64, %edx; jb .L_copy_short_string", "", X86_FEATURE_FSRM + ALTERNATIVE "cmpl $64, %edx; jb copy_user_short_string", "", X86_FEATURE_FSRM movl %edx,%ecx 1: rep movsb xorl %eax,%eax
@@ -243,6 +213,53 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail) SYM_CODE_END(.Lcopy_user_handle_tail) +/* + * Finish memcpy of less than 64 bytes. #AC should already be set. + * + * Input: + * rdi destination + * rsi source + * rdx count (< 64) + * + * Output: + * eax uncopied bytes or 0 if successful. + */ +SYM_CODE_START_LOCAL(copy_user_short_string) + movl %edx,%ecx + andl $7,%edx + shrl $3,%ecx + jz .Lcopy_user_short_string_bytes +18: movq (%rsi),%r8 +19: movq %r8,(%rdi) + leaq 8(%rsi),%rsi + leaq 8(%rdi),%rdi + decl %ecx + jnz 18b +.Lcopy_user_short_string_bytes: + andl %edx,%edx + jz 23f + movl %edx,%ecx +21: movb (%rsi),%al +22: movb %al,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz 21b +23: xor %eax,%eax + ASM_CLAC + RET + +40: leal (%rdx,%rcx,8),%edx + jmp 60f +50: movl %ecx,%edx /* ecx is zerorest also */ +60: jmp .Lcopy_user_handle_tail + + _ASM_EXTABLE_CPY(18b, 40b) + _ASM_EXTABLE_CPY(19b, 40b) + _ASM_EXTABLE_CPY(21b, 50b) + _ASM_EXTABLE_CPY(22b, 50b) +SYM_CODE_END(copy_user_short_string) + /* * copy_user_nocache - Uncached memory copy with exception handling * This will force destination out of cache for more performance.
Signature · 1 line
2.34.1
PATCH 2/4 objtool: Don't set 'jump_dest' for sibling calls
In reply to Josh PoimboeufFor most sibling calls, 'jump_dest' is NULL because objtool treats the jump like a call and sets 'call_dest'. But there are a few edge cases where that's not true. Make it consistent to avoid unexpected behavior.
--- tools/objtool/check.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-)
tools/objtool/check.c +22 -13
@@ -1271,7 +1271,7 @@ static bool is_first_func_insn(struct objtool_file *file, struct instruction *in*/ static int add_jump_destinations (struct objtool_file *file) { -struct instruction *insn; +struct instruction *insn, *jump_dest;struct reloc *reloc;struct section *dest_sec;unsigned long dest_off;
@@ -1291,7 +1291,10 @@ static int add_jump_destinations(struct objtool_file *file)add_retpoline_call (file, insn);continue ; }else if (insn->func) { -/* internal or external sibling call (with reloc) */ +/* +* External sibling call or internal sibling call with +* STT_FUNC reloc. +*/ add_call_dest (file, insn, reloc->sym, true);continue ; }else if (reloc->sym->sec->idx) {
@@ -1303,8 +1306,8 @@ static int add_jump_destinations(struct objtool_file *file)continue ; } - insn->jump_dest =find_insn (file, dest_sec, dest_off); -if (!insn->jump_dest) { + jump_dest =find_insn (file, dest_sec, dest_off); +if (!jump_dest) {/* * This is a special case where an alt instruction
@@ -1323,8 +1326,8 @@ static int add_jump_destinations(struct objtool_file *file)/* * Cross-function jump. */ -if (insn->func && insn->jump_dest->func && - insn->func != insn->jump_dest->func) { +if (insn->func && jump_dest->func && + insn->func != jump_dest->func) {/* * For GCC 8+, create parent/child links for any cold
@@ -1342,16 +1345,22 @@ static int add_jump_destinations(struct objtool_file *file)* subfunction is through a jump table. */ if (!strstr (insn->func->name,".cold" ) && -strstr (insn->jump_dest->func->name,".cold" )) { - insn->func->cfunc = insn->jump_dest->func; - insn->jump_dest->func->pfunc = insn->func; +strstr (jump_dest->func->name,".cold" )) { + insn->func->cfunc = jump_dest->func; + jump_dest->func->pfunc = insn->func; - }else if (!same_function (insn, insn->jump_dest) && -is_first_func_insn (file, insn->jump_dest)) { -/* internal sibling call (without reloc) */ -add_call_dest (file, insn, insn->jump_dest->func, true); + }else if (!same_function (insn, jump_dest) && +is_first_func_insn (file, jump_dest)) { +/* +* Internal sibling call without reloc or with +* STT_SECTION reloc. +*/ +add_call_dest (file, insn, jump_dest->func, true); +continue ; } } + + insn->jump_dest = jump_dest; }return 0 ;
Signature · 1 line
2.34.1
PATCH 3/4 objtool: Fix sibling call detection in alternatives
In reply to Josh PoimboeufIn add_jump_destinations(), sibling call detection requires 'insn->func' to be valid. But alternative instructions get their 'func' set in handle_group_alt(), which runs *after* add_jump_destinations(). So sibling calls in alternatives code don't get properly detected.
Fix that by changing the initialization order: call add_special_section_alts() *before* add_jump_destinations().
This also means the special case for a missing 'jump_dest' in add_jump_destinations() can be removed, as it has already been dealt with.
--- tools/objtool/check.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-)
tools/objtool/check.c +17 -19
@@ -1277,6 +1277,13 @@ static int add_jump_destinations(struct objtool_file *file)unsigned long dest_off;for_each_insn (file, insn) { +if (insn->jump_dest) { +/* +* handle_group_alt() may have previously set +* 'jump_dest' for some alternatives. +*/ +continue ; + }if (!is_static_jump (insn))continue ;
@@ -1308,15 +1315,6 @@ static int add_jump_destinations(struct objtool_file *file) jump_dest =find_insn (file, dest_sec, dest_off);if (!jump_dest) { - -/* -* This is a special case where an alt instruction -* jumps past the end of the section. These are -* handled later in handle_group_alt(). -*/ -if (!strcmp (insn->sec->name,".altinstr_replacement" )) -continue ; -WARN_FUNC ("can't find jump dest instruction at %s+0x%lx" , insn->sec, insn->offset, dest_sec->name, dest_off);
@@ -1549,13 +1547,13 @@ static int handle_group_alt(struct objtool_file *file,continue ; dest_off =arch_jump_destination (insn); -if (dest_off == special_alt->new_off + special_alt->new_len) +if (dest_off == special_alt->new_off + special_alt->new_len) { insn->jump_dest =next_insn_same_sec (file, last_orig_insn); - -if (!insn->jump_dest) { -WARN_FUNC ("can't find alternative jump destination" , - insn->sec, insn->offset); -return -1 ; +if (!insn->jump_dest) { +WARN_FUNC ("can't find alternative jump destination" , + insn->sec, insn->offset); +return -1 ; + } } }
@@ -2254,14 +2252,14 @@ static int decode_sections(struct objtool_file *file)return ret;/* -* Must be before add_special_section_alts() as that depends on -* jump_dest being set. +* Must be before add_jump_destinations(), which depends on 'func' +* being set for alternatives, to enable proper sibling call detection. */ - ret =add_jump_destinations (file); + ret =add_special_section_alts (file);if (ret)return ret; - ret =add_special_section_alts (file); + ret =add_jump_destinations (file);if (ret)return ret;
Signature · 1 line
2.34.1
PATCH 4/4 objtool: Fix function fallthrough detection for vmlinux
In reply to Josh PoimboeufObjtool's function fallthrough detection only works on C objects. The distinction between C and assembly objects no longer makes sense with objtool running on vmlinux.o.
Now that copy_user_64.S has been fixed up, and an objtool sibling call detection bug has been fixed, the asm code is in "compliance" and this hack is no longer needed. Remove it.
--- tools/objtool/check.c | 2 +- tools/objtool/include/objtool/objtool.h | 2 +- tools/objtool/objtool.c | 1 - 3 files changed, 2 insertions(+), 3 deletions(-)
tools/objtool/check.c +1 -1
@@ -3310,7 +3310,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,while (1 ) { next_insn =next_insn_to_validate (file, insn); -if (file->c_file && func && insn->func && func != insn->func->pfunc) { +if (func && insn->func && func != insn->func->pfunc) {WARN ("%s() falls through to next function %s()" , func->name, insn->func->name);return 1 ;
tools/objtool/include/objtool/objtool.h +1 -1
@@ -27,7 +27,7 @@ struct objtool_file {struct list_head static_call_list;struct list_head mcount_loc_list;struct list_head endbr_list; -bool ignore_unreachables, c_file, hints, rodata; +bool ignore_unreachables, hints, rodata;unsigned int nr_endbr;unsigned int nr_endbr_int;
tools/objtool/objtool.c +0 -1
@@ -129,7 +129,6 @@ struct objtool_file *objtool_open_read(const char *_objname)INIT_LIST_HEAD (&file.static_call_list);INIT_LIST_HEAD (&file.mcount_loc_list);INIT_LIST_HEAD (&file.endbr_list); - file.c_file = !vmlinux &&find_section_by_name (file.elf,".comment" ); file.ignore_unreachables = no_unreachable; file.hints = false;
Signature · 1 line
2.34.1
Re: [PATCH 4/4] objtool: Fix function fallthrough detection for vmlinux
Quoted 9 lines · Josh Poimboeuf
Objtool's function fallthrough detection only works on C objects. The distinction between C and assembly objects no longer makes sense with objtool running on vmlinux.o.
Now that
copy_user_64.Shas been fixed up, and an objtool sibling call detection bug has been fixed, the asm code is in "compliance" and this hack is no longer needed. Remove it.Signed-off-by Josh Poimboeuf <[email protected]>
Signature · 1 line
Josh
[tip: objtool/urgent] objtool: Fix function fallthrough detection for vmlinux
In reply to Josh PoimboeufThe following commit has been merged into the objtool/urgent branch of tip:
The following commit has been merged into the objtool/urgent branch of tip:
Commit-ID: 08feafe8d1958febf3a9733a3d1564d8fc23340e Gitweb: https://git.kernel.org/tip/08feafe8d1958febf3a9733a3d1564d8fc23340e Author: Josh Poimboeuf <[email protected]> AuthorDate: Mon, 11 Apr 2022 16:10:32 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 19 Apr 2022 21:58:53 +02:00
objtool: Fix function fallthrough detection for vmlinux
Objtool's function fallthrough detection only works on C objects. The distinction between C and assembly objects no longer makes sense with objtool running on vmlinux.o.
Now that copy_user_64.S has been fixed up, and an objtool sibling call detection bug has been fixed, the asm code is in "compliance" and this hack is no longer needed. Remove it.
--- tools/objtool/check.c | 2 +- tools/objtool/include/objtool/objtool.h | 2 +- tools/objtool/objtool.c | 1 - 3 files changed, 2 insertions(+), 3 deletions(-)
tools/objtool/check.c +1 -1
@@ -3310,7 +3310,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,while (1 ) { next_insn =next_insn_to_validate (file, insn); -if (file->c_file && func && insn->func && func != insn->func->pfunc) { +if (func && insn->func && func != insn->func->pfunc) {WARN ("%s() falls through to next function %s()" , func->name, insn->func->name);return 1 ;
tools/objtool/include/objtool/objtool.h +1 -1
@@ -27,7 +27,7 @@ struct objtool_file {struct list_head static_call_list;struct list_head mcount_loc_list;struct list_head endbr_list; -bool ignore_unreachables, c_file, hints, rodata; +bool ignore_unreachables, hints, rodata;unsigned int nr_endbr;unsigned int nr_endbr_int;
tools/objtool/objtool.c +0 -1
@@ -129,7 +129,6 @@ struct objtool_file *objtool_open_read(const char *_objname)INIT_LIST_HEAD (&file.static_call_list);INIT_LIST_HEAD (&file.mcount_loc_list);INIT_LIST_HEAD (&file.endbr_list); - file.c_file = !vmlinux &&find_section_by_name (file.elf,".comment" ); file.ignore_unreachables = no_unreachable; file.hints = false;
[tip: objtool/urgent] objtool: Fix sibling call detection in alternatives
In reply to Josh PoimboeufThe following commit has been merged into the objtool/urgent branch of tip:
The following commit has been merged into the objtool/urgent branch of tip:
Commit-ID: 34c861e806478ac2ea4032721defbf1d6967df08 Gitweb: https://git.kernel.org/tip/34c861e806478ac2ea4032721defbf1d6967df08 Author: Josh Poimboeuf <[email protected]> AuthorDate: Mon, 11 Apr 2022 16:10:31 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 19 Apr 2022 21:58:53 +02:00
objtool: Fix sibling call detection in alternatives
In add_jump_destinations(), sibling call detection requires 'insn->func' to be valid. But alternative instructions get their 'func' set in handle_group_alt(), which runs *after* add_jump_destinations(). So sibling calls in alternatives code don't get properly detected.
Fix that by changing the initialization order: call add_special_section_alts() *before* add_jump_destinations().
This also means the special case for a missing 'jump_dest' in add_jump_destinations() can be removed, as it has already been dealt with.
--- tools/objtool/check.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-)
tools/objtool/check.c +17 -19
@@ -1277,6 +1277,13 @@ static int add_jump_destinations(struct objtool_file *file)unsigned long dest_off;for_each_insn (file, insn) { +if (insn->jump_dest) { +/* +* handle_group_alt() may have previously set +* 'jump_dest' for some alternatives. +*/ +continue ; + }if (!is_static_jump (insn))continue ;
@@ -1308,15 +1315,6 @@ static int add_jump_destinations(struct objtool_file *file) jump_dest =find_insn (file, dest_sec, dest_off);if (!jump_dest) { - -/* -* This is a special case where an alt instruction -* jumps past the end of the section. These are -* handled later in handle_group_alt(). -*/ -if (!strcmp (insn->sec->name,".altinstr_replacement" )) -continue ; -WARN_FUNC ("can't find jump dest instruction at %s+0x%lx" , insn->sec, insn->offset, dest_sec->name, dest_off);
@@ -1549,13 +1547,13 @@ static int handle_group_alt(struct objtool_file *file,continue ; dest_off =arch_jump_destination (insn); -if (dest_off == special_alt->new_off + special_alt->new_len) +if (dest_off == special_alt->new_off + special_alt->new_len) { insn->jump_dest =next_insn_same_sec (file, last_orig_insn); - -if (!insn->jump_dest) { -WARN_FUNC ("can't find alternative jump destination" , - insn->sec, insn->offset); -return -1 ; +if (!insn->jump_dest) { +WARN_FUNC ("can't find alternative jump destination" , + insn->sec, insn->offset); +return -1 ; + } } }
@@ -2254,14 +2252,14 @@ static int decode_sections(struct objtool_file *file)return ret;/* -* Must be before add_special_section_alts() as that depends on -* jump_dest being set. +* Must be before add_jump_destinations(), which depends on 'func' +* being set for alternatives, to enable proper sibling call detection. */ - ret =add_jump_destinations (file); + ret =add_special_section_alts (file);if (ret)return ret; - ret =add_special_section_alts (file); + ret =add_jump_destinations (file);if (ret)return ret;
[tip: objtool/urgent] objtool: Don't set 'jump_dest' for sibling calls
In reply to Josh PoimboeufThe following commit has been merged into the objtool/urgent branch of tip:
The following commit has been merged into the objtool/urgent branch of tip:
Commit-ID: 26ff604102c98df79c3fe2614d1b9bb068d4c28c Gitweb: https://git.kernel.org/tip/26ff604102c98df79c3fe2614d1b9bb068d4c28c Author: Josh Poimboeuf <[email protected]> AuthorDate: Mon, 11 Apr 2022 16:10:30 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 19 Apr 2022 21:58:53 +02:00
objtool: Don't set 'jump_dest' for sibling calls
For most sibling calls, 'jump_dest' is NULL because objtool treats the jump like a call and sets 'call_dest'. But there are a few edge cases where that's not true. Make it consistent to avoid unexpected behavior.
--- tools/objtool/check.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-)
tools/objtool/check.c +22 -13
@@ -1271,7 +1271,7 @@ static bool is_first_func_insn(struct objtool_file *file, struct instruction *in*/ static int add_jump_destinations (struct objtool_file *file) { -struct instruction *insn; +struct instruction *insn, *jump_dest;struct reloc *reloc;struct section *dest_sec;unsigned long dest_off;
@@ -1291,7 +1291,10 @@ static int add_jump_destinations(struct objtool_file *file)add_retpoline_call (file, insn);continue ; }else if (insn->func) { -/* internal or external sibling call (with reloc) */ +/* +* External sibling call or internal sibling call with +* STT_FUNC reloc. +*/ add_call_dest (file, insn, reloc->sym, true);continue ; }else if (reloc->sym->sec->idx) {
@@ -1303,8 +1306,8 @@ static int add_jump_destinations(struct objtool_file *file)continue ; } - insn->jump_dest =find_insn (file, dest_sec, dest_off); -if (!insn->jump_dest) { + jump_dest =find_insn (file, dest_sec, dest_off); +if (!jump_dest) {/* * This is a special case where an alt instruction
@@ -1323,8 +1326,8 @@ static int add_jump_destinations(struct objtool_file *file)/* * Cross-function jump. */ -if (insn->func && insn->jump_dest->func && - insn->func != insn->jump_dest->func) { +if (insn->func && jump_dest->func && + insn->func != jump_dest->func) {/* * For GCC 8+, create parent/child links for any cold
@@ -1342,16 +1345,22 @@ static int add_jump_destinations(struct objtool_file *file)* subfunction is through a jump table. */ if (!strstr (insn->func->name,".cold" ) && -strstr (insn->jump_dest->func->name,".cold" )) { - insn->func->cfunc = insn->jump_dest->func; - insn->jump_dest->func->pfunc = insn->func; +strstr (jump_dest->func->name,".cold" )) { + insn->func->cfunc = jump_dest->func; + jump_dest->func->pfunc = insn->func; - }else if (!same_function (insn, insn->jump_dest) && -is_first_func_insn (file, insn->jump_dest)) { -/* internal sibling call (without reloc) */ -add_call_dest (file, insn, insn->jump_dest->func, true); + }else if (!same_function (insn, jump_dest) && +is_first_func_insn (file, jump_dest)) { +/* +* Internal sibling call without reloc or with +* STT_SECTION reloc. +*/ +add_call_dest (file, insn, jump_dest->func, true); +continue ; } } + + insn->jump_dest = jump_dest; }return 0 ;
[tip: objtool/urgent] x86/uaccess: Don't jump between functions
In reply to Josh PoimboeufThe following commit has been merged into the objtool/urgent branch of tip:
The following commit has been merged into the objtool/urgent branch of tip:
Commit-ID: 02041b32256628aef0d18ec15d3658fe41bc1afe Gitweb: https://git.kernel.org/tip/02041b32256628aef0d18ec15d3658fe41bc1afe Author: Josh Poimboeuf <[email protected]> AuthorDate: Mon, 11 Apr 2022 16:10:29 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Tue, 19 Apr 2022 21:58:53 +02:00
x86/uaccess: Don't jump between functions
For unwinding sanity, a function shouldn't jump to the middle of another function. Move the short string user copy code out to a separate non-function code snippet.
--- arch/x86/lib/copy_user_64.S | 87 +++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 35 deletions(-)
arch/x86/lib/copy_user_64.S +52 -35
@@ -53,12 +53,12 @@ SYM_FUNC_START(copy_user_generic_unrolled) ASM_STAC cmpl $8,%edx - jb 20f /* less then 8 bytes, go to byte copy loop */ + jb .Lcopy_user_short_string_bytes ALIGN_DESTINATION movl %edx,%ecx andl $63,%edx shrl $6,%ecx - jz .L_copy_short_string + jz copy_user_short_string 1: movq (%rsi),%r8 2: movq 1*8(%rsi),%r9 3: movq 2*8(%rsi),%r10
@@ -79,37 +79,11 @@ SYM_FUNC_START(copy_user_generic_unrolled) leaq 64(%rdi),%rdi decl %ecx jnz 1b -.L_copy_short_string: - movl %edx,%ecx - andl $7,%edx - shrl $3,%ecx - jz 20f -18: movq (%rsi),%r8 -19: movq %r8,(%rdi) - leaq 8(%rsi),%rsi - leaq 8(%rdi),%rdi - decl %ecx - jnz 18b -20: andl %edx,%edx - jz 23f - movl %edx,%ecx -21: movb (%rsi),%al -22: movb %al,(%rdi) - incq %rsi - incq %rdi - decl %ecx - jnz 21b -23: xor %eax,%eax - ASM_CLAC - RET + jmp copy_user_short_string 30: shll $6,%ecx addl %ecx,%edx - jmp 60f -40: leal (%rdx,%rcx,8),%edx - jmp 60f -50: movl %ecx,%edx -60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */ + jmp .Lcopy_user_handle_tail _ASM_EXTABLE_CPY(1b, 30b) _ASM_EXTABLE_CPY(2b, 30b)
@@ -127,10 +101,6 @@ SYM_FUNC_START(copy_user_generic_unrolled) _ASM_EXTABLE_CPY(14b, 30b) _ASM_EXTABLE_CPY(15b, 30b) _ASM_EXTABLE_CPY(16b, 30b) - _ASM_EXTABLE_CPY(18b, 40b) - _ASM_EXTABLE_CPY(19b, 40b) - _ASM_EXTABLE_CPY(21b, 50b) - _ASM_EXTABLE_CPY(22b, 50b) SYM_FUNC_END(copy_user_generic_unrolled) EXPORT_SYMBOL(copy_user_generic_unrolled)
@@ -191,7 +161,7 @@ EXPORT_SYMBOL(copy_user_generic_string) SYM_FUNC_START(copy_user_enhanced_fast_string) ASM_STAC /* CPUs without FSRM should avoid rep movsb for short copies */ - ALTERNATIVE "cmpl $64, %edx; jb .L_copy_short_string", "", X86_FEATURE_FSRM + ALTERNATIVE "cmpl $64, %edx; jb copy_user_short_string", "", X86_FEATURE_FSRM movl %edx,%ecx 1: rep movsb xorl %eax,%eax
@@ -244,6 +214,53 @@ SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail) SYM_CODE_END(.Lcopy_user_handle_tail) /* + * Finish memcpy of less than 64 bytes. #AC should already be set. + * + * Input: + * rdi destination + * rsi source + * rdx count (< 64) + * + * Output: + * eax uncopied bytes or 0 if successful. + */ +SYM_CODE_START_LOCAL(copy_user_short_string) + movl %edx,%ecx + andl $7,%edx + shrl $3,%ecx + jz .Lcopy_user_short_string_bytes +18: movq (%rsi),%r8 +19: movq %r8,(%rdi) + leaq 8(%rsi),%rsi + leaq 8(%rdi),%rdi + decl %ecx + jnz 18b +.Lcopy_user_short_string_bytes: + andl %edx,%edx + jz 23f + movl %edx,%ecx +21: movb (%rsi),%al +22: movb %al,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz 21b +23: xor %eax,%eax + ASM_CLAC + RET + +40: leal (%rdx,%rcx,8),%edx + jmp 60f +50: movl %ecx,%edx /* ecx is zerorest also */ +60: jmp .Lcopy_user_handle_tail + + _ASM_EXTABLE_CPY(18b, 40b) + _ASM_EXTABLE_CPY(19b, 40b) + _ASM_EXTABLE_CPY(21b, 50b) + _ASM_EXTABLE_CPY(22b, 50b) +SYM_CODE_END(copy_user_short_string) + +/* * copy_user_nocache - Uncached memory copy with exception handling * This will force destination out of cache for more performance. *