arm64: optimize mov_q assembler macro
PATCH 1/3 arm64: optimize mov_q assembler macro
From: Remi Denis-Courmont <[email protected]>
This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:
1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.
2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.
3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.
4) Use MOVN if it saves one or more MOVK over MOVZ.
Note that the assembler uses -1 as truth value (not +1 like C).
--- arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-)
arch/arm64/include/asm/assembler.h +40 -5
@@ -473,22 +473,57 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU/* * mov_q - move an immediate constant into a 64-bit register using -* between 2 and 4 movz/movk instructions (depending on the +* between 1 and 4 mov/movk instructions (depending on the * magnitude and sign of the operand) */ .macro mov_q, reg, val - .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) - movz \reg, :abs_g1_s:\val + .if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff)) + .if (((\val) >> 16) == 0) + movz \reg, :abs_g0:\val .else - .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) - movz \reg, :abs_g2_s:\val + .if (((\val) >> 32) == 0) + movz \reg, :abs_g1:\val + .else + .if (((\val) >> 48) == 0) + movz \reg, :abs_g2:\val .else movz \reg, :abs_g3:\val + .if ((((\val) >> 32) & 0xffff) != 0) movk \reg, :abs_g2_nc:\val .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0) movk \reg, :abs_g1_nc:\val .endif + .endif + .if (((\val) & 0xffff) != 0) movk \reg, :abs_g0_nc:\val + .endif + .endif + .else + .if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff) + movn \reg, :abs_g0:~\val + .else + .if ((((\val) >> 32) & 0xffffffff) == 0xffffffff) + movn \reg, :abs_g1:~\val + .else + .if ((((\val) >> 48) & 0xffff) == 0xffff) + movn \reg, :abs_g2:~\val + .else + movn \reg, :abs_g3:~\val + .if ((((\val) >> 32) & 0xffff) != 0xffff) + movk \reg, :abs_g2_nc:\val + .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0xffff) + movk \reg, :abs_g1_nc:\val + .endif + .endif + .if (((\val) & 0xffff) != 0xffff) + movk \reg, :abs_g0_nc:\val + .endif + .endif + .endif .endm/*
Signature · 7 lines
2.25.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Mark Rutland
reviewer[adding arm64 folk]
When sendign a series like this, please send a cover letter, and thread patches in reply to the cover letter.
Use git format-patch --cover to generate that, and git send-email should thread things that way by default.
Quoted 11 lines · Rémi Denis-Courmont
From: Remi Denis-Courmont <[email protected]>
This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:
1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.
2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.
Quoted 8 lines
3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.
4) Use MOVN if it saves one or more MOVK over MOVZ.
Note that the assembler uses -1 as truth value (not +1 like C).
Signed-off-by Remi Denis-Courmont <[email protected]>
To be honest, I don't think this is worthwhile given how complex it makes the mov_q macro.
Given the places we use mov_q today, I don't think this is going to give any measurable performance improvement, but will make debugging more painful than necessary.
Thanks,
Mark
Quoted 79 lines
--- arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-)arch/arm64/include/asm/assembler.h +40
-5@@ -473,22 +473,57 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU/* * mov_q - move an immediate constant into a 64-bit register using -* between 2 and 4 movz/movk instructions (depending on the +* between 1 and 4 mov/movk instructions (depending on the * magnitude and sign of the operand) */ .macro mov_q, reg, val - .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) - movz \reg, :abs_g1_s:\val + .if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff)) + .if (((\val) >> 16) == 0) + movz \reg, :abs_g0:\val .else - .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) - movz \reg, :abs_g2_s:\val + .if (((\val) >> 32) == 0) + movz \reg, :abs_g1:\val + .else + .if (((\val) >> 48) == 0) + movz \reg, :abs_g2:\val .else movz \reg, :abs_g3:\val + .if ((((\val) >> 32) & 0xffff) != 0) movk \reg, :abs_g2_nc:\val .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0) movk \reg, :abs_g1_nc:\val .endif + .endif + .if (((\val) & 0xffff) != 0) movk \reg, :abs_g0_nc:\val + .endif + .endif + .else + .if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff) + movn \reg, :abs_g0:~\val + .else + .if ((((\val) >> 32) & 0xffffffff) == 0xffffffff) + movn \reg, :abs_g1:~\val + .else + .if ((((\val) >> 48) & 0xffff) == 0xffff) + movn \reg, :abs_g2:~\val + .else + movn \reg, :abs_g3:~\val + .if ((((\val) >> 32) & 0xffff) != 0xffff) + movk \reg, :abs_g2_nc:\val + .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0xffff) + movk \reg, :abs_g1_nc:\val + .endif + .endif + .if (((\val) & 0xffff) != 0xffff) + movk \reg, :abs_g0_nc:\val + .endif + .endif + .endif .endm/* Signature · 7 lines
2.25.1 _______________________________________________ linux-arm-kernel mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
Quoted 28 lines · Mark Rutland
[adding arm64 folk]
When sendign a series like this, please send a cover letter, and thread patches in reply to the cover letter.
Use
git format-patch --coverto generate that, andgit send-emailshould thread things that way by default.Quoted 18 lines · Rémi Denis-Courmont
From: Remi Denis-Courmont <[email protected]>
This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:
1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.
2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.
3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.
4) Use MOVN if it saves one or more MOVK over MOVZ.
Note that the assembler uses -1 as truth value (not +1 like C).
Quoted 9 lines
Quoted 1 line
Signed-off-by Remi Denis-Courmont <[email protected]>To be honest, I don't think this is worthwhile given how complex it makes the
mov_qmacro.Given the places we use
mov_qtoday, I don't think this is going to give any measurable performance improvement, but will make debugging more painful than necessary.
Agreed. For macros this complex, you better have some test code that we can run to prove its correctness. But actually, this is something we should bring up with the toolchain people: it would be much more useful to have a pseudo-op for this that we can use everywhere, and which will resolve to the 4 instruction sequence (and the associated ELF relocations) by default, and gets optimized into something smaller by the assembler if the value is known at that point.
Quoted 75 lines
Quoted 75 lines
--- arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-)arch/arm64/include/asm/assembler.h +40
-6@@ -473,22 +473,57 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU/* * mov_q - move an immediate constant into a 64-bit register using -* between 2 and 4 movz/movk instructions (depending on the +* between 1 and 4 mov/movk instructions (depending on the * magnitude and sign of the operand) */ .macro mov_q, reg, val - .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) - movz \reg, :abs_g1_s:\val + .if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff)) + .if (((\val) >> 16) == 0) + movz \reg, :abs_g0:\val .else - .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) - movz \reg, :abs_g2_s:\val + .if (((\val) >> 32) == 0) + movz \reg, :abs_g1:\val + .else + .if (((\val) >> 48) == 0) + movz \reg, :abs_g2:\val .else movz \reg, :abs_g3:\val + .if ((((\val) >> 32) & 0xffff) != 0) movk \reg, :abs_g2_nc:\val .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0) movk \reg, :abs_g1_nc:\val .endif + .endif + .if (((\val) & 0xffff) != 0) movk \reg, :abs_g0_nc:\val + .endif + .endif + .else + .if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff) + movn \reg, :abs_g0:~\val + .else + .if ((((\val) >> 32) & 0xffffffff) == 0xffffffff) + movn \reg, :abs_g1:~\val + .else + .if ((((\val) >> 48) & 0xffff) == 0xffff) + movn \reg, :abs_g2:~\val + .else + movn \reg, :abs_g3:~\val + .if ((((\val) >> 32) & 0xffff) != 0xffff) + movk \reg, :abs_g2_nc:\val + .endif + .endif + .if ((((\val) >> 16) & 0xffff) != 0xffff) + movk \reg, :abs_g1_nc:\val + .endif + .endif + .if (((\val) & 0xffff) != 0xffff) + movk \reg, :abs_g0_nc:\val + .endif + .endif + .endif .endm/* -- 2.25.1
Quoted 4 lines
Quoted 4 lines
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel