Skip to contentSkip to conversation

arm64: optimize mov_q assembler macro

PATCH Applied Rémi Denis-Courmont 3 participants 3 messages Mar 4, 2020 arm64

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 arm64: optimize mov_q assembler macro

From: Remi Denis-Courmont <[email protected]>

This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:

1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.

2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.

3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.

4) Use MOVN if it saves one or more MOVK over MOVZ.

Note that the assembler uses -1 as truth value (not +1 like C).

---
 arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 5 deletions(-)
arch/arm64/include/asm/assembler.h +40 -5
index af03001293c6..2b98e2c5c8a2 100644
@@ -473,22 +473,57 @@ USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU
 
 	/*
 	 * mov_q - move an immediate constant into a 64-bit register using
-	 *         between 2 and 4 movz/movk instructions (depending on the
+	 *         between 1 and 4 mov/movk instructions (depending on the
 	 *         magnitude and sign of the operand)
 	 */
 	.macro	mov_q, reg, val
-	.if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
-	movz	\reg, :abs_g1_s:\val
+	.if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff))
+	.if (((\val) >> 16) == 0)
+	movz	\reg, :abs_g0:\val
 	.else
-	.if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
-	movz	\reg, :abs_g2_s:\val
+	.if (((\val) >> 32) == 0)
+	movz	\reg, :abs_g1:\val
+	.else
+	.if (((\val) >> 48) == 0)
+	movz	\reg, :abs_g2:\val
 	.else
 	movz	\reg, :abs_g3:\val
+	.if ((((\val) >> 32) & 0xffff) != 0)
 	movk	\reg, :abs_g2_nc:\val
 	.endif
+	.endif
+	.if ((((\val) >> 16) & 0xffff) != 0)
 	movk	\reg, :abs_g1_nc:\val
 	.endif
+	.endif
+	.if (((\val) & 0xffff) != 0)
 	movk	\reg, :abs_g0_nc:\val
+	.endif
+	.endif
+	.else
+	.if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff)
+	movn	\reg, :abs_g0:~\val
+	.else
+	.if ((((\val) >> 32) & 0xffffffff) == 0xffffffff)
+	movn	\reg, :abs_g1:~\val
+	.else
+	.if ((((\val) >> 48) & 0xffff) == 0xffff)
+	movn	\reg, :abs_g2:~\val
+	.else
+	movn	\reg, :abs_g3:~\val
+	.if ((((\val) >> 32) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g2_nc:\val
+	.endif
+	.endif
+	.if ((((\val) >> 16) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g1_nc:\val
+	.endif
+	.endif
+	.if (((\val) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g0_nc:\val
+	.endif
+	.endif
+	.endif
 	.endm
 
 /*
Signature · 7 lines
2.25.1


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
reviewer
#1 lore raw

[adding arm64 folk]

When sendign a series like this, please send a cover letter, and thread patches in reply to the cover letter.

Use git format-patch --cover to generate that, and git send-email should thread things that way by default.

Quoted 11 lines · Rémi Denis-Courmont

From: Remi Denis-Courmont <[email protected]>

This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:

1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.

2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.

Quoted 8 lines

3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.

4) Use MOVN if it saves one or more MOVK over MOVZ.

Note that the assembler uses -1 as truth value (not +1 like C).

To be honest, I don't think this is worthwhile given how complex it makes the mov_q macro.

Given the places we use mov_q today, I don't think this is going to give any measurable performance improvement, but will make debugging more painful than necessary.

Thanks,
Mark

Quoted 79 lines
---
 arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 5 deletions(-)
arch/arm64/include/asm/assembler.h +40 -5
index af03001293c6..2b98e2c5c8a2 100644
@@ -473,22 +473,57 @@ USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU
 
 	/*
 	 * mov_q - move an immediate constant into a 64-bit register using
-	 *         between 2 and 4 movz/movk instructions (depending on the
+	 *         between 1 and 4 mov/movk instructions (depending on the
 	 *         magnitude and sign of the operand)
 	 */
 	.macro	mov_q, reg, val
-	.if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
-	movz	\reg, :abs_g1_s:\val
+	.if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff))
+	.if (((\val) >> 16) == 0)
+	movz	\reg, :abs_g0:\val
 	.else
-	.if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
-	movz	\reg, :abs_g2_s:\val
+	.if (((\val) >> 32) == 0)
+	movz	\reg, :abs_g1:\val
+	.else
+	.if (((\val) >> 48) == 0)
+	movz	\reg, :abs_g2:\val
 	.else
 	movz	\reg, :abs_g3:\val
+	.if ((((\val) >> 32) & 0xffff) != 0)
 	movk	\reg, :abs_g2_nc:\val
 	.endif
+	.endif
+	.if ((((\val) >> 16) & 0xffff) != 0)
 	movk	\reg, :abs_g1_nc:\val
 	.endif
+	.endif
+	.if (((\val) & 0xffff) != 0)
 	movk	\reg, :abs_g0_nc:\val
+	.endif
+	.endif
+	.else
+	.if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff)
+	movn	\reg, :abs_g0:~\val
+	.else
+	.if ((((\val) >> 32) & 0xffffffff) == 0xffffffff)
+	movn	\reg, :abs_g1:~\val
+	.else
+	.if ((((\val) >> 48) & 0xffff) == 0xffff)
+	movn	\reg, :abs_g2:~\val
+	.else
+	movn	\reg, :abs_g3:~\val
+	.if ((((\val) >> 32) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g2_nc:\val
+	.endif
+	.endif
+	.if ((((\val) >> 16) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g1_nc:\val
+	.endif
+	.endif
+	.if (((\val) & 0xffff) != 0xffff)
+	movk	\reg, :abs_g0_nc:\val
+	.endif
+	.endif
+	.endif
 	.endm
 
 /*
Signature · 7 lines
2.25.1


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

#2 lore raw
Quoted 28 lines · Mark Rutland

[adding arm64 folk]

When sendign a series like this, please send a cover letter, and thread patches in reply to the cover letter.

Use git format-patch --cover to generate that, and git send-email should thread things that way by default.

Quoted 18 lines · Rémi Denis-Courmont

From: Remi Denis-Courmont <[email protected]>

This reduces the number of MOVZ and MOVK to expand a constant. This makes several changes:

1) Save one instruction for exactly 32 or 48 bits unsigned values, using unsigned MOV rather than signed MOV.

2) Save one instruction for unsigned 16 bits (or less) values, not treating them as 32-bits values.

3) Skip any redundant MOVK instructions the 16-bits immediate would be zero.

4) Use MOVN if it saves one or more MOVK over MOVZ.

Note that the assembler uses -1 as truth value (not +1 like C).

Quoted 9 lines
Quoted 1 line

To be honest, I don't think this is worthwhile given how complex it makes the mov_q macro.

Given the places we use mov_q today, I don't think this is going to give any measurable performance improvement, but will make debugging more painful than necessary.

Agreed. For macros this complex, you better have some test code that we can run to prove its correctness. But actually, this is something we should bring up with the toolchain people: it would be much more useful to have a pseudo-op for this that we can use everywhere, and which will resolve to the 4 instruction sequence (and the associated ELF relocations) by default, and gets optimized into something smaller by the assembler if the value is known at that point.

Quoted 75 lines
Quoted 75 lines
---
 arch/arm64/include/asm/assembler.h | 45 ++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 5 deletions(-)
arch/arm64/include/asm/assembler.h +40 -6
index af03001293c6..2b98e2c5c8a2 100644
@@ -473,22 +473,57 @@ USER(\label, ic ivau, \tmp2)                    // invalidate I line PoU
 
      /*
       * mov_q - move an immediate constant into a 64-bit register using
-      *         between 2 and 4 movz/movk instructions (depending on the
+      *         between 1 and 4 mov/movk instructions (depending on the
       *         magnitude and sign of the operand)
       */
      .macro  mov_q, reg, val
-     .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
-     movz    \reg, :abs_g1_s:\val
+     .if ((((\val) & 0xffff) == 0) + ((((\val) >> 16) & 0xffff) == 0) + ((((\val) >> 32) & 0xffff) == 0) + ((((\val) >> 48) & 0xffff) == 0) <= (((\val) & 0xffff) == 0xffff) + ((((\val) >> 16) & 0xffff) == 0xffff) + ((((\val) >> 32) & 0xffff) == 0xffff) + ((((\val) >> 48) & 0xffff) == 0xffff))
+     .if (((\val) >> 16) == 0)
+     movz    \reg, :abs_g0:\val
      .else
-     .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
-     movz    \reg, :abs_g2_s:\val
+     .if (((\val) >> 32) == 0)
+     movz    \reg, :abs_g1:\val
+     .else
+     .if (((\val) >> 48) == 0)
+     movz    \reg, :abs_g2:\val
      .else
      movz    \reg, :abs_g3:\val
+     .if ((((\val) >> 32) & 0xffff) != 0)
      movk    \reg, :abs_g2_nc:\val
      .endif
+     .endif
+     .if ((((\val) >> 16) & 0xffff) != 0)
      movk    \reg, :abs_g1_nc:\val
      .endif
+     .endif
+     .if (((\val) & 0xffff) != 0)
      movk    \reg, :abs_g0_nc:\val
+     .endif
+     .endif
+     .else
+     .if ((((\val) >> 16) & 0xffffffffffff) == 0xffffffffffff)
+     movn    \reg, :abs_g0:~\val
+     .else
+     .if ((((\val) >> 32) & 0xffffffff) == 0xffffffff)
+     movn    \reg, :abs_g1:~\val
+     .else
+     .if ((((\val) >> 48) & 0xffff) == 0xffff)
+     movn    \reg, :abs_g2:~\val
+     .else
+     movn    \reg, :abs_g3:~\val
+     .if ((((\val) >> 32) & 0xffff) != 0xffff)
+     movk    \reg, :abs_g2_nc:\val
+     .endif
+     .endif
+     .if ((((\val) >> 16) & 0xffff) != 0xffff)
+     movk    \reg, :abs_g1_nc:\val
+     .endif
+     .endif
+     .if (((\val) & 0xffff) != 0xffff)
+     movk    \reg, :abs_g0_nc:\val
+     .endif
+     .endif
+     .endif
      .endm
 
 /*
--

2.25.1

Quoted 4 lines
Quoted 4 lines

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel