Skip to contentSkip to conversation

ARM: Use udiv/sdiv for __aeabi_{u}idiv library functions

PATCH In review Stephen Boyd 8 participants 20 messages Nov 8, 2013 – Nov 12, 2013 Arm (32-bit)

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ARM: Use udiv/sdiv for __aeabi_{u}idiv library functions

If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.

Running the perf messaging benchmark in pipe mode

 $ perf bench sched messaging -p

shows a modest improvement on my v7 CPU.

before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805

after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538

(5.805 - 5.538) / 5.805 = 4.6%

---

Changes since v1:
* Replace signed with unsigned in unsigned divide function
* drop & in inline assembly
* Use IS_ENABLED() instead of #ifdef
* Pass DIV_V7 into lib1funcs.S instead of depending on ZIMAGE or CPU_V7

 arch/arm/kernel/setup.c  | 13 ++++++++++-
 arch/arm/lib/Makefile    |  6 +++++
 arch/arm/lib/div-v7.c    | 58 ++++++++++++++++++++++++++++++++++++++++++++++++
 arch/arm/lib/lib1funcs.S | 16 +++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm/lib/div-v7.c
arch/arm/kernel/setup.c +12 -1
index 0e1e2b3..f9e577a 100644
@@ -30,6 +30,7 @@
 #include <linux/bug.h>
 #include <linux/compiler.h>
 #include <linux/sort.h>
+#include <linux/static_key.h>
 
 #include <asm/unified.h>
 #include <asm/cp15.h>
@@ -365,9 +366,11 @@ void __init early_print(const char *str, ...)
 	printk("%s", buf);
 }
 
+struct static_key cpu_has_idiv = STATIC_KEY_INIT_FALSE;
+
 static void __init cpuid_init_hwcaps(void)
 {
-	unsigned int divide_instrs, vmsa;
+	unsigned int divide_instrs, vmsa, idiv_mask;
 
 	if (cpu_architecture() < CPU_ARCH_ARMv7)
 		return;
@@ -381,6 +384,14 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
+		idiv_mask = HWCAP_IDIVT;
+	else
+		idiv_mask = HWCAP_IDIVA;
+
+	if (elf_hwcap & idiv_mask)
+		static_key_slow_inc(&cpu_has_idiv);
+
 	/* LPAE implies atomic ldrd/strd instructions */
 	vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0;
 	if (vmsa >= 5)
arch/arm/lib/Makefile +6 -0
index bd454b0..38621729 100644
@@ -15,6 +15,12 @@ lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o  \
 		   call_with_stack.o
 
+lib-$(CONFIG_CPU_V7) += div-v7.o
+CFLAGS_div-v7.o := -march=armv7-a
+ifeq ($(CONFIG_CPU_V7),y)
+  AFLAGS_lib1funcs.o := -DDIV_V7
+endif
+
 mmu-y	:= clear_user.o copy_page.o getuser.o putuser.o
 
 # the code in uaccess.S is not preemption safe and
arch/arm/lib/div-v7.c new file +58 -0

58 lines not on this page · raw message

arch/arm/lib/lib1funcs.S +16 -0
index c562f64..82bbcc7 100644
@@ -205,8 +205,12 @@ Boston, MA 02111-1307, USA.  */
 .endm
 
 
+#ifdef DIV_V7
+ENTRY(___aeabi_uidiv)
+#else
 ENTRY(__udivsi3)
 ENTRY(__aeabi_uidiv)
+#endif
 UNWIND(.fnstart)
 
 	subs	r2, r1, #1
@@ -232,8 +236,12 @@ UNWIND(.fnstart)
 	mov	pc, lr
 
 UNWIND(.fnend)
+#ifdef DIV_V7
+ENDPROC(___aeabi_uidiv)
+#else
 ENDPROC(__udivsi3)
 ENDPROC(__aeabi_uidiv)
+#endif
 
 ENTRY(__umodsi3)
 UNWIND(.fnstart)
@@ -253,8 +261,12 @@ UNWIND(.fnstart)
 UNWIND(.fnend)
 ENDPROC(__umodsi3)
 
+#ifdef DIV_V7
+ENTRY(___aeabi_idiv)
+#else
 ENTRY(__divsi3)
 ENTRY(__aeabi_idiv)
+#endif
 UNWIND(.fnstart)
 
 	cmp	r1, #0
@@ -293,8 +305,12 @@ UNWIND(.fnstart)
 	mov	pc, lr
 
 UNWIND(.fnend)
+#ifdef DIV_V7
+ENDPROC(___aeabi_idiv)
+#else
 ENDPROC(__divsi3)
 ENDPROC(__aeabi_idiv)
+#endif
 
 ENTRY(__modsi3)
 UNWIND(.fnstart)
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
hosted by The Linux Foundation
#1 lore raw
Quoted 10 lines · Stephen Boyd

If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.

Running the perf messaging benchmark in pipe mode

 $ perf bench sched messaging -p

shows a modest improvement on my v7 CPU.

Quoted 7 lines

before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805

after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538

(5.805 - 5.538) / 5.805 = 4.6%

Even with the change to the output constraint suggested by Mans, you get absolutely identical benchmark results? There's a lot of variance in any case..

BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?

Ta,
Matt Sealey <[email protected]>

#2 lore raw
Quoted 3 lines · Matt Sealey

BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?

The branches themselves probably have minimal overhead. There will however be code to preserve call-clobbered registers (and move the values to/from r0/r1) that would not be needed if the div instructions were done inline (obviously such a kernel could only run on hardware with division support).

Signature · 2 lines
M?ns Rullg?rd
mans at mansr.com
#4 lore raw
In reply to Stephen Boyd
Quoted 13 lines · Stephen Boyd

If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.

Running the perf messaging benchmark in pipe mode

 $ perf bench sched messaging -p

shows a modest improvement on my v7 CPU.

before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805

Quoted 6 lines

after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538

(5.805 - 5.538) / 5.805 = 4.6%

Bah..... NAK.

We are doing runtime patching of the kernel for many many things already. So why not do the same here?

The obvious strategy is to simply overwrite the start of the existing __aeabi_idiv code with the "sdiv r0, r0, r1" and "bx lr" opcodes.

Similarly for the unsigned case.

That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.

Nicolas

#18 lore raw
In reply to Stephen Boyd

Hello,

arch/arm/lib/Makefile lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ · line 18

Quoted 9 lines · Stephen Boyd
arch/arm/lib/Makefile +1 -0
index bd454b0..38621729 100644
@@ -15,6 +15,12 @@ lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o  \
 		   call_with_stack.o
 
+lib-$(CONFIG_CPU_V7) += div-v7.o

CPU_V7M could make use of that, too.
(If you follow Nico's advice to use runtime patching I cannot test it for you on v7-M though as my machine has to use an XIP kernel.)

Best regards
Uwe

Signature · 2 lines
Pengutronix e.K.                           | Uwe Kleine-K?nig            |
Industrial Linux Solutions                 | http://www.pengutronix.de/  |
#3 lore raw
In reply to Matt Sealey
Quoted 14 lines · Matt Sealey
Quoted 13 lines · Stephen Boyd

If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.

Running the perf messaging benchmark in pipe mode

 $ perf bench sched messaging -p

shows a modest improvement on my v7 CPU.

before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805

Quoted 7 lines
Quoted 4 lines

after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538

(5.805 - 5.538) / 5.805 = 4.6%

Even with the change to the output constraint suggested by Mans, you get absolutely identical benchmark results? There's a lot of variance in any case..

Yeah sorry I didn't run the testcase again to see if numbers changed because I assumed one less instruction would be in the noise. I agree there is a lot of variance so if you have any better benchmarks/testcases please let me know.

Quoted 4 lines

BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?

I haven't done any. I'll factor that in for the next round.

Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by The Linux Foundation
#5 lore raw
In reply to Nicolas Pitre
Quoted 2 lines · Nicolas Pitre

Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?

static keys are a form of runtime patching, albeit not as extreme as you're suggesting.

Quoted 5 lines

The obvious strategy is to simply overwrite the start of the existing __aeabi_idiv code with the "sdiv r0, r0, r1" and "bx lr" opcodes.

Similarly for the unsigned case.

I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.

Quoted 3 lines

That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.

The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time

00000000 <__aeabi_idiv>:

   0:   nop     {0}
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

and after we test and find support for the instruction it will be replaced with

00000000 <__aeabi_idiv>:

   0:   b       8
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

Unfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.

Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by The Linux Foundation
#19 lore raw
In reply to Uwe Kleine-König
Quoted 11 lines · Uwe Kleine-K?nig

Hello,

Quoted 8 lines · Stephen Boyd
arch/arm/lib/Makefile +0 -0
index bd454b0..38621729 100644
@@ -15,6 +15,12 @@ lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   io-readsb.o io-writesb.o io-readsl.o io-writesl.o  \
 		   call_with_stack.o
 
Quoted 5 lines

arch/arm/lib/Makefile lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ · line 18

Quoted 1 line
arch/arm/lib/Makefile +1 -0
+lib-$(CONFIG_CPU_V7) += div-v7.o

CPU_V7M could make use of that, too.
(If you follow Nico's advice to use runtime patching I cannot test it for you on v7-M though as my machine has to use an XIP kernel.)

It already is runtime patching so I suspect you won't be able to test it anyway. I suppose we need another config like MIGHT_HAVE_IDIV or something that both v7 and v7M select?

Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by The Linux Foundation
#6 lore raw
In reply to Stephen Boyd
Quoted 34 lines · Stephen Boyd
Quoted 2 lines · Nicolas Pitre

Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?

static keys are a form of runtime patching, albeit not as extreme as you're suggesting.

Quoted 5 lines

The obvious strategy is to simply overwrite the start of the existing __aeabi_idiv code with the "sdiv r0, r0, r1" and "bx lr" opcodes.

Similarly for the unsigned case.

I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.

Quoted 3 lines

That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.

The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time

00000000 <__aeabi_idiv>:

   0:   nop     {0}
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

and after we test and find support for the instruction it will be replaced with

00000000 <__aeabi_idiv>:

Quoted 8 lines
   0:   b       8
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

Unfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.

Ideally the bl instruction at the call site would be patched over with sdiv/udiv when supported. This would leave things exactly as they are for hardware without div capability and incur only the call setup cost (but no actual call) on div-capable hardware. No, I don't know how to achieve this.

Signature · 2 lines
M?ns Rullg?rd
mans at mansr.com
#7 lore raw
In reply to Stephen Boyd
Quoted 34 lines · Stephen Boyd
Quoted 2 lines · Nicolas Pitre

Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?

static keys are a form of runtime patching, albeit not as extreme as you're suggesting.

Quoted 5 lines

The obvious strategy is to simply overwrite the start of the existing __aeabi_idiv code with the "sdiv r0, r0, r1" and "bx lr" opcodes.

Similarly for the unsigned case.

I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.

Quoted 3 lines

That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.

The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time

00000000 <__aeabi_idiv>:

   0:   nop     {0}
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

and after we test and find support for the instruction it will be replaced with

00000000 <__aeabi_idiv>:

Quoted 8 lines
   0:   b       8
   4:   b       0 <___aeabi_idiv>
   8:   sdiv    r0, r0, r1
   c:   bx      lr

Unfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +28 -0
index 6a1b8a81b1..379cffe4ab 100644
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u16 *uidiv = (u16 *)&__aeabi_uidiv;
+		u16 *idiv = (u16 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xfbb0;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xf0f1;
+		uidiv[2] = 0x4770;  /* bx lr */
+
+		idiv[0] = 0xfb90;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xf0f1;  
+		idiv[2] = 0x4770;  /* bx lr */
+	} else if (!IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVA)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u32 *uidiv = (u32 *)&__aeabi_uidiv;
+		u32 *idiv = (u32 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xe730f110;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xe12fff1e;  /* bx lr */
+
+		idiv[0] = 0xe710f110;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xe12fff1e;  /* bx lr */
+	}
+
 	/* LPAE implies atomic ldrd/strd instructions */
 	vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0;
 	if (vmsa >= 5)

Nicolas

#8 lore raw
Quoted 29 lines · Nicolas Pitre

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +17 -0
index 6a1b8a81b1..379cffe4ab 100644
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u16 *uidiv = (u16 *)&__aeabi_uidiv;
+		u16 *idiv = (u16 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xfbb0;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xf0f1;
+		uidiv[2] = 0x4770;  /* bx lr */
+
+		idiv[0] = 0xfb90;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xf0f1;  
+		idiv[2] = 0x4770;  /* bx lr */
+	} else if (!IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVA)) {

arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412

Quoted 10 lines
arch/arm/kernel/setup.c +10 -0
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u32 *uidiv = (u32 *)&__aeabi_uidiv;
+		u32 *idiv = (u32 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xe730f110;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xe12fff1e;  /* bx lr */
+
+		idiv[0] = 0xe710f110;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xe12fff1e;  /* bx lr */
+	}

What about endianness, and what if XIP is enabled?

#9 lore raw
Quoted 34 lines · Russell King - ARM Linux
Quoted 33 lines · Nicolas Pitre

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +21 -0
index 6a1b8a81b1..379cffe4ab 100644
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u16 *uidiv = (u16 *)&__aeabi_uidiv;
+		u16 *idiv = (u16 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xfbb0;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xf0f1;
+		uidiv[2] = 0x4770;  /* bx lr */
+
+		idiv[0] = 0xfb90;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xf0f1;  
+		idiv[2] = 0x4770;  /* bx lr */
+	} else if (!IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVA)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u32 *uidiv = (u32 *)&__aeabi_uidiv;
+		u32 *idiv = (u32 *)&__aeabi_idiv;
+
Quoted 8 lines

arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412

Quoted 6 lines
arch/arm/kernel/setup.c +6 -0
+		uidiv[0] = 0xe730f110;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xe12fff1e;  /* bx lr */
+
+		idiv[0] = 0xe710f110;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xe12fff1e;  /* bx lr */
+	}

What about endianness, and what if XIP is enabled?

Just as I said above the diff: this needs refined.

Obviously XIP can't use this and doesn't need it either as a XIP kernel should be optimized for the very platform it will run onto i.e. gcc should already emit those div opcodes inline if appropriate.

Nicolas

#10 lore raw
In reply to Russell King - ARM Linux
Quoted 34 lines · Russell King - ARM Linux
Quoted 33 lines · Nicolas Pitre

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +21 -0
index 6a1b8a81b1..379cffe4ab 100644
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
  		elf_hwcap |= HWCAP_IDIVT;
  	}
 
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u16 *uidiv = (u16 *)&__aeabi_uidiv;
+		u16 *idiv = (u16 *)&__aeabi_idiv;
+
+		uidiv[0] = 0xfbb0;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xf0f1;
+		uidiv[2] = 0x4770;  /* bx lr */
+
+		idiv[0] = 0xfb90;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xf0f1;
+		idiv[2] = 0x4770;  /* bx lr */
+	} else if (!IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVA)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;
+		u32 *uidiv = (u32 *)&__aeabi_uidiv;
+		u32 *idiv = (u32 *)&__aeabi_idiv;
+
Quoted 8 lines

arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412

Quoted 6 lines
arch/arm/kernel/setup.c +6 -0
+		uidiv[0] = 0xe730f110;  /* udiv r0, r0, r1 */
+		uidiv[1] = 0xe12fff1e;  /* bx lr */
+
+		idiv[0] = 0xe710f110;  /* sdiv r0, r0, r1 */
+		idiv[1] = 0xe12fff1e;  /* bx lr */
+	}

What about endianness, and what if XIP is enabled?

I was also going to add a note about endian-ness.

Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?

Signature · 2 lines
Ben Dooks				http://www.codethink.co.uk/
Senior Engineer				Codethink - Providing Genius
#16 lore raw
In reply to Nicolas Pitre
Quoted 9 lines · Nicolas Pitre

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +0 -0
index 6a1b8a81b1..379cffe4ab 100644

arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 391

Quoted 9 lines
arch/arm/kernel/setup.c +6 -0
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;

It would be safer to declare these as arrays of unspecified size. Otherwise the compiler might do evil things with what to it looks like out of bounds indexing.

There should also be some cache maintenance after this patching, or is that already happening for some other reason?

Signature · 2 lines
M?ns Rullg?rd
mans at mansr.com
#11 lore raw
In reply to Ben Dooks
Quoted 3 lines · Ben Dooks

Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?

Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.

You cannot just scan the binary for the appropriate branch opcode either as you may turn up false positives in literal pools.

Nicolas

#17 lore raw
In reply to Måns Rullgård
Quoted 14 lines · M?ns Rullg?rd
Quoted 12 lines · Nicolas Pitre

What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.

arch/arm/kernel/setup.c +0 -0
index 6a1b8a81b1..379cffe4ab 100644
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void)
 		elf_hwcap |= HWCAP_IDIVT;
 	}
 
Quoted 10 lines

arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 391

Quoted 6 lines
arch/arm/kernel/setup.c +6 -0
+	/*
+	 * Patch our division routines with the corresponding opcode
+	 * if the hardware supports it.
+	 */
+	if (IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVT)) {
+		extern char __aeabi_uidiv, __aeabi_idiv;

It would be safer to declare these as arrays of unspecified size. Otherwise the compiler might do evil things with what to it looks like out of bounds indexing.

Right.

Quoted 3 lines

There should also be some cache maintenance after this patching, or is that already happening for some other reason?

This is so early during boot that the MMU isn't even fully initialized yet. The cache will be flushed.

Nicolas

#12 lore raw
In reply to Nicolas Pitre
Quoted 11 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks

Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?

Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.

It might be possible to extract this information from relocation tables.

Signature · 2 lines
M?ns Rullg?rd
mans at mansr.com
#13 lore raw
Quoted 8 lines · M?ns Rullg?rd
Quoted 6 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks

Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?

Quoted 7 lines
Quoted 5 lines

Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.

It might be possible to extract this information from relocation tables.

True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.

Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.

Is it worth it? I'm not sure.

Nicolas

#14 lore raw
Quoted 10 lines · Nicolas Pitre
Quoted 1 line · M?ns Rullg?rd

It might be possible to extract this information from relocation tables.

True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.

Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.

Another solution is to patch the call site from within __aeabi_idiv at run time using lr. That wouldn't work in the presence of tail call optimization though.

Again this might not be worth it and patching __aeabi_idiv instead might be a good enough compromize.

Nicolas

#15 lore raw
In reply to Nicolas Pitre
Quoted 16 lines · Nicolas Pitre
Quoted 14 lines · M?ns Rullg?rd
Quoted 11 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks

Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?

Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.

Quoted 8 lines
Quoted 1 line

It might be possible to extract this information from relocation tables.

True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.

Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.

OK, here's an extremely ugly hootenanny of a patch. It seems to work on an A7 Cubieboard2. I would never suggest actually doing this, but maybe it can be useful for comparing performance against the more palatable solutions.

arch/arm/Makefile +1 -1
index 7397db6..cf1cd30 100644
@@ -113,7 +113,7 @@ endif
 endif
 
 # Need -Uarm for gcc < 3.x
-KBUILD_CFLAGS	+=$(CFLAGS_ABI) $(CFLAGS_THUMB2) $(arch-y) $(tune-y) $(call cc-option,-mshort-load-bytes,$(call cc-option,-malignment-traps,)) -msoft-float -Uarm
+KBUILD_CFLAGS	+=$(CFLAGS_ABI) $(CFLAGS_THUMB2) $(arch-y) $(tune-y) $(call cc-option,-mshort-load-bytes,$(call cc-option,-malignment-traps,)) -msoft-float -Uarm -include asm/divhack.h
 KBUILD_AFLAGS	+=$(CFLAGS_ABI) $(AFLAGS_THUMB2) $(arch-y) $(tune-y) -include asm/unified.h -msoft-float
 
 CHECKFLAGS	+= -D__arm__
arch/arm/include/asm/divhack.h new file +23 -0
new file mode 100644
index 0000000..c750b78
@@ -0,0 +1,23 @@
+__asm__ (".macro dobl tgt                                       \n"
+         "    .ifc \\tgt, __aeabi_idiv                          \n"
+         "        .L.sdiv.\\@:                                  \n"
+         "        .pushsection .sdiv_tab.init, \"a\", %progbits \n"
+         "        .word    .L.sdiv.\\@                          \n"
+         "        .popsection                                   \n"
+         "    .endif                                            \n"
+         "    .ifc \\tgt, __aeabi_uidiv                         \n"
+         "        .L.udiv.\\@:                                  \n"
+         "        .pushsection .udiv_tab.init, \"a\", %progbits \n"
+         "        .word    .L.udiv.\\@                          \n"
+         "        .popsection                                   \n"
+         "    .endif                                            \n"
+         "    bl \\tgt                                          \n"
+         ".endm                                                 \n"
+         ".macro defbl                                          \n"
+         "    .macro bl tgt                                     \n"
+         "        .purgem bl                                    \n"
+         "        dobl \\tgt                                    \n"
+         "        defbl                                         \n"
+         "    .endm                                             \n"
+         ".endm                                                 \n"
+         "defbl                                                 \n");
arch/arm/kernel/setup.c +12 -0
index 067815c1..b3a3fe1 100644
@@ -375,6 +375,18 @@ static void __init cpuid_init_hwcaps(void)
 	case 1:
 		elf_hwcap |= HWCAP_IDIVT;
 	}
+
+	if (!IS_ENABLED(CONFIG_THUMB2_KERNEL) && (elf_hwcap & HWCAP_IDIVA)) {
+		extern u32 __sdiv_tab_start, __sdiv_tab_end;
+		extern u32 __udiv_tab_start, __udiv_tab_end;
+		u32 *div;
+
+		for (div = &__sdiv_tab_start; div < &__sdiv_tab_end; div++)
+			*(u32 *)*div = 0xe710f110;
+
+		for (div = &__udiv_tab_start; div < &__udiv_tab_end; div++)
+			*(u32 *)*div = 0xe730f110;
+	}
 }
 
 static void __init feat_v6_fixup(void)
arch/arm/kernel/vmlinux.lds.S +2 -0
index 43a31fb..3d5c103 100644
@@ -176,6 +176,8 @@ SECTIONS
 		CON_INITCALL
 		SECURITY_INITCALL
 		INIT_RAM_FS
+		__sdiv_tab_start = .; *(.sdiv_tab.init); __sdiv_tab_end = .;
+		__udiv_tab_start = .; *(.udiv_tab.init); __udiv_tab_end = .;
 	}
 #ifndef CONFIG_XIP_KERNEL
 	.exit.data : {
Signature · 2 lines
M?ns Rullg?rd
[email protected]