ARM: Use udiv/sdiv for __aeabi_{u}idiv library functions
PATCH v2 1/1 ARM: Use udiv/sdiv for __aeabi_{u}idiv library functions
If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.
Running the perf messaging benchmark in pipe mode
$ perf bench sched messaging -p
shows a modest improvement on my v7 CPU.
before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805
after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538
(5.805 - 5.538) / 5.805 = 4.6%
---
Changes since v1:
* Replace signed with unsigned in unsigned divide function
* drop & in inline assembly
* Use IS_ENABLED() instead of #ifdef
* Pass DIV_V7 into lib1funcs.S instead of depending on ZIMAGE or CPU_V7
arch/arm/kernel/setup.c | 13 ++++++++++- arch/arm/lib/Makefile | 6 +++++ arch/arm/lib/div-v7.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ arch/arm/lib/lib1funcs.S | 16 +++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 arch/arm/lib/div-v7.c
arch/arm/kernel/setup.c +12 -1
@@ -30,6 +30,7 @@#include <linux/bug.h> #include <linux/compiler.h> #include <linux/sort.h> +#include <linux/static_key.h> #include <asm/unified.h> #include <asm/cp15.h>
@@ -365,9 +366,11 @@ void __init early_print(const char *str, ...)printk ("%s" , buf); } +struct static_key cpu_has_idiv =STATIC_KEY_INIT_FALSE ; +static void __init cpuid_init_hwcaps (void ) { -unsigned int divide_instrs, vmsa; +unsigned int divide_instrs, vmsa, idiv_mask;if (cpu_architecture () < CPU_ARCH_ARMv7)return ;
@@ -381,6 +384,14 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +if (IS_ENABLED (CONFIG_THUMB2_KERNEL )) + idiv_mask =HWCAP_IDIVT ; +else + idiv_mask =HWCAP_IDIVA ; + +if (elf_hwcap & idiv_mask) +static_key_slow_inc (&cpu_has_idiv); +/* LPAE implies atomic ldrd/strd instructions */ vmsa = (read_cpuid_ext (CPUID_EXT_MMFR0 ) &0xf ) >>0 ;if (vmsa >=5 )
arch/arm/lib/Makefile +6 -0
@@ -15,6 +15,12 @@ lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ io-readsb.o io-writesb.o io-readsl.o io-writesl.o \ call_with_stack.o +lib-$(CONFIG_CPU_V7) += div-v7.o +CFLAGS_div-v7.o := -march=armv7-a +ifeq ($(CONFIG_CPU_V7),y) + AFLAGS_lib1funcs.o := -DDIV_V7 +endif + mmu-y := clear_user.o copy_page.o getuser.o putuser.o # the code in uaccess.S is not preemption safe and
arch/arm/lib/div-v7.c new file +58 -0
58 lines not on this page · raw message
arch/arm/lib/lib1funcs.S +16 -0
@@ -205,8 +205,12 @@ Boston, MA 02111-1307, USA. */ .endm +#ifdef DIV_V7 +ENTRY(___aeabi_uidiv) +#else ENTRY(__udivsi3) ENTRY(__aeabi_uidiv) +#endif UNWIND(.fnstart) subs r2, r1, #1
@@ -232,8 +236,12 @@ UNWIND(.fnstart) mov pc, lr UNWIND(.fnend) +#ifdef DIV_V7 +ENDPROC(___aeabi_uidiv) +#else ENDPROC(__udivsi3) ENDPROC(__aeabi_uidiv) +#endif ENTRY(__umodsi3) UNWIND(.fnstart)
@@ -253,8 +261,12 @@ UNWIND(.fnstart) UNWIND(.fnend) ENDPROC(__umodsi3) +#ifdef DIV_V7 +ENTRY(___aeabi_idiv) +#else ENTRY(__divsi3) ENTRY(__aeabi_idiv) +#endif UNWIND(.fnstart) cmp r1, #0
@@ -293,8 +305,12 @@ UNWIND(.fnstart) mov pc, lr UNWIND(.fnend) +#ifdef DIV_V7 +ENDPROC(___aeabi_idiv) +#else ENDPROC(__divsi3) ENDPROC(__aeabi_idiv) +#endif ENTRY(__modsi3) UNWIND(.fnstart)
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, hosted by The Linux Foundation
Quoted 10 lines · Stephen Boyd
If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.
Running the perf messaging benchmark in pipe mode
$ perf bench sched messaging -pshows a modest improvement on my v7 CPU.
Quoted 7 lines
before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538(5.805 - 5.538) / 5.805 = 4.6%
Even with the change to the output constraint suggested by Mans, you get absolutely identical benchmark results? There's a lot of variance in any case..
BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?
Ta,
Matt Sealey <[email protected]>
Quoted 3 lines · Matt Sealey
BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?
The branches themselves probably have minimal overhead. There will however be code to preserve call-clobbered registers (and move the values to/from r0/r1) that would not be needed if the div instructions were done inline (obviously such a kernel could only run on hardware with division support).
Signature · 2 lines
M?ns Rullg?rd mans at mansr.com
Quoted 13 lines · Stephen Boyd
If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.
Running the perf messaging benchmark in pipe mode
$ perf bench sched messaging -pshows a modest improvement on my v7 CPU.
before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805
Quoted 6 lines
after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538(5.805 - 5.538) / 5.805 = 4.6%
Signed-off-by Stephen Boyd <[email protected]>
Bah..... NAK.
We are doing runtime patching of the kernel for many many things already. So why not do the same here?
The obvious strategy is to simply overwrite the start of the existing __aeabi_idiv code with the "sdiv r0, r0, r1" and "bx lr" opcodes.
Similarly for the unsigned case.
That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.
Nicolas
Hello,
arch/arm/lib/Makefile lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ · line 18
Quoted 9 lines · Stephen Boyd
arch/arm/lib/Makefile +1
-0@@ -15,6 +15,12 @@ lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ io-readsb.o io-writesb.o io-readsl.o io-writesl.o \ call_with_stack.o +lib-$(CONFIG_CPU_V7) += div-v7.o
CPU_V7M could make use of that, too.
(If you follow Nico's advice to use runtime patching I cannot test it for you on v7-M though as my machine has to use an XIP kernel.)
Best regards
Uwe
Signature · 2 lines
Pengutronix e.K. | Uwe Kleine-K?nig | Industrial Linux Solutions | http://www.pengutronix.de/ |
Quoted 14 lines · Matt Sealey
Quoted 13 lines · Stephen Boyd
If we're running on a v7 ARM CPU, detect if the CPU supports the sdiv/udiv instructions and replace the signed and unsigned division library functions with an sdiv/udiv instruction.
Running the perf messaging benchmark in pipe mode
$ perf bench sched messaging -pshows a modest improvement on my v7 CPU.
before:
(5.060 + 5.960 + 5.971 + 5.643 + 6.029 + 5.665 + 6.050 + 5.870 + 6.117 + 5.683) / 10 = 5.805
Quoted 7 lines
Quoted 4 lines
after:
(4.884 + 5.549 + 5.749 + 6.001 + 5.460 + 5.103 + 5.956 + 6.112 + 5.468 + 5.093) / 10 = 5.538(5.805 - 5.538) / 5.805 = 4.6%
Even with the change to the output constraint suggested by Mans, you get absolutely identical benchmark results? There's a lot of variance in any case..
Yeah sorry I didn't run the testcase again to see if numbers changed because I assumed one less instruction would be in the noise. I agree there is a lot of variance so if you have any better benchmarks/testcases please let me know.
Quoted 4 lines
BTW has there been any evaluation of the penalty for the extra branching, or the performance hit for the ARMv7-without-division cases?
I haven't done any. I'll factor that in for the next round.
Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Quoted 2 lines · Nicolas Pitre
Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?
static keys are a form of runtime patching, albeit not as extreme as you're suggesting.
Quoted 5 lines
The obvious strategy is to simply overwrite the start of the existing
__aeabi_idivcode with the "sdiv r0, r0, r1" and "bx lr" opcodes.Similarly for the unsigned case.
I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.
Quoted 3 lines
That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.
The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time
00000000 <__aeabi_idiv>:
0: nop {0}
4: b 0 <___aeabi_idiv>
8: sdiv r0, r0, r1
c: bx lrand after we test and find support for the instruction it will be replaced with
00000000 <__aeabi_idiv>:
0: b 8 4: b 0 <___aeabi_idiv> 8: sdiv r0, r0, r1 c: bx lr
Unfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.
Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Quoted 11 lines · Uwe Kleine-K?nig
Hello,
Quoted 8 lines · Stephen Boyd
arch/arm/lib/Makefile +0
-0@@ -15,6 +15,12 @@ lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ io-readsb.o io-writesb.o io-readsl.o io-writesl.o \ call_with_stack.o
Quoted 5 lines
arch/arm/lib/Makefile lib-y := backtrace.o changebit.o csumipv6.o csumpartial.o \ · line 18
Quoted 1 line
arch/arm/lib/Makefile +1
-0+lib-$(CONFIG_CPU_V7) += div-v7.o
CPU_V7Mcould make use of that, too.
(If you follow Nico's advice to use runtime patching I cannot test it for you on v7-M though as my machine has to use an XIP kernel.)
It already is runtime patching so I suspect you won't be able to test it anyway. I suppose we need another config like MIGHT_HAVE_IDIV or something that both v7 and v7M select?
Signature · 2 lines
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Quoted 34 lines · Stephen Boyd
Quoted 2 lines · Nicolas Pitre
Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?
static keys are a form of runtime patching, albeit not as extreme as you're suggesting.
Quoted 5 lines
The obvious strategy is to simply overwrite the start of the existing
__aeabi_idivcode with the "sdiv r0, r0, r1" and "bx lr" opcodes.Similarly for the unsigned case.
I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.
Quoted 3 lines
That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.
The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time
00000000 <
__aeabi_idiv>:0: nop {0} 4: b 0 <___aeabi_idiv> 8: sdiv r0, r0, r1 c: bx lrand after we test and find support for the instruction it will be replaced with
00000000 <
__aeabi_idiv>:
Quoted 8 lines
0: b 8 4: b 0 <___aeabi_idiv> 8: sdiv r0, r0, r1 c: bx lrUnfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.
Ideally the bl instruction at the call site would be patched over with sdiv/udiv when supported. This would leave things exactly as they are for hardware without div capability and incur only the call setup cost (but no actual call) on div-capable hardware. No, I don't know how to achieve this.
Signature · 2 lines
M?ns Rullg?rd mans at mansr.com
Quoted 34 lines · Stephen Boyd
Quoted 2 lines · Nicolas Pitre
Bah..... NAK. We are doing runtime patching of the kernel for many many things already. So why not do the same here?
static keys are a form of runtime patching, albeit not as extreme as you're suggesting.
Quoted 5 lines
The obvious strategy is to simply overwrite the start of the existing
__aeabi_idivcode with the "sdiv r0, r0, r1" and "bx lr" opcodes.Similarly for the unsigned case.
I was thinking the same thing when I wrote this, but I didn't know how to tell the compiler to either inline this function or to let me inilne an assembly stub with some section magic.
Quoted 3 lines
That let you test the hardware capability only once during boot instead of everytime a divide operation is performed.
The test for hardware capability really isn't done more than once during boot. The assembly is like so at compile time
00000000 <
__aeabi_idiv>:0: nop {0} 4: b 0 <___aeabi_idiv> 8: sdiv r0, r0, r1 c: bx lrand after we test and find support for the instruction it will be replaced with
00000000 <
__aeabi_idiv>:
Quoted 8 lines
0: b 8 4: b 0 <___aeabi_idiv> 8: sdiv r0, r0, r1 c: bx lrUnfortunately we still have to jump to this function. It would be great if we could inline this function at the call site but as I already said I don't know how to do that.
What about this patch which I think is currently your best option. Note it would need to use the facilities from asm/opcodes.h to make it endian agnostic.
arch/arm/kernel/setup.c +28 -0
@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u16 *uidiv = (u16 *)&__aeabi_uidiv; +u16 *idiv = (u16 *)&__aeabi_idiv; + + uidiv[0 ] =0xfbb0 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xf0f1 ; + uidiv[2 ] =0x4770 ;/* bx lr */ + + idiv[0 ] =0xfb90 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xf0f1 ; + idiv[2 ] =0x4770 ;/* bx lr */ + }else if (!IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVA )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u32 *uidiv = (u32 *)&__aeabi_uidiv; +u32 *idiv = (u32 *)&__aeabi_idiv; + + uidiv[0 ] =0xe730f110 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xe12fff1e ;/* bx lr */ + + idiv[0 ] =0xe710f110 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xe12fff1e ;/* bx lr */ + } +/* LPAE implies atomic ldrd/strd instructions */ vmsa = (read_cpuid_ext (CPUID_EXT_MMFR0 ) &0xf ) >>0 ;if (vmsa >=5 )
Nicolas
Quoted 29 lines · Nicolas Pitre
What about this patch which I think is currently your best option. Note it would need to use the facilities from
asm/opcodes.hto make it endian agnostic.arch/arm/kernel/setup.c +17
-0@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u16 *uidiv = (u16 *)&__aeabi_uidiv; +u16 *idiv = (u16 *)&__aeabi_idiv; + + uidiv[0 ] =0xfbb0 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xf0f1 ; + uidiv[2 ] =0x4770 ;/* bx lr */ + + idiv[0 ] =0xfb90 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xf0f1 ; + idiv[2 ] =0x4770 ;/* bx lr */ + }else if (!IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVA )) {
arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412
Quoted 10 lines
arch/arm/kernel/setup.c +10
-0+extern char __aeabi_uidiv, __aeabi_idiv; +u32 *uidiv = (u32 *)&__aeabi_uidiv; +u32 *idiv = (u32 *)&__aeabi_idiv; + + uidiv[0 ] =0xe730f110 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xe12fff1e ;/* bx lr */ + + idiv[0 ] =0xe710f110 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xe12fff1e ;/* bx lr */ + }
What about endianness, and what if XIP is enabled?
Quoted 34 lines · Russell King - ARM Linux
Quoted 33 lines · Nicolas Pitre
What about this patch which I think is currently your best option. Note it would need to use the facilities from
asm/opcodes.hto make it endian agnostic.arch/arm/kernel/setup.c +21
-0@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u16 *uidiv = (u16 *)&__aeabi_uidiv; +u16 *idiv = (u16 *)&__aeabi_idiv; + + uidiv[0 ] =0xfbb0 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xf0f1 ; + uidiv[2 ] =0x4770 ;/* bx lr */ + + idiv[0 ] =0xfb90 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xf0f1 ; + idiv[2 ] =0x4770 ;/* bx lr */ + }else if (!IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVA )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u32 *uidiv = (u32 *)&__aeabi_uidiv; +u32 *idiv = (u32 *)&__aeabi_idiv; +
Quoted 8 lines
arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412
Quoted 6 lines
arch/arm/kernel/setup.c +6
-0+ uidiv[0 ] =0xe730f110 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xe12fff1e ;/* bx lr */ + + idiv[0 ] =0xe710f110 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xe12fff1e ;/* bx lr */ + }What about endianness, and what if XIP is enabled?
Just as I said above the diff: this needs refined.
Obviously XIP can't use this and doesn't need it either as a XIP kernel should be optimized for the very platform it will run onto i.e. gcc should already emit those div opcodes inline if appropriate.
Nicolas
Quoted 34 lines · Russell King - ARM Linux
Quoted 33 lines · Nicolas Pitre
What about this patch which I think is currently your best option. Note it would need to use the facilities from
asm/opcodes.hto make it endian agnostic.arch/arm/kernel/setup.c +21
-0@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u16 *uidiv = (u16 *)&__aeabi_uidiv; +u16 *idiv = (u16 *)&__aeabi_idiv; + + uidiv[0 ] =0xfbb0 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xf0f1 ; + uidiv[2 ] =0x4770 ;/* bx lr */ + + idiv[0 ] =0xfb90 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xf0f1 ; + idiv[2 ] =0x4770 ;/* bx lr */ + }else if (!IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVA )) { +extern char __aeabi_uidiv, __aeabi_idiv; +u32 *uidiv = (u32 *)&__aeabi_uidiv; +u32 *idiv = (u32 *)&__aeabi_idiv; +
Quoted 8 lines
arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 412
Quoted 6 lines
arch/arm/kernel/setup.c +6
-0+ uidiv[0 ] =0xe730f110 ;/* udiv r0, r0, r1 */ + uidiv[1 ] =0xe12fff1e ;/* bx lr */ + + idiv[0 ] =0xe710f110 ;/* sdiv r0, r0, r1 */ + idiv[1 ] =0xe12fff1e ;/* bx lr */ + }What about endianness, and what if XIP is enabled?
I was also going to add a note about endian-ness.
Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?
Signature · 2 lines
Ben Dooks http://www.codethink.co.uk/ Senior Engineer Codethink - Providing Genius
Quoted 9 lines · Nicolas Pitre
What about this patch which I think is currently your best option. Note it would need to use the facilities from
asm/opcodes.hto make it endian agnostic.arch/arm/kernel/setup.c +0
-0
arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 391
Quoted 9 lines
arch/arm/kernel/setup.c +6
-0@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; } +/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv;
It would be safer to declare these as arrays of unspecified size. Otherwise the compiler might do evil things with what to it looks like out of bounds indexing.
There should also be some cache maintenance after this patching, or is that already happening for some other reason?
Signature · 2 lines
M?ns Rullg?rd mans at mansr.com
Quoted 3 lines · Ben Dooks
Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?
Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.
You cannot just scan the binary for the appropriate branch opcode either as you may turn up false positives in literal pools.
Nicolas
Quoted 14 lines · M?ns Rullg?rd
Quoted 12 lines · Nicolas Pitre
What about this patch which I think is currently your best option. Note it would need to use the facilities from
asm/opcodes.hto make it endian agnostic.arch/arm/kernel/setup.c +0
-0@@ -383,6 +383,34 @@ static void __init cpuid_init_hwcaps(void) elf_hwcap |=HWCAP_IDIVT ; }
Quoted 10 lines
arch/arm/kernel/setup.c static void __init cpuid_init_hwcaps(void) · line 391
Quoted 6 lines
arch/arm/kernel/setup.c +6
-0+/* +* Patch our division routines with the corresponding opcode +* if the hardware supports it. +*/ +if (IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVT )) { +extern char __aeabi_uidiv, __aeabi_idiv;It would be safer to declare these as arrays of unspecified size. Otherwise the compiler might do evil things with what to it looks like out of bounds indexing.
Right.
Quoted 3 lines
There should also be some cache maintenance after this patching, or is that already happening for some other reason?
This is so early during boot that the MMU isn't even fully initialized yet. The cache will be flushed.
Nicolas
Quoted 11 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks
Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?
Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.
It might be possible to extract this information from relocation tables.
Signature · 2 lines
M?ns Rullg?rd mans at mansr.com
Quoted 8 lines · M?ns Rullg?rd
Quoted 6 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks
Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?
Quoted 7 lines
Quoted 5 lines
Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.
It might be possible to extract this information from relocation tables.
True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.
Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.
Is it worth it? I'm not sure.
Nicolas
Quoted 10 lines · Nicolas Pitre
Quoted 1 line · M?ns Rullg?rd
It might be possible to extract this information from relocation tables.
True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.
Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.
Another solution is to patch the call site from within __aeabi_idiv at run time using lr. That wouldn't work in the presence of tail call optimization though.
Again this might not be worth it and patching __aeabi_idiv instead might be a good enough compromize.
Nicolas
Quoted 16 lines · Nicolas Pitre
Quoted 14 lines · M?ns Rullg?rd
Quoted 11 lines · Nicolas Pitre
Quoted 3 lines · Ben Dooks
Given these are single instructoins for ARM, is it possible we could make a table of all the callers and fix them up when we initialise as we do for the SMP/UP case and for page-offset?
Not really. Calls to those functions are generated by the compiler implicitly when a divisor operand is used and therefore we cannot annotate those calls. We'd have to use special accessors everywhere to replace the standard division operand (like we do for 64 by 32 bit divisions) but I doubt that people would accept that.
Quoted 8 lines
Quoted 1 line
It might be possible to extract this information from relocation tables.
True, but only for individual .o files. Once the linker puts them together the information is lost, and trying to infer what the linker has done is insane.
Filtering the compiler output to annotate idiv calls before it is assembled would probably be a better solution.
OK, here's an extremely ugly hootenanny of a patch. It seems to work on an A7 Cubieboard2. I would never suggest actually doing this, but maybe it can be useful for comparing performance against the more palatable solutions.
arch/arm/Makefile +1 -1
@@ -113,7 +113,7 @@ endif endif # Need -Uarm for gcc < 3.x -KBUILD_CFLAGS +=$(CFLAGS_ABI) $(CFLAGS_THUMB2) $(arch-y) $(tune-y) $(call cc-option,-mshort-load-bytes,$(call cc-option,-malignment-traps,)) -msoft-float -Uarm +KBUILD_CFLAGS +=$(CFLAGS_ABI) $(CFLAGS_THUMB2) $(arch-y) $(tune-y) $(call cc-option,-mshort-load-bytes,$(call cc-option,-malignment-traps,)) -msoft-float -Uarm -include asm/divhack.h KBUILD_AFLAGS +=$(CFLAGS_ABI) $(AFLAGS_THUMB2) $(arch-y) $(tune-y) -include asm/unified.h -msoft-float CHECKFLAGS += -D__arm__
arch/arm/include/asm/divhack.h new file +23 -0
@@ -0,0 +1,23 @@ +__asm__ (".macro dobl tgt \n" +" .ifc \\tgt, __aeabi_idiv \n" +" .L.sdiv.\\@: \n" +" .pushsection .sdiv_tab.init, \"a\", %progbits \n" +" .word .L.sdiv.\\@ \n" +" .popsection \n" +" .endif \n" +" .ifc \\tgt, __aeabi_uidiv \n" +" .L.udiv.\\@: \n" +" .pushsection .udiv_tab.init, \"a\", %progbits \n" +" .word .L.udiv.\\@ \n" +" .popsection \n" +" .endif \n" +" bl \\tgt \n" +".endm \n" +".macro defbl \n" +" .macro bl tgt \n" +" .purgem bl \n" +" dobl \\tgt \n" +" defbl \n" +" .endm \n" +".endm \n" +"defbl \n" );
arch/arm/kernel/setup.c +12 -0
@@ -375,6 +375,18 @@ static void __init cpuid_init_hwcaps(void)case 1 : elf_hwcap |=HWCAP_IDIVT ; } + +if (!IS_ENABLED (CONFIG_THUMB2_KERNEL ) && (elf_hwcap &HWCAP_IDIVA )) { +extern u32 __sdiv_tab_start, __sdiv_tab_end; +extern u32 __udiv_tab_start, __udiv_tab_end; +u32 *div; + +for (div = &__sdiv_tab_start; div < &__sdiv_tab_end; div++) + *(u32 *)*div =0xe710f110 ; + +for (div = &__udiv_tab_start; div < &__udiv_tab_end; div++) + *(u32 *)*div =0xe730f110 ; + } }static void __init feat_v6_fixup (void )
arch/arm/kernel/vmlinux.lds.S +2 -0
@@ -176,6 +176,8 @@ SECTIONS CON_INITCALL SECURITY_INITCALL INIT_RAM_FS + __sdiv_tab_start = .; *(.sdiv_tab.init); __sdiv_tab_end = .; + __udiv_tab_start = .; *(.udiv_tab.init); __udiv_tab_end = .; } #ifndef CONFIG_XIP_KERNEL .exit.data : {
Signature · 2 lines
M?ns Rullg?rd [email protected]