powerpc/pseries: Use jump labels for hcall tracepoints
Subsystems (2)
PATCH 1/2 powerpc/pseries: Use jump labels for hcall tracepoints
hcall tracepoints add quite a few instructions to our hcall path:
plpar_hcall:
mr r2,r2 mfcr r0 stw r0,8(r1) b 164 <---- start ld r12,0(r2) std r12,32(r1) cmpdi r12,0 beq 164 <---- end
...
We have an unconditional branch that gets noped out during boot and a load/compare/branch. We also store the tracepoint value to the stack for the hcall_exit path to use.
By using jump labels we can simplify this to just a single nop that gets replaced with a branch when the tracepoint is enabled:
plpar_hcall:
mr r2,r2 mfcr r0 stw r0,8(r1) nop <----
...
If jump labels are not enabled, we fall back to the old method.
Signed-off-by Anton Blanchard <[email protected]>
---
arch/powerpc/include/asm/jump_label.h +9 -0
@@ -10,6 +10,7 @@* 2 of the License, or (at your option) any later version. */ +#ifndef __ASSEMBLY__#include <linux/types.h> #include <asm/feature-fixups.h>
@@ -42,4 +43,12 @@ struct jump_entry {jump_label_t key; }; +#else +#define ARCH_STATIC_BRANCH (LABEL ,KEY ) \ +1098: nop; \ + .pushsection __jump_table, "aw"; \ + FTR_ENTRY_LONG 1098b, LABEL, KEY; \ + .popsection +#endif +#endif /* _ASM_POWERPC_JUMP_LABEL_H */
arch/powerpc/platforms/pseries/hvCall.S +110 -34
238 lines not on this page · raw message
arch/powerpc/platforms/pseries/lpar.c +23 -7
@@ -26,6 +26,7 @@#include <linux/dma-mapping.h> #include <linux/console.h> #include <linux/export.h> +#include <linux/static_key.h> #include <asm/processor.h> #include <asm/mmu.h> #include <asm/page.h>
@@ -649,6 +650,19 @@ EXPORT_SYMBOL(arch_free_page);#endif #ifdef CONFIG_TRACEPOINTS +#ifdef CONFIG_JUMP_LABEL +struct static_key hcall_tracepoint_key =STATIC_KEY_INIT ; + +void hcall_tracepoint_regfunc (void ) +{ +static_key_slow_inc (&hcall_tracepoint_key); +} + +void hcall_tracepoint_unregfunc (void ) +{ +static_key_slow_dec (&hcall_tracepoint_key); +} +#else /* * We optimise our hcall path by placing hcall_tracepoint_refcount * directly in the TOC so we can check if the hcall tracepoints are
@@ -658,13 +672,6 @@ EXPORT_SYMBOL(arch_free_page);/* NB: reg/unreg are called while guarded with the tracepoints_mutex */ extern long hcall_tracepoint_refcount; -/* -* Since the tracing code might execute hcalls we need to guard against -* recursion. One example of this are spinlocks calling H_YIELD on -* shared processor partitions. -*/ -static DEFINE_PER_CPU(unsigned int, hcall_trace_depth); -void hcall_tracepoint_regfunc (void ) { hcall_tracepoint_refcount++;
@@ -674,6 +681,15 @@ void hcall_tracepoint_unregfunc(void) { hcall_tracepoint_refcount--; } +#endif + +/* +* Since the tracing code might execute hcalls we need to guard against +* recursion. One example of this are spinlocks calling H_YIELD on +* shared processor partitions. +*/ +static DEFINE_PER_CPU(unsigned int, hcall_trace_depth); +void __trace_hcall_entry (unsigned long opcode,unsigned long *args) {
PATCH 2/2 powerpc/pseries: optimise hcall tracepoints
Now that we execute the hcall tracepoint entry and exit code out of line, we can use the same stack across both functions.
Signed-off-by Anton Blanchard <[email protected]>
---
arch/powerpc/platforms/pseries/hvCall.S +16 -22
@@ -46,33 +46,27 @@ hcall_tracepoint_refcount: addi r4,r1,STK_PARAM(FIRST_REG); \ stdu r1,-STACK_FRAME_OVERHEAD(r1); \ bl __trace_hcall_entry; \ - addi r1,r1,STACK_FRAME_OVERHEAD; \ - ld r0,16(r1); \ - ld r3,STK_PARAM(R3)(r1); \ - ld r4,STK_PARAM(R4)(r1); \ - ld r5,STK_PARAM(R5)(r1); \ - ld r6,STK_PARAM(R6)(r1); \ - ld r7,STK_PARAM(R7)(r1); \ - ld r8,STK_PARAM(R8)(r1); \ - ld r9,STK_PARAM(R9)(r1); \ - ld r10,STK_PARAM(R10)(r1); \ - mtlr r0 + ld r3,STACK_FRAME_OVERHEAD+STK_PARAM(R3)(r1); \ + ld r4,STACK_FRAME_OVERHEAD+STK_PARAM(R4)(r1); \ + ld r5,STACK_FRAME_OVERHEAD+STK_PARAM(R5)(r1); \ + ld r6,STACK_FRAME_OVERHEAD+STK_PARAM(R6)(r1); \ + ld r7,STACK_FRAME_OVERHEAD+STK_PARAM(R7)(r1); \ + ld r8,STACK_FRAME_OVERHEAD+STK_PARAM(R8)(r1); \ + ld r9,STACK_FRAME_OVERHEAD+STK_PARAM(R9)(r1); \ + ld r10,STACK_FRAME_OVERHEAD+STK_PARAM(R10)(r1) /* * postcall is performed immediately before function return which * allows liberal use of volatile registers. */ #define __HCALL_INST_POSTCALL \ - mflr r0; \ - ld r6,STK_PARAM(R3)(r1); \ - std r3,STK_PARAM(R3)(r1); \ + ld r0,STACK_FRAME_OVERHEAD+STK_PARAM(R3)(r1); \ + std r3,STACK_FRAME_OVERHEAD+STK_PARAM(R3)(r1); \ mr r4,r3; \ - mr r3,r6; \ - std r0,16(r1); \ - stdu r1,-STACK_FRAME_OVERHEAD(r1); \ + mr r3,r0; \ bl __trace_hcall_exit; \ + ld r0,STACK_FRAME_OVERHEAD+16(r1); \ addi r1,r1,STACK_FRAME_OVERHEAD; \ - ld r0,16(r1); \ ld r3,STK_PARAM(R3)(r1); \ mtlr r0
@@ -279,14 +273,14 @@ plpar_hcall9_trace: mr r7,r8 mr r8,r9 mr r9,r10 - ld r10,STK_PARAM(R11)(r1) - ld r11,STK_PARAM(R12)(r1) - ld r12,STK_PARAM(R13)(r1) + ld r10,STACK_FRAME_OVERHEAD+STK_PARAM(R11)(r1) + ld r11,STACK_FRAME_OVERHEAD+STK_PARAM(R12)(r1) + ld r12,STACK_FRAME_OVERHEAD+STK_PARAM(R13)(r1) HVSC mr r0,r12 - ld r12,STK_PARAM(R4)(r1) + ld r12,STACK_FRAME_OVERHEAD+STK_PARAM(R4)(r1) std r4,0(r12) std r5,8(r12) std r6,16(r12)