Skip to contentSkip to conversation

bpf, arm64: inline bpf_get_current_task/_btf() helpers

PATCH Applied Puranjay Mohan 4 participants 4 messages Jun 19, 2024 – Jun 21, 2024

Subsystems (3)
lore
Share & tools
lore

Puranjay Mohan

author maintainer
#0 lore raw

PATCH 1/1 bpf, arm64: inline bpf_get_current_task/_btf() helpers

On ARM64, the pointer to task_struct is always available in the sp_el0 register and therefore the calls to bpf_get_current_task() and bpf_get_current_task_btf() can be inlined into a single MRS instruction.

Here is the difference before and after this change:

Before:

; struct task_struct *task = bpf_get_current_task_btf();

  54:   mov     x10, #0xffffffffffff7978        // #-34440
  58:   movk    x10, #0x802b, lsl #16
  5c:   movk    x10, #0x8000, lsl #32
  60:   blr     x10          -------------->    0xffff8000802b7978 <+0>:     mrs     x0, sp_el0
  64:   add     x7, x0, #0x0 <--------------    0xffff8000802b797c <+4>:     ret

After:

; struct task_struct *task = bpf_get_current_task_btf();
54: mrs x7, sp_el0

This shows around 1% performance improvement in artificial microbenchmark.

---
 arch/arm64/net/bpf_jit_comp.c | 9 +++++++++
 1 file changed, 9 insertions(+)
arch/arm64/net/bpf_jit_comp.c +9 -0
index 720336d28856..b838dab3bd26 100644
@@ -1244,6 +1244,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		}
 
+		/* Implement helper call to bpf_get_current_task/_btf() inline */
+		if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
+					   insn->imm == BPF_FUNC_get_current_task_btf)) {
+			emit(A64_MRS_SP_EL0(r0), ctx);
+			break;
+		}
+
 		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
 					    &func_addr, &func_addr_fixed);
 		if (ret < 0)
@@ -2581,6 +2588,8 @@ bool bpf_jit_inlines_helper_call(s32 imm)
 {
 	switch (imm) {
 	case BPF_FUNC_get_smp_processor_id:
+	case BPF_FUNC_get_current_task:
+	case BPF_FUNC_get_current_task_btf:
 		return true;
 	default:
 		return false;
Signature · 1 line
2.40.1
reviewer Acked-by on v1 1/1
#1 lore raw
Quoted 15 lines · Puranjay Mohan

On ARM64, the pointer to task_struct is always available in the sp_el0 register and therefore the calls to bpf_get_current_task() and bpf_get_current_task_btf() can be inlined into a single MRS instruction.

Here is the difference before and after this change:

Before:

; struct task_struct *task = bpf_get_current_task_btf();
   54:   mov     x10, #0xffffffffffff7978        // #-34440
   58:   movk    x10, #0x802b, lsl #16
   5c:   movk    x10, #0x8000, lsl #32
   60:   blr     x10          -------------->    0xffff8000802b7978 <+0>:     mrs     x0, sp_el0
   64:   add     x7, x0, #0x0 <--------------    0xffff8000802b797c <+4>:     ret
Quoted 7 lines

After:

; struct task_struct *task = bpf_get_current_task_btf();
   54:   mrs     x7, sp_el0

This shows around 1% performance improvement in artificial microbenchmark.

I think it would be better if more detailed data could be provided.

Quoted 24 lines
---
  arch/arm64/net/bpf_jit_comp.c | 9 +++++++++
  1 file changed, 9 insertions(+)
arch/arm64/net/bpf_jit_comp.c +7 -0
index 720336d28856..b838dab3bd26 100644
@@ -1244,6 +1244,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
  			break;
  		}
 
+		/* Implement helper call to bpf_get_current_task/_btf() inline */
+		if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
+					   insn->imm == BPF_FUNC_get_current_task_btf)) {
+			emit(A64_MRS_SP_EL0(r0), ctx);
+			break;
+		}
+
  		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
  					    &func_addr, &func_addr_fixed);
  		if (ret < 0)

arch/arm64/net/bpf_jit_comp.c bool bpf_jit_inlines_helper_call(s32 imm) · line 2595

Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +2 -0
@@ -2581,6 +2588,8 @@ bool bpf_jit_inlines_helper_call(s32 imm)
  {
  	switch (imm) {
  	case BPF_FUNC_get_smp_processor_id:
+	case BPF_FUNC_get_current_task:
+	case BPF_FUNC_get_current_task_btf:
  		return true;
  	default:
  		return false;
Acked-by on v1 1/1
#2 lore raw
In reply to Puranjay Mohan
Quoted 21 lines · Puranjay Mohan

On ARM64, the pointer to task_struct is always available in the sp_el0 register and therefore the calls to bpf_get_current_task() and bpf_get_current_task_btf() can be inlined into a single MRS instruction.

Here is the difference before and after this change:

Before:

; struct task_struct *task = bpf_get_current_task_btf();

  54:   mov     x10, #0xffffffffffff7978        // #-34440
  58:   movk    x10, #0x802b, lsl #16
  5c:   movk    x10, #0x8000, lsl #32
  60:   blr     x10          -------------->    0xffff8000802b7978 <+0>:     mrs     x0, sp_el0
  64:   add     x7, x0, #0x0 <--------------    0xffff8000802b797c <+4>:     ret

After:

; struct task_struct *task = bpf_get_current_task_btf();
54: mrs x7, sp_el0

Quoted 7 lines

This shows around 1% performance improvement in artificial microbenchmark.

---
 arch/arm64/net/bpf_jit_comp.c | 9 +++++++++
 1 file changed, 9 insertions(+)

These are frequently used helpers, similarly to get_smp_processor_id(), so I think it makes sense to optimize them.

Quoted 30 lines
arch/arm64/net/bpf_jit_comp.c +9 -1
index 720336d28856..b838dab3bd26 100644
@@ -1244,6 +1244,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
                        break;
                }
 
+               /* Implement helper call to bpf_get_current_task/_btf() inline */
+               if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
+                                          insn->imm == BPF_FUNC_get_current_task_btf)) {
+                       emit(A64_MRS_SP_EL0(r0), ctx);
+                       break;
+               }
+
                ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
                                            &func_addr, &func_addr_fixed);
                if (ret < 0)
@@ -2581,6 +2588,8 @@ bool bpf_jit_inlines_helper_call(s32 imm)
 {
        switch (imm) {
        case BPF_FUNC_get_smp_processor_id:
+       case BPF_FUNC_get_current_task:
+       case BPF_FUNC_get_current_task_btf:
                return true;
        default:
                return false;
--

2.40.1

#3 lore raw
In reply to Puranjay Mohan
This patch was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Hello:

This patch was applied to bpf/bpf-next.git (master) by Andrii Nakryiko <[email protected]>:

Quoted 9 lines · On Wed, 19 Jun 2024 13:13:34 +0000 you wrote:

On ARM64, the pointer to task_struct is always available in the sp_el0 register and therefore the calls to bpf_get_current_task() and bpf_get_current_task_btf() can be inlined into a single MRS instruction.

Here is the difference before and after this change:

Before:

[...]

Here is the summary with links:
- bpf, arm64: inline bpf_get_current_task/_btf() helpers

    https://git.kernel.org/bpf/bpf-next/c/2bb138cb20a6

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html