Skip to contentSkip to conversation

bpf, arm64: Fix stack-passed arguments for indirect trampolines

PATCH In review Puranjay Mohan 4 participants 5 messages Jul 17 – Jul 21

Subsystems (3)
lore
Share & tools
lore

Puranjay Mohan

author maintainer
#0 lore raw

PATCH 1/1 bpf, arm64: Fix stack-passed arguments for indirect trampolines

save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.

This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.

Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
---
 arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +12 -4
index c6756b964c843..4fe235f7ce894 100644
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
 static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		      const struct btf_func_model *m,
 		      const struct arg_aux *a,
-		      bool for_call_origin)
+		      bool for_call_origin, bool is_struct_ops)
 {
 	int i;
 	int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		bargs_off += 8;
 	}
 
-	soff = 32; /* on stack arguments start from FP + 32 */
+	/*
+	 * On-stack arguments start above the frame(s) pushed by the
+	 * trampoline prologue. A traced function is entered through the
+	 * fentry call, so both the parent (FP/x9) and the traced function
+	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
+	 * struct_ops callback is called indirectly, so only the FP/LR frame
+	 * is saved and the arguments start at FP + 16.
+	 */
+	soff = is_struct_ops ? 16 : 32;
 	doff = (for_call_origin ? oargs_off : bargs_off);
 
 	/* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 	store_func_meta(ctx, func_meta, func_meta_off);
 
 	/* save args for bpf */
-	save_args(ctx, bargs_off, oargs_off, m, a, false);
+	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
 	/* save callee saved registers */
 	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/* save args for original func */
-		save_args(ctx, bargs_off, oargs_off, m, a, true);
+		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
 		/* call original func */
 		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
 		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d

Signature · 1 line
2.53.0-Meta
#1 lore raw

+cc Xu

Quoted 75 lines · Puranjay Mohan

save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.

This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.

Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
---
 arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +12 -4
index c6756b964c843..4fe235f7ce894 100644
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
 static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
                      const struct btf_func_model *m,
                      const struct arg_aux *a,
-                     bool for_call_origin)
+                     bool for_call_origin, bool is_struct_ops)
 {
        int i;
        int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
                bargs_off += 8;
        }
 
-       soff = 32; /* on stack arguments start from FP + 32 */
+       /*
+        * On-stack arguments start above the frame(s) pushed by the
+        * trampoline prologue. A traced function is entered through the
+        * fentry call, so both the parent (FP/x9) and the traced function
+        * (FP/LR) frames are saved and the arguments start at FP + 32. A
+        * struct_ops callback is called indirectly, so only the FP/LR frame
+        * is saved and the arguments start at FP + 16.
+        */
+       soff = is_struct_ops ? 16 : 32;
        doff = (for_call_origin ? oargs_off : bargs_off);
 
        /* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
        store_func_meta(ctx, func_meta, func_meta_off);
 
        /* save args for bpf */
-       save_args(ctx, bargs_off, oargs_off, m, a, false);
+       save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
        /* save callee saved registers */
        emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                /* save args for original func */
-               save_args(ctx, bargs_off, oargs_off, m, a, true);
+               save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
                /* call original func */
                emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
                emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d --
2.53.0-Meta

reviewer
#2 lore raw
In reply to Puranjay Mohan
Quoted 11 lines · Puranjay Mohan

save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.

Quoted 4 lines

This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.

Which x86 fix? Add a reference to it?

Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
---
  arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
  1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +11 -3
index c6756b964c843..4fe235f7ce894 100644
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
  static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
  		      const struct btf_func_model *m,
  		      const struct arg_aux *a,
-		      bool for_call_origin)
+		      bool for_call_origin, bool is_struct_ops)
  {
  	int i;
  	int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
  		bargs_off += 8;
  	}
 
-	soff = 32; /* on stack arguments start from FP + 32 */
+	/*
+	 * On-stack arguments start above the frame(s) pushed by the
+	 * trampoline prologue. A traced function is entered through the
+	 * fentry call, so both the parent (FP/x9) and the traced function
+	 * (FP/LR) frames are saved and the arguments start at FP + 32. A
+	 * struct_ops callback is called indirectly, so only the FP/LR frame
+	 * is saved and the arguments start at FP + 16.
+	 */
+	soff = is_struct_ops ? 16 : 32;
  	doff = (for_call_origin ? oargs_off : bargs_off);
 
  	/* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
  	store_func_meta(ctx, func_meta, func_meta_off);
 
  	/* save args for bpf */
-	save_args(ctx, bargs_off, oargs_off, m, a, false);
+	save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
  	/* save callee saved registers */
  	emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);

arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785

Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1 -1
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
  	if (flags & BPF_TRAMP_F_CALL_ORIG) {
  		/* save args for original func */
-		save_args(ctx, bargs_off, oargs_off, m, a, true);
+		save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
  		/* call original func */
  		emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
  		emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.

Quoted 1 line

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d

#3 lore raw
Quoted 67 lines · Xu Kuohai
Quoted 11 lines · Puranjay Mohan

save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.

Quoted 4 lines

This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.

Which x86 fix? Add a reference to it?

Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
---
  arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
  1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +11 -3
index c6756b964c843..4fe235f7ce894 100644
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
  static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
                    const struct btf_func_model *m,
                    const struct arg_aux *a,
-                   bool for_call_origin)
+                   bool for_call_origin, bool is_struct_ops)
  {
      int i;
      int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
              bargs_off += 8;
      }
 
-     soff = 32; /* on stack arguments start from FP + 32 */
+     /*
+      * On-stack arguments start above the frame(s) pushed by the
+      * trampoline prologue. A traced function is entered through the
+      * fentry call, so both the parent (FP/x9) and the traced function
+      * (FP/LR) frames are saved and the arguments start at FP + 32. A
+      * struct_ops callback is called indirectly, so only the FP/LR frame
+      * is saved and the arguments start at FP + 16.
+      */
+     soff = is_struct_ops ? 16 : 32;
      doff = (for_call_origin ? oargs_off : bargs_off);
 
      /* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
      store_func_meta(ctx, func_meta, func_meta_off);
 
      /* save args for bpf */
-     save_args(ctx, bargs_off, oargs_off, m, a, false);
+     save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
      /* save callee saved registers */
      emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
Quoted 11 lines

arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785

Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1 -1
      if (flags & BPF_TRAMP_F_CALL_ORIG) {
              /* save args for original func */
-             save_args(ctx, bargs_off, oargs_off, m, a, true);
+             save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
              /* call original func */
              emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
              emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.

Kumar is working on a similar fix for x86 and will add a selftest with it. I will wait and repost this patch again when Kumar's patchset lands.
Thanks for your review.

#4 lore raw
Quoted 80 lines · Puranjay Mohan
Quoted 67 lines · Xu Kuohai
Quoted 11 lines · Puranjay Mohan

save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.

An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.

Quoted 4 lines

This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.

Which x86 fix? Add a reference to it?

Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")
---
  arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++----
  1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +11 -3
index c6756b964c843..4fe235f7ce894 100644
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
  static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
                    const struct btf_func_model *m,
                    const struct arg_aux *a,
-                   bool for_call_origin)
+                   bool for_call_origin, bool is_struct_ops)
  {
      int i;
      int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
              bargs_off += 8;
      }
 
-     soff = 32; /* on stack arguments start from FP + 32 */
+     /*
+      * On-stack arguments start above the frame(s) pushed by the
+      * trampoline prologue. A traced function is entered through the
+      * fentry call, so both the parent (FP/x9) and the traced function
+      * (FP/LR) frames are saved and the arguments start at FP + 32. A
+      * struct_ops callback is called indirectly, so only the FP/LR frame
+      * is saved and the arguments start at FP + 16.
+      */
+     soff = is_struct_ops ? 16 : 32;
      doff = (for_call_origin ? oargs_off : bargs_off);
 
      /* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
      store_func_meta(ctx, func_meta, func_meta_off);
 
      /* save args for bpf */
-     save_args(ctx, bargs_off, oargs_off, m, a, false);
+     save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);
 
      /* save callee saved registers */
      emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 
Quoted 11 lines

arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785

Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1 -1
      if (flags & BPF_TRAMP_F_CALL_ORIG) {
              /* save args for original func */
-             save_args(ctx, bargs_off, oargs_off, m, a, true);
+             save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);
              /* call original func */
              emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
              emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.

Quoted 4 lines

Kumar is working on a similar fix for x86 and will add a selftest with it. I will wait and repost this patch again when Kumar's patchset lands.
Thanks for your review.

Yes, please resend when that lands, along with changes to enable tests on arm64 and the remaining JIT-side changes needed to translate arena args.

pw-bot: cr