bpf, arm64: Fix stack-passed arguments for indirect trampolines
Subsystems (3)
Puranjay Mohan
maintainerPATCH 1/1 bpf, arm64: Fix stack-passed arguments for indirect trampolines
save_args() reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.
An indirect trampoline for a struct_ops callback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of a struct_ops callback with more than eight argument slots is read two slots off.
This has gone unnoticed because no in-tree struct_ops member passes arguments on the stack. Pass is_struct_ops into save_args() and pick the offset accordingly, mirroring the x86 fix.
--- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)
arch/arm64/net/bpf_jit_comp.c +12 -4
@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)static void save_args (struct jit_ctx *ctx,int bargs_off,int oargs_off,const struct btf_func_model *m,const struct arg_aux *a, -bool for_call_origin) +bool for_call_origin,bool is_struct_ops) {int i;int reg;
@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off +=8 ; } - soff =32 ;/* on stack arguments start from FP + 32 */ +/* +* On-stack arguments start above the frame(s) pushed by the +* trampoline prologue. A traced function is entered through the +* fentry call, so both the parent (FP/x9) and the traced function +* (FP/LR) frames are saved and the arguments start at FP + 32. A +* struct_ops callback is called indirectly, so only the FP/LR frame +* is saved and the arguments start at FP + 16. +*/ + soff = is_struct_ops ?16 :32 ; doff = (for_call_origin ? oargs_off : bargs_off);/* save on stack arguments */
@@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,store_func_meta (ctx, func_meta, func_meta_off);/* save args for bpf */ -save_args (ctx, bargs_off, oargs_off, m, a, false); +save_args (ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);/* save callee saved registers */ emit (A64_STR64I (A64_R (19 ),A64_SP , regs_off), ctx);
@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* save args for original func */ -save_args (ctx, bargs_off, oargs_off, m, a, true); +save_args (ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);/* call original func */ emit (A64_LDR64I (A64_R (10 ),A64_SP , retaddr_off), ctx);emit (A64_ADR (A64_LR ,AARCH64_INSN_SIZE *2 ), ctx);
base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
Signature · 1 line
2.53.0-Meta
+cc Xu
Quoted 75 lines · Puranjay Mohan
save_args()reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.An indirect trampoline for a
struct_opscallback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of astruct_opscallback with more than eight argument slots is read two slots off.This has gone unnoticed because no in-tree
struct_opsmember passes arguments on the stack. Passis_struct_opsintosave_args()and pick the offset accordingly, mirroring the x86 fix.Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)arch/arm64/net/bpf_jit_comp.c +12
-4@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)static void save_args (struct jit_ctx *ctx,int bargs_off,int oargs_off,const struct btf_func_model *m,const struct arg_aux *a, -bool for_call_origin) +bool for_call_origin,bool is_struct_ops) {int i;int reg;@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off +=8 ; } - soff =32 ;/* on stack arguments start from FP + 32 */ +/* +* On-stack arguments start above the frame(s) pushed by the +* trampoline prologue. A traced function is entered through the +* fentry call, so both the parent (FP/x9) and the traced function +* (FP/LR) frames are saved and the arguments start at FP + 32. A +* struct_ops callback is called indirectly, so only the FP/LR frame +* is saved and the arguments start at FP + 16. +*/ + soff = is_struct_ops ?16 :32 ; doff = (for_call_origin ? oargs_off : bargs_off);/* save on stack arguments */ @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,store_func_meta (ctx, func_meta, func_meta_off);/* save args for bpf */ -save_args (ctx, bargs_off, oargs_off, m, a, false); +save_args (ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);/* save callee saved registers */ emit (A64_STR64I (A64_R (19 ),A64_SP , regs_off), ctx);@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* save args for original func */ -save_args (ctx, bargs_off, oargs_off, m, a, true); +save_args (ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);/* call original func */ emit (A64_LDR64I (A64_R (10 ),A64_SP , retaddr_off), ctx);emit (A64_ADR (A64_LR ,AARCH64_INSN_SIZE *2 ), ctx);base-commit:
1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d--
2.53.0-Meta
Xu Kuohai
reviewerQuoted 11 lines · Puranjay Mohan
save_args()reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.An indirect trampoline for a
struct_opscallback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of astruct_opscallback with more than eight argument slots is read two slots off.
Quoted 4 lines
This has gone unnoticed because no in-tree
struct_opsmember passes arguments on the stack. Passis_struct_opsintosave_args()and pick the offset accordingly, mirroring the x86 fix.
Which x86 fix? Add a reference to it?
Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)arch/arm64/net/bpf_jit_comp.c +11
-3@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)static void save_args (struct jit_ctx *ctx,int bargs_off,int oargs_off,const struct btf_func_model *m,const struct arg_aux *a, -bool for_call_origin) +bool for_call_origin,bool is_struct_ops) {int i;int reg;@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off +=8 ; } - soff =32 ;/* on stack arguments start from FP + 32 */ +/* +* On-stack arguments start above the frame(s) pushed by the +* trampoline prologue. A traced function is entered through the +* fentry call, so both the parent (FP/x9) and the traced function +* (FP/LR) frames are saved and the arguments start at FP + 32. A +* struct_ops callback is called indirectly, so only the FP/LR frame +* is saved and the arguments start at FP + 16. +*/ + soff = is_struct_ops ?16 :32 ; doff = (for_call_origin ? oargs_off : bargs_off);/* save on stack arguments */ @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,store_func_meta (ctx, func_meta, func_meta_off);/* save args for bpf */ -save_args (ctx, bargs_off, oargs_off, m, a, false); +save_args (ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);/* save callee saved registers */ emit (A64_STR64I (A64_R (19 ),A64_SP , regs_off), ctx);
arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785
Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1
-1@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* save args for original func */ -save_args (ctx, bargs_off, oargs_off, m, a, true); +save_args (ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);/* call original func */ emit (A64_LDR64I (A64_R (10 ),A64_SP , retaddr_off), ctx);emit (A64_ADR (A64_LR ,AARCH64_INSN_SIZE *2 ), ctx);
LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.
Quoted 1 line
base-commit:
1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d
Quoted 67 lines · Xu Kuohai
Quoted 11 lines · Puranjay Mohan
save_args()reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.An indirect trampoline for a
struct_opscallback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of astruct_opscallback with more than eight argument slots is read two slots off.Quoted 4 lines
This has gone unnoticed because no in-tree
struct_opsmember passes arguments on the stack. Passis_struct_opsintosave_args()and pick the offset accordingly, mirroring the x86 fix.Which x86 fix? Add a reference to it?
Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)arch/arm64/net/bpf_jit_comp.c +11
-3@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)static void save_args (struct jit_ctx *ctx,int bargs_off,int oargs_off,const struct btf_func_model *m,const struct arg_aux *a, -bool for_call_origin) +bool for_call_origin,bool is_struct_ops) {int i;int reg;@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off +=8 ; } - soff =32 ;/* on stack arguments start from FP + 32 */ +/* +* On-stack arguments start above the frame(s) pushed by the +* trampoline prologue. A traced function is entered through the +* fentry call, so both the parent (FP/x9) and the traced function +* (FP/LR) frames are saved and the arguments start at FP + 32. A +* struct_ops callback is called indirectly, so only the FP/LR frame +* is saved and the arguments start at FP + 16. +*/ + soff = is_struct_ops ?16 :32 ; doff = (for_call_origin ? oargs_off : bargs_off);/* save on stack arguments */ @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,store_func_meta (ctx, func_meta, func_meta_off);/* save args for bpf */ -save_args (ctx, bargs_off, oargs_off, m, a, false); +save_args (ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);/* save callee saved registers */ emit (A64_STR64I (A64_R (19 ),A64_SP , regs_off), ctx);@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
Quoted 11 lines
arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785
Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1
-1if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* save args for original func */ -save_args (ctx, bargs_off, oargs_off, m, a, true); +save_args (ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);/* call original func */ emit (A64_LDR64I (A64_R (10 ),A64_SP , retaddr_off), ctx);emit (A64_ADR (A64_LR ,AARCH64_INSN_SIZE *2 ), ctx);LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.
Kumar is working on a similar fix for x86 and will add a selftest with it. I will wait and repost this patch again when Kumar's patchset lands.
Thanks for your review.
Kumar Kartikeya Dwivedi
maintainerQuoted 80 lines · Puranjay Mohan
Quoted 67 lines · Xu Kuohai
Quoted 11 lines · Puranjay Mohan
save_args()reads stack-passed arguments relative to FP assuming the trampoline is entered through the fentry call from a traced function, in which case both the parent frame (FP/x9) and the traced function frame (FP/LR) are saved before FP is set, so the arguments start at FP + 32.An indirect trampoline for a
struct_opscallback is entered through a function pointer (blr), so only the FP/LR frame is pushed and the arguments start at FP + 16, not FP + 32. Every stack-passed argument of astruct_opscallback with more than eight argument slots is read two slots off.Quoted 4 lines
This has gone unnoticed because no in-tree
struct_opsmember passes arguments on the stack. Passis_struct_opsintosave_args()and pick the offset accordingly, mirroring the x86 fix.Which x86 fix? Add a reference to it?
Quoted 47 lines
Fixes 9014cf56f13d ("bpf, arm64: Support up to 12 function arguments")Signed-off-by Puranjay Mohan <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-)arch/arm64/net/bpf_jit_comp.c +11
-3@@ -2506,7 +2506,7 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)static void save_args (struct jit_ctx *ctx,int bargs_off,int oargs_off,const struct btf_func_model *m,const struct arg_aux *a, -bool for_call_origin) +bool for_call_origin,bool is_struct_ops) {int i;int reg;@@ -2526,7 +2526,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off +=8 ; } - soff =32 ;/* on stack arguments start from FP + 32 */ +/* +* On-stack arguments start above the frame(s) pushed by the +* trampoline prologue. A traced function is entered through the +* fentry call, so both the parent (FP/x9) and the traced function +* (FP/LR) frames are saved and the arguments start at FP + 32. A +* struct_ops callback is called indirectly, so only the FP/LR frame +* is saved and the arguments start at FP + 16. +*/ + soff = is_struct_ops ?16 :32 ; doff = (for_call_origin ? oargs_off : bargs_off);/* save on stack arguments */ @@ -2722,7 +2730,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,store_func_meta (ctx, func_meta, func_meta_off);/* save args for bpf */ -save_args (ctx, bargs_off, oargs_off, m, a, false); +save_args (ctx, bargs_off, oargs_off, m, a, false, is_struct_ops);/* save callee saved registers */ emit (A64_STR64I (A64_R (19 ),A64_SP , regs_off), ctx);@@ -2771,7 +2779,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,Quoted 11 lines
arch/arm64/net/bpf_jit_comp.c static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · line 2785
Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +1
-1if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* save args for original func */ -save_args (ctx, bargs_off, oargs_off, m, a, true); +save_args (ctx, bargs_off, oargs_off, m, a, true, is_struct_ops);/* call original func */ emit (A64_LDR64I (A64_R (10 ),A64_SP , retaddr_off), ctx);emit (A64_ADR (A64_LR ,AARCH64_INSN_SIZE *2 ), ctx);LGTM. Since this issue is not covered by the existing selftests, I think we need to add a new one.
Quoted 4 lines
Kumar is working on a similar fix for x86 and will add a selftest with it. I will wait and repost this patch again when Kumar's patchset lands.
Thanks for your review.
Yes, please resend when that lands, along with changes to enable tests on arm64 and the remaining JIT-side changes needed to translate arena args.
pw-bot: cr