powerpc64/bpf: various fixes
Subsystems (4)
Hari Bathini
maintainerv2 cover powerpc64/bpf: various fixes
List of fixes for powerpc64 BPF JIT:
- Patches 1 & 2 fix issues with existing powerpc64 JIT and are tagged for -stable.
- Patches 3 & 4 resolve issues with bpf2bpf tailcall support added recently.
- Patch 5 resolves an issue reported by [email protected] while adding exceptions support on powerpc64 BPF JIT.
Changes since v1:
- Addressed couple of review comments from [email protected] in patch 3 & 5.
Hari Bathini (5):
powerpc64/bpf: do not increment tailcall count when prog is NULL powerpc64/bpf: fix the address returned by bpf_get_func_ip powerpc64/bpf: use consistent tailcall offset in trampoline powerpc64/bpf: remove BPF redzone protection in trampoline stack powerpc64/bpf: fix handling of BPF stack in exception callback
arch/powerpc/net/bpf_jit.h | 5 -- arch/powerpc/net/bpf_jit_comp.c | 118 +++++++++++++----------------- arch/powerpc/net/bpf_jit_comp64.c | 74 ++++++++++++------- 3 files changed, 96 insertions(+), 101 deletions(-)
Signature · 1 line
2.53.0
Hari Bathini
maintainerPATCH v2 1/5 powerpc64/bpf: do not increment tailcall count when prog is NULL
Do not increment tailcall count, if tailcall did not succeed due to missing BPF program.
---
* No changes since v1.
arch/powerpc/net/bpf_jit_comp64.c | 39 +++++++++++++++++-------------- 1 file changed, 21 insertions(+), 18 deletions(-)
arch/powerpc/net/bpf_jit_comp64.c +21 -18
@@ -522,9 +522,30 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o/* * tail_call_info++; <- Actual value of tcc here +* Writeback this updated value only if tailcall succeeds. */ EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ),1 )); +/* prog = array->ptrs[index]; */ +EMIT (PPC_RAW_MULI (bpf_to_ppc (TMP_REG_2 ), b2p_index,8 )); +EMIT (PPC_RAW_ADD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), b2p_bpf_array)); +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + offsetof(struct bpf_array , ptrs))); + +/* +* if (prog == NULL) +* goto out; +*/ +EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_2 ),0 )); +PPC_BCC_SHORT (COND_EQ , out); + +/* goto *(prog->bpf_func + prologue_size); */ +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + offsetof(struct bpf_prog , bpf_func))); +EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), +FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); +EMIT (PPC_RAW_MTCTR (bpf_to_ppc (TMP_REG_2 ))); +/* * Before writing updated tail_call_info, distinguish if current frame * is storing a reference to tail_call_info or actual tcc value in
@@ -539,24 +560,6 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o/* Writeback updated value to tail_call_info */ EMIT (PPC_RAW_STD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_2 ),0 )); -/* prog = array->ptrs[index]; */ -EMIT (PPC_RAW_MULI (bpf_to_ppc (TMP_REG_1 ), b2p_index,8 )); -EMIT (PPC_RAW_ADD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), b2p_bpf_array)); -EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), offsetof(struct bpf_array , ptrs))); - -/* -* if (prog == NULL) -* goto out; -*/ -EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_1 ),0 )); -PPC_BCC_SHORT (COND_EQ , out); - -/* goto *(prog->bpf_func + prologue_size); */ -EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), offsetof(struct bpf_prog , bpf_func))); -EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), -FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); -EMIT (PPC_RAW_MTCTR (bpf_to_ppc (TMP_REG_1 ))); -/* tear down stack, restore NVRs, ... */ bpf_jit_emit_common_epilogue (image, ctx);
Signature · 1 line
2.53.0
Hari Bathini
maintainerPATCH v2 2/5 powerpc64/bpf: fix the address returned by bpf_get_func_ip
In reply to Hari Bathinibpf_get_func_ip() helper function returns the address of the traced function. It relies on the IP address stored at ctx - 16 by the bpf trampoline. On 64-bit powerpc, this address is recovered from LR accounting for OOL trampoline. But the address stored here was off by 4-bytes. Ensure the address is the actual start of the traced function.
---
* No changes since v1.
arch/powerpc/net/bpf_jit_comp.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)
arch/powerpc/net/bpf_jit_comp.c +13 -8
@@ -786,8 +786,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im* [ reg argN ] * [ ... ] * regs_off [ reg_arg1 ] prog ctx context -* nregs_off [ args count ] -* ip_off [ traced function ] +* nregs_off [ args count ] ((u64 *)prog_ctx)[-1] +* ip_off [ traced function ] ((u64 *)prog_ctx)[-2] * [ ... ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ]
@@ -895,7 +895,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imbpf_trampoline_save_args (image, ctx, func_frame_offset, nr_regs, regs_off); -/* Save our return address */ +/* Save our LR/return address */ EMIT (PPC_RAW_MFLR (_R3));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ))EMIT (PPC_RAW_STL (_R3, _R1, alt_lr_off));
@@ -903,24 +903,29 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF ));/* -* Save ip address of the traced function. -* We could recover this from LR, but we will need to address for OOL trampoline, -* and optional GEP area. +* Get IP address of the traced function. +* In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR +* points to the instruction after the 'bl' instruction in the OOL stub. +* Refer to ftrace_init_ool_stub() and bpf_arch_text_poke() for OOL stub +* of kernel functions and bpf programs respectively. +* Recover kernel function/bpf program address from the unconditional +* branch instruction at the end of OOL stub. */ if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ) || flags &BPF_TRAMP_F_IP_ARG ) {EMIT (PPC_RAW_LWZ (_R4, _R3,4 ));EMIT (PPC_RAW_SLWI (_R4, _R4,6 ));EMIT (PPC_RAW_SRAWI (_R4, _R4,6 ));EMIT (PPC_RAW_ADD (_R3, _R3, _R4)); -EMIT (PPC_RAW_ADDI (_R3, _R3,4 )); }if (flags &BPF_TRAMP_F_IP_ARG )EMIT (PPC_RAW_STL (_R3, _R1, ip_off)); -if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) +if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) {/* Fake our LR for unwind */ +EMIT (PPC_RAW_ADDI (_R3, _R3,4 ));EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); + }/* Save function arg count -- see bpf_get_func_arg_cnt() */ EMIT (PPC_RAW_LI (_R3, nr_regs));
Signature · 1 line
2.53.0
Hari Bathini
maintainerPATCH v2 3/5 powerpc64/bpf: use consistent tailcall offset in trampoline
In reply to Hari BathiniIdeally, the offset used to load the tail call info field and to find the pass by reference address for tail call field should be the same. But while setting up the tail call info in the trampoline, this was not followed. This can be misleading and can lead to unpredictable results if and when bpf_has_stack_frame() ends up returning true for trampoline frame. Since commit 15513beeb673 ("powerpc64/bpf:
Moving tail_call_cnt to bottom of frame") and commit 2ed2d8f6fb38 ("powerpc64/bpf: Support tailcalls with subprogs") ensured tail call field is at the bottom of the stack frame for BPF programs as well as BPF trampoline, avoid relying on bpf_jit_stack_tailcallinfo_offset() and bpf_has_stack_frame() for trampoline frame and always calculate tail call field offset with reference to older frame.
---
Changes since v1:
* Fixed spelling error in changelog.
* Fixed a comment in bpf_trampoline_setup_tail_call_info().
arch/powerpc/net/bpf_jit.h | 5 ----- arch/powerpc/net/bpf_jit_comp.c | 12 +++++------- arch/powerpc/net/bpf_jit_comp64.c | 5 ++++- 3 files changed, 9 insertions(+), 13 deletions(-)
arch/powerpc/net/bpf_jit.h +0 -5
@@ -81,9 +81,6 @@#ifdef CONFIG_PPC64 -/* for gpr non volatile registers BPG_REG_6 to 10 */ -#define BPF_PPC_STACK_SAVE (6 * 8) -/* If dummy pass (!image), account for maximum possible instructions */ #define PPC_LI64 (d, i) do { \ if (!image) \
@@ -219,8 +216,6 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg,int bpf_add_extable_entry (struct bpf_prog *fp,u32 *image,u32 *fimage,int pass,struct codegen_context *ctx,int insn_idx,int jmp_off,int dst_reg,u32 code); - -int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx);#endif #endif
arch/powerpc/net/bpf_jit_comp.c +5 -7
@@ -642,15 +642,13 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte int bpf_dummy_frame_size, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { - /* See Generated stack layout */ - int tailcallinfo_offset = BPF_PPC_TAILCALL; -/* * func_frame_offset = ...(1) * bpf_dummy_frame_size + trampoline_frame_size */ EMIT (PPC_RAW_LD (_R4 ,_R1 ,func_frame_offset )); -EMIT (PPC_RAW_LD (_R3, _R4, -tailcallinfo_offset)); +/* Refer to trampoline's Generated stack layout */ +EMIT (PPC_RAW_LD (_R3, _R4, -BPF_PPC_TAILCALL ));/* * Setting the tail_call_info in trampoline's frame
@@ -658,7 +656,7 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte*/ EMIT (PPC_RAW_CMPLWI (_R3,MAX_TAIL_CALL_CNT ));PPC_BCC_CONST_SHORT (COND_GT ,8 ); -EMIT (PPC_RAW_ADDI (_R3, _R4,bpf_jit_stack_tailcallinfo_offset (ctx))); +EMIT (PPC_RAW_ADDI (_R3, _R4, -BPF_PPC_TAILCALL ));/* * From ...(1) above: * trampoline_frame_bottom = ...(2)
@@ -666,14 +664,14 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte* * Using ...(2) derived above: * trampoline_tail_call_info_offset = ...(3) -* trampoline_frame_bottom - tailcallinfo_offset +* trampoline_frame_bottom - BPF_PPC_TAILCALL * * From ...(3): * Use trampoline_tail_call_info_offset to write reference of main's * tail_call_info in trampoline frame. */ EMIT (PPC_RAW_STL (_R3, _R1, (func_frame_offset - bpf_dummy_frame_size) - - tailcallinfo_offset)); + -BPF_PPC_TAILCALL )); }else {/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT (PPC_RAW_LL (_R4, _R1, r4_off));
arch/powerpc/net/bpf_jit_comp64.c +4 -1
@@ -42,6 +42,9 @@* exception boundary. */ +/* BPF non-volatile registers save area size */ +#define BPF_PPC_STACK_SAVE (6*8) +/* for bpf JIT code internal usage */ #define BPF_PPC_STACK_LOCALS 24/*
@@ -148,7 +151,7 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) } } -int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx) +static int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx) {return bpf_jit_stack_local (ctx) +BPF_PPC_STACK_LOCALS +BPF_PPC_STACK_SAVE ; }
Signature · 1 line
2.53.0
Hari Bathini
maintainerPATCH v2 4/5 powerpc64/bpf: remove BPF redzone protection in trampoline stack
In reply to Hari BathiniSince bpf2bpf tailcall support is enabled for 64-bit powerpc with kernel commit 2ed2d8f6fb38 ("powerpc64/bpf: Support tailcalls with subprogs"), 'tailcalls/tailcall_bpf2bpf_hierarchy_fexit' BPF selftest is triggering "corrupted stack end detected inside scheduler" with the config option CONFIG_SCHED_STACK_END_CHECK enabled. While reviewing the stack layout for BPF trampoline, observed that the dummy frame is trying to protect the redzone of BPF program. This is because tail call info and NVRs save area are in redzone at the time of tailcall as the current BPF program stack frame is teared down before the tailcall. But saving this redzone in the dummy frame of trampoline is unnecessary because of the follow reasons:
1) Firstly, trampoline can be attached to BPF entry/main program or subprog. But prologue part of the BPF entry/main program, where the trampoline attachpoint is, is skipped during tailcall. So, protecting the redzone does not arise when the trampoline is not even triggered in this scenario.
2) In case of subprog, the caller's stackframe is already setup and the subprog's stackframe is yet to be setup. So, nothing on the redzone to be protected.
Also, using dummy frame in BPF trampoline, wastes critically scarce kernel stack space, especially in tailcall sequence, for marginal benefit in stack unwinding. So, drop setting up the dummy frame. Instead, save return address in bpf trampoline frame and use it as appropriate. Pruning this unnecessary stack usage mitigates the likelihood of stack overflow in scenarios where bpf2bpf tailcalls and fexit programs are mixed.
--- arch/powerpc/net/bpf_jit_comp.c | 89 ++++++++++++--------------------- 1 file changed, 33 insertions(+), 56 deletions(-)
arch/powerpc/net/bpf_jit_comp.c +33 -56
@@ -638,15 +638,10 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context* for the traced function (BPF subprog/callee) to fetch it. */ static void bpf_trampoline_setup_tail_call_info (u32 *image,struct codegen_context *ctx, -int func_frame_offset, -int bpf_dummy_frame_size,int r4_off) +int bpf_frame_size,int r4_off) {if (IS_ENABLED (CONFIG_PPC64 )) { -/* -* func_frame_offset = ...(1) -* bpf_dummy_frame_size + trampoline_frame_size -*/ -EMIT (PPC_RAW_LD (_R4, _R1, func_frame_offset)); +EMIT (PPC_RAW_LD (_R4, _R1, bpf_frame_size));/* Refer to trampoline's Generated stack layout */ EMIT (PPC_RAW_LD (_R3, _R4, -BPF_PPC_TAILCALL ));
@@ -657,21 +652,13 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conteEMIT (PPC_RAW_CMPLWI (_R3,MAX_TAIL_CALL_CNT ));PPC_BCC_CONST_SHORT (COND_GT ,8 );EMIT (PPC_RAW_ADDI (_R3, _R4, -BPF_PPC_TAILCALL )); +/* -* From ...(1) above: -* trampoline_frame_bottom = ...(2) -* func_frame_offset - bpf_dummy_frame_size -* -* Using ...(2) derived above: -* trampoline_tail_call_info_offset = ...(3) -* trampoline_frame_bottom - BPF_PPC_TAILCALL -* -* From ...(3): -* Use trampoline_tail_call_info_offset to write reference of main's -* tail_call_info in trampoline frame. +* Trampoline's tail_call_info is at the same offset, as that of +* any bpf program, with reference to previous frame. Update the +* address of main's tail_call_info in trampoline frame. */ -EMIT (PPC_RAW_STL (_R3, _R1, (func_frame_offset - bpf_dummy_frame_size) - -BPF_PPC_TAILCALL )); +EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size -BPF_PPC_TAILCALL )); }else {/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT (PPC_RAW_LL (_R4, _R1, r4_off));
@@ -679,7 +666,7 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte }static void bpf_trampoline_restore_tail_call_cnt (u32 *image,struct codegen_context *ctx, -int func_frame_offset,int r4_off) +int bpf_frame_size,int r4_off) {if (IS_ENABLED (CONFIG_PPC32 )) {/*
@@ -690,12 +677,12 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont } } -static void bpf_trampoline_save_args (u32 *image,struct codegen_context *ctx,int func_frame_offset, -int nr_regs,int regs_off) +static void bpf_trampoline_save_args (u32 *image,struct codegen_context *ctx, +int bpf_frame_size,int nr_regs,int regs_off) {int param_save_area_offset; - param_save_area_offset = func_frame_offset;/* the two frames we alloted */ + param_save_area_offset = bpf_frame_size; param_save_area_offset +=STACK_FRAME_MIN_SIZE ;/* param save area is past frame header */ for (int i =0 ; i < nr_regs; i++) {
@@ -718,11 +705,11 @@ static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context/* Used when we call into the traced function. Replicate parameter save area */ static void bpf_trampoline_restore_args_stack (u32 *image,struct codegen_context *ctx, -int func_frame_offset,int nr_regs,int regs_off) +int bpf_frame_size,int nr_regs,int regs_off) {int param_save_area_offset; - param_save_area_offset = func_frame_offset;/* the two frames we alloted */ + param_save_area_offset = bpf_frame_size; param_save_area_offset +=STACK_FRAME_MIN_SIZE ;/* param save area is past frame header */ for (int i =8 ; i < nr_regs; i++) {
@@ -739,10 +726,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im void *func_addr) {int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off =0 ; -int i, ret, nr_regs, bpf_frame_size =0 , bpf_dummy_frame_size =0 , func_frame_offset;struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN ];struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY ];struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT ]; +int i, ret, nr_regs, retaddr_off, bpf_frame_size =0 ;struct codegen_context codegen_ctx, *ctx;u32 *image = (u32 *)rw_image;ppc_inst_t branch_insn;
@@ -768,16 +755,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im* Generated stack layout: * * func prev back chain [ back chain ] -* [ ] -* bpf prog redzone/tailcallcnt [ ... ] 64 bytes (64-bit powerpc) -* [ ] -- -* LR save area [ r0 save (64-bit) ] | header -* [ r0 save (32-bit) ] | -* dummy frame for unwind [ back chain 1 ] -- * [ tail_call_info ] optional - 64-bit powerpc * [ padding ] align stack frame * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc * alt_lr_off [ real lr (ool stub)] optional - actual lr +* retaddr_off [ return address ] * [ r26 ] * nvr_off [ r25 ] nvr save area * retval_off [ return value ]
@@ -841,6 +823,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im nvr_off = bpf_frame_size; bpf_frame_size +=2 *SZL ; +/* Save area for return address */ + retaddr_off = bpf_frame_size; + bpf_frame_size +=SZL ; +/* Optional save area for actual LR in case of ool ftrace */ if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) { alt_lr_off = bpf_frame_size;
@@ -867,16 +853,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Padding to align stack frame, if any */ bpf_frame_size =round_up (bpf_frame_size,SZL *2 ); -/* Dummy frame size for proper unwind - includes 64-bytes red zone for 64-bit powerpc */ - bpf_dummy_frame_size =STACK_FRAME_MIN_SIZE +64 ; - -/* Offset to the traced function's stack frame */ - func_frame_offset = bpf_dummy_frame_size + bpf_frame_size; - -/* Create dummy frame for unwind, store original return value */ +/* Store original return value */ EMIT (PPC_RAW_STL (_R0, _R1,PPC_LR_STKOFF )); -/* Protect red zone where tail call count goes */ -EMIT (PPC_RAW_STLU (_R1, _R1, -bpf_dummy_frame_size));/* Create our stack frame */ EMIT (PPC_RAW_STLU (_R1, _R1, -bpf_frame_size));
@@ -891,14 +869,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imif (IS_ENABLED (CONFIG_PPC32 ) && nr_regs <2 )EMIT (PPC_RAW_STL (_R4, _R1, r4_off)); -bpf_trampoline_save_args (image, ctx, func_frame_offset, nr_regs, regs_off); +bpf_trampoline_save_args (image, ctx, bpf_frame_size, nr_regs, regs_off);/* Save our LR/return address */ EMIT (PPC_RAW_MFLR (_R3));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ))EMIT (PPC_RAW_STL (_R3, _R1, alt_lr_off));else -EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_STL (_R3, _R1, retaddr_off));/* * Get IP address of the traced function.
@@ -920,9 +898,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_STL (_R3, _R1, ip_off));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) { -/* Fake our LR for unwind */ +/* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ EMIT (PPC_RAW_ADDI (_R3, _R3,4 )); -EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_STL (_R3, _R1, retaddr_off)); }/* Save function arg count -- see bpf_get_func_arg_cnt() */
@@ -961,20 +939,19 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Call the traced function */ if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* -* The address in LR save area points to the correct point in the original function +* retaddr on trampoline stack points to the correct point in the original function * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction * sequence */ -EMIT (PPC_RAW_LL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_LL (_R3, _R1, retaddr_off));EMIT (PPC_RAW_MTCTR (_R3));/* Replicate tail_call_cnt before calling the original BPF prog */ if (flags &BPF_TRAMP_F_TAIL_CALL_CTX ) -bpf_trampoline_setup_tail_call_info (image, ctx, func_frame_offset, - bpf_dummy_frame_size, r4_off); +bpf_trampoline_setup_tail_call_info (image, ctx, bpf_frame_size, r4_off);/* Restore args */ -bpf_trampoline_restore_args_stack (image, ctx, func_frame_offset, nr_regs, regs_off); +bpf_trampoline_restore_args_stack (image, ctx, bpf_frame_size, nr_regs, regs_off);/* Restore TOC for 64-bit */ if (IS_ENABLED (CONFIG_PPC64_ELF_ABI_V2 ) && !IS_ENABLED (CONFIG_PPC_KERNEL_PCREL ))
@@ -988,7 +965,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Restore updated tail_call_cnt */ if (flags &BPF_TRAMP_F_TAIL_CALL_CTX ) -bpf_trampoline_restore_tail_call_cnt (image, ctx, func_frame_offset, r4_off); +bpf_trampoline_restore_tail_call_cnt (image, ctx, bpf_frame_size, r4_off);/* Reserve space to patch branch instruction to skip fexit progs */ if (ro_image)/* image is NULL for dummy pass */
@@ -1040,7 +1017,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_LD (_R2, _R1,24 ));if (flags &BPF_TRAMP_F_SKIP_FRAME ) {/* Skip the traced function and return to parent */ -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));EMIT (PPC_RAW_MTLR (_R0));EMIT (PPC_RAW_BLR ());
@@ -1048,13 +1025,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imif (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) {EMIT (PPC_RAW_LL (_R0, _R1, alt_lr_off));EMIT (PPC_RAW_MTLR (_R0)); -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));EMIT (PPC_RAW_BLR ()); }else { -EMIT (PPC_RAW_LL (_R0, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_LL (_R0, _R1, retaddr_off));EMIT (PPC_RAW_MTCTR (_R0)); -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));EMIT (PPC_RAW_MTLR (_R0));EMIT (PPC_RAW_BCTR ());
Signature · 1 line
2.53.0
Hari Bathini
maintainerPATCH v2 5/5 powerpc64/bpf: fix handling of BPF stack in exception callback
In reply to Hari BathiniException callback reuses the stack frame of exception boundary. When exception boundary and exception callback programs have different BPF stack depth, the current stack unwind in exception callback will fail. Adjust the stack frame size of exception callback, in its prologue, if its BPF stack depth is different from that of exception boundary.
---
Changes since v1:
* Fixed incorrect use of PPC_RAW_SUB() as pointed out by [email protected].
arch/powerpc/net/bpf_jit_comp64.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-)
arch/powerpc/net/bpf_jit_comp64.c +22 -8
@@ -32,14 +32,15 @@* * [ prev sp ] <------------- * [ tail_call_info ] 8 | -* [ nv gpr save area ] 6*8 + (12*8) | +* [ nv gpr save area ] 6*8 | +* [ addl. nv gpr save area] (12*8) | <--- exception boundary/callback program * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 | * sp (r1) ---> [ stack pointer ] -------------- * * Additional (12*8) in 'nv gpr save area' only in case of -* exception boundary. +* exception boundary/callback. */ /* BPF non-volatile registers save area size */
@@ -128,12 +129,13 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx)* [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- * [ tail_call_info ] 8 -* [ nv gpr save area ] 6*8 + (12*8) +* [ nv gpr save area ] 6*8 +* [ addl. nv gpr save area] (12*8) <--- exception boundary/callback program * [ local_tmp_var ] 24 * [ unused red zone ] 224 * * Additional (12*8) in 'nv gpr save area' only in case of -* exception boundary. +* exception boundary/callback. */ static int bpf_jit_stack_local (struct codegen_context *ctx) {
@@ -240,10 +242,6 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)if (bpf_has_stack_frame (ctx) && !ctx->exception_cb) {/* -* exception_cb uses boundary frame after stack walk. -* It can simply use redzone, this optimization reduces -* stack walk loop by one level. -* * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions */
@@ -287,6 +285,22 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)* program(main prog) as third arg */ EMIT (PPC_RAW_MR (_R1, _R5)); +/* +* Exception callback reuses the stack frame of exception boundary. +* But BPF stack depth of exception callback and exception boundary +* don't have to be same. If BPF stack depth is different, adjust the +* stack frame size considering BPF stack depth of exception callback. +* The non-volatile register save area remains unchanged. These non- +* volatile registers are restored in exception callback's epilogue. +*/ +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ), _R5,0 )); +EMIT (PPC_RAW_SUB (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_1 ), _R1)); +EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + -BPF_PPC_EXC_STACKFRAME )); +EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_2 ), ctx->stack_size)); +PPC_BCC_CONST_SHORT (COND_EQ ,12 ); +EMIT (PPC_RAW_MR (_R1,bpf_to_ppc (TMP_REG_1 ))); +EMIT (PPC_RAW_STDU (_R1, _R1, -(BPF_PPC_EXC_STACKFRAME + ctx->stack_size))); }/*
Signature · 1 line
2.53.0
Re: [PATCH v2 1/5] powerpc64/bpf: do not increment tailcall count when prog is NULL
In reply to Hari BathiniQuoted 64 lines · Hari Bathini
Do not increment tailcall count, if tailcall did not succeed due to missing BPF program.
Fixes ce0761419fae ("powerpc/bpf: Implement support for tail calls")Signed-off-by Hari Bathini <[email protected]>---
* No changes since v1.
arch/powerpc/net/bpf_jit_comp64.c | 39 +++++++++++++++++-------------- 1 file changed, 21 insertions(+), 18 deletions(-)arch/powerpc/net/bpf_jit_comp64.c +21
-11@@ -522,9 +522,30 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o/* * tail_call_info++; <- Actual value of tcc here +* Writeback this updated value only if tailcall succeeds. */ EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ),1 )); +/* prog = array->ptrs[index]; */ +EMIT (PPC_RAW_MULI (bpf_to_ppc (TMP_REG_2 ), b2p_index,8 )); +EMIT (PPC_RAW_ADD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), b2p_bpf_array)); +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + offsetof(struct bpf_array , ptrs))); + +/* +* if (prog == NULL) +* goto out; +*/ +EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_2 ),0 )); +PPC_BCC_SHORT (COND_EQ , out); + +/* goto *(prog->bpf_func + prologue_size); */ +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + offsetof(struct bpf_prog , bpf_func))); +EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), +FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); +EMIT (PPC_RAW_MTCTR (bpf_to_ppc (TMP_REG_2 ))); +/* * Before writing updated tail_call_info, distinguish if current frame * is storing a reference to tail_call_info or actual tcc value in @@ -539,24 +560,6 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o/* Writeback updated value to tail_call_info */ EMIT (PPC_RAW_STD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_2 ),0 )); -/* prog = array->ptrs[index]; */ -EMIT (PPC_RAW_MULI (bpf_to_ppc (TMP_REG_1 ), b2p_index,8 )); -EMIT (PPC_RAW_ADD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), b2p_bpf_array)); -EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), offsetof(struct bpf_array , ptrs))); - -/* -* if (prog == NULL) -* goto out; -*/ -EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_1 ),0 )); -PPC_BCC_SHORT (COND_EQ , out);
arch/powerpc/net/bpf_jit_comp64.c static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o · line 565
Quoted 10 lines
arch/powerpc/net/bpf_jit_comp64.c +0
-7- -/* goto *(prog->bpf_func + prologue_size); */ -EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), offsetof(struct bpf_prog , bpf_func))); -EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_1 ),bpf_to_ppc (TMP_REG_1 ), -FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); -EMIT (PPC_RAW_MTCTR (bpf_to_ppc (TMP_REG_1 ))); -/* tear down stack, restore NVRs, ... */ bpf_jit_emit_common_epilogue (image, ctx);
Tested this by patch, Please add below tag.
Regards,
Venkat.
Re: [PATCH v2 2/5] powerpc64/bpf: fix the address returned by bpf_get_func_ip
In reply to Hari BathiniQuoted 68 lines · Hari Bathini
bpf_get_func_ip()helper function returns the address of the traced function. It relies on the IP address stored at ctx - 16 by the bpf trampoline. On 64-bit powerpc, this address is recovered from LR accounting for OOL trampoline. But the address stored here was off by 4-bytes. Ensure the address is the actual start of the traced function.Reported-by Abhishek Dubey <[email protected]>Fixes d243b62b7bd3 ("powerpc64/bpf: Add support for bpf trampolines")Signed-off-by Hari Bathini <[email protected]>---
* No changes since v1.
arch/powerpc/net/bpf_jit_comp.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)arch/powerpc/net/bpf_jit_comp.c +10
-7@@ -786,8 +786,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im* [ reg argN ] * [ ... ] * regs_off [ reg_arg1 ] prog ctx context -* nregs_off [ args count ] -* ip_off [ traced function ] +* nregs_off [ args count ] ((u64 *)prog_ctx)[-1] +* ip_off [ traced function ] ((u64 *)prog_ctx)[-2] * [ ... ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] @@ -895,7 +895,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imbpf_trampoline_save_args (image, ctx, func_frame_offset, nr_regs, regs_off); -/* Save our return address */ +/* Save our LR/return address */ EMIT (PPC_RAW_MFLR (_R3));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ))EMIT (PPC_RAW_STL (_R3, _R1, alt_lr_off));@@ -903,24 +903,29 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF ));/* -* Save ip address of the traced function. -* We could recover this from LR, but we will need to address for OOL trampoline, -* and optional GEP area. +* Get IP address of the traced function. +* In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR +* points to the instruction after the 'bl' instruction in the OOL stub. +* Refer to ftrace_init_ool_stub() and bpf_arch_text_poke() for OOL stub +* of kernel functions and bpf programs respectively. +* Recover kernel function/bpf program address from the unconditional +* branch instruction at the end of OOL stub. */ if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ) || flags &BPF_TRAMP_F_IP_ARG ) {EMIT (PPC_RAW_LWZ (_R4, _R3,4 ));EMIT (PPC_RAW_SLWI (_R4, _R4,6 ));EMIT (PPC_RAW_SRAWI (_R4, _R4,6 ));EMIT (PPC_RAW_ADD (_R3, _R3, _R4)); -EMIT (PPC_RAW_ADDI (_R3, _R3,4 )); }if (flags &BPF_TRAMP_F_IP_ARG )EMIT (PPC_RAW_STL (_R3, _R1, ip_off));
arch/powerpc/net/bpf_jit_comp.c static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im · line 931
Quoted 10 lines
arch/powerpc/net/bpf_jit_comp.c +3
-1-if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) +if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) {/* Fake our LR for unwind */ +EMIT (PPC_RAW_ADDI (_R3, _R3,4 ));EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); + }/* Save function arg count -- see bpf_get_func_arg_cnt() */ EMIT (PPC_RAW_LI (_R3, nr_regs));
./test_progs -t get_func_ip_test
#139 get_func_ip_test:OK
Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
Regards,
Venkat.
Re: [PATCH v2 4/5] powerpc64/bpf: remove BPF redzone protection in trampoline stack
In reply to Hari BathiniQuoted 262 lines · Hari Bathini
Since bpf2bpf tailcall support is enabled for 64-bit powerpc with kernel commit
2ed2d8f6fb38("powerpc64/bpf: Support tailcalls with subprogs"), 'tailcalls/tailcall_bpf2bpf_hierarchy_fexit' BPF selftest is triggering "corrupted stack end detected inside scheduler" with the config optionCONFIG_SCHED_STACK_END_CHECKenabled. While reviewing the stack layout for BPF trampoline, observed that the dummy frame is trying to protect the redzone of BPF program. This is because tail call info and NVRs save area are in redzone at the time of tailcall as the current BPF program stack frame is teared down before the tailcall. But saving this redzone in the dummy frame of trampoline is unnecessary because of the follow reasons:1) Firstly, trampoline can be attached to BPF entry/main program or subprog. But prologue part of the BPF entry/main program, where the trampoline attachpoint is, is skipped during tailcall. So, protecting the redzone does not arise when the trampoline is not even triggered in this scenario.
2) In case of subprog, the caller's stackframe is already setup and the subprog's stackframe is yet to be setup. So, nothing on the redzone to be protected.Also, using dummy frame in BPF trampoline, wastes critically scarce kernel stack space, especially in tailcall sequence, for marginal benefit in stack unwinding. So, drop setting up the dummy frame. Instead, save return address in bpf trampoline frame and use it as appropriate. Pruning this unnecessary stack usage mitigates the likelihood of stack overflow in scenarios where bpf2bpf tailcalls and fexit programs are mixed.
Reported-by Saket Kumar Bhaskar <[email protected]>Fixes 2ed2d8f6fb38 ("powerpc64/bpf: Support tailcalls with subprogs")Signed-off-by Hari Bathini <[email protected]>--- arch/powerpc/net/bpf_jit_comp.c | 89 ++++++++++++--------------------- 1 file changed, 33 insertions(+), 56 deletions(-)arch/powerpc/net/bpf_jit_comp.c +31
-54@@ -638,15 +638,10 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context* for the traced function (BPF subprog/callee) to fetch it. */ static void bpf_trampoline_setup_tail_call_info (u32 *image,struct codegen_context *ctx, -int func_frame_offset, -int bpf_dummy_frame_size,int r4_off) +int bpf_frame_size,int r4_off) {if (IS_ENABLED (CONFIG_PPC64 )) { -/* -* func_frame_offset = ...(1) -* bpf_dummy_frame_size + trampoline_frame_size -*/ -EMIT (PPC_RAW_LD (_R4, _R1, func_frame_offset)); +EMIT (PPC_RAW_LD (_R4, _R1, bpf_frame_size));/* Refer to trampoline's Generated stack layout */ EMIT (PPC_RAW_LD (_R3, _R4, -BPF_PPC_TAILCALL ));@@ -657,21 +652,13 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conteEMIT (PPC_RAW_CMPLWI (_R3,MAX_TAIL_CALL_CNT ));PPC_BCC_CONST_SHORT (COND_GT ,8 );EMIT (PPC_RAW_ADDI (_R3, _R4, -BPF_PPC_TAILCALL )); +/* -* From ...(1) above: -* trampoline_frame_bottom = ...(2) -* func_frame_offset - bpf_dummy_frame_size -* -* Using ...(2) derived above: -* trampoline_tail_call_info_offset = ...(3) -* trampoline_frame_bottom - BPF_PPC_TAILCALL -* -* From ...(3): -* Use trampoline_tail_call_info_offset to write reference of main's -* tail_call_info in trampoline frame. +* Trampoline's tail_call_info is at the same offset, as that of +* any bpf program, with reference to previous frame. Update the +* address of main's tail_call_info in trampoline frame. */ -EMIT (PPC_RAW_STL (_R3, _R1, (func_frame_offset - bpf_dummy_frame_size) - -BPF_PPC_TAILCALL )); +EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size -BPF_PPC_TAILCALL )); }else {/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT (PPC_RAW_LL (_R4, _R1, r4_off));@@ -679,7 +666,7 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte }static void bpf_trampoline_restore_tail_call_cnt (u32 *image,struct codegen_context *ctx, -int func_frame_offset,int r4_off) +int bpf_frame_size,int r4_off) {if (IS_ENABLED (CONFIG_PPC32 )) {/* @@ -690,12 +677,12 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont } } -static void bpf_trampoline_save_args (u32 *image,struct codegen_context *ctx,int func_frame_offset, -int nr_regs,int regs_off) +static void bpf_trampoline_save_args (u32 *image,struct codegen_context *ctx, +int bpf_frame_size,int nr_regs,int regs_off) {int param_save_area_offset; - param_save_area_offset = func_frame_offset;/* the two frames we alloted */ + param_save_area_offset = bpf_frame_size; param_save_area_offset +=STACK_FRAME_MIN_SIZE ;/* param save area is past frame header */ for (int i =0 ; i < nr_regs; i++) {@@ -718,11 +705,11 @@ static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context/* Used when we call into the traced function. Replicate parameter save area */ static void bpf_trampoline_restore_args_stack (u32 *image,struct codegen_context *ctx, -int func_frame_offset,int nr_regs,int regs_off) +int bpf_frame_size,int nr_regs,int regs_off) {int param_save_area_offset; - param_save_area_offset = func_frame_offset;/* the two frames we alloted */ + param_save_area_offset = bpf_frame_size; param_save_area_offset +=STACK_FRAME_MIN_SIZE ;/* param save area is past frame header */ for (int i =8 ; i < nr_regs; i++) {@@ -739,10 +726,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im void *func_addr) {int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off =0 ; -int i, ret, nr_regs, bpf_frame_size =0 , bpf_dummy_frame_size =0 , func_frame_offset;struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN ];struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY ];struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT ]; +int i, ret, nr_regs, retaddr_off, bpf_frame_size =0 ;struct codegen_context codegen_ctx, *ctx;u32 *image = (u32 *)rw_image;ppc_inst_t branch_insn;@@ -768,16 +755,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im* Generated stack layout: * * func prev back chain [ back chain ] -* [ ] -* bpf prog redzone/tailcallcnt [ ... ] 64 bytes (64-bit powerpc) -* [ ] -- -* LR save area [ r0 save (64-bit) ] | header -* [ r0 save (32-bit) ] | -* dummy frame for unwind [ back chain 1 ] -- * [ tail_call_info ] optional - 64-bit powerpc * [ padding ] align stack frame * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc * alt_lr_off [ real lr (ool stub)] optional - actual lr +* retaddr_off [ return address ] * [ r26 ] * nvr_off [ r25 ] nvr save area * retval_off [ return value ] @@ -841,6 +823,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im nvr_off = bpf_frame_size; bpf_frame_size +=2 *SZL ; +/* Save area for return address */ + retaddr_off = bpf_frame_size; + bpf_frame_size +=SZL ; +/* Optional save area for actual LR in case of ool ftrace */ if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) { alt_lr_off = bpf_frame_size;@@ -867,16 +853,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Padding to align stack frame, if any */ bpf_frame_size =round_up (bpf_frame_size,SZL *2 ); -/* Dummy frame size for proper unwind - includes 64-bytes red zone for 64-bit powerpc */ - bpf_dummy_frame_size =STACK_FRAME_MIN_SIZE +64 ; - -/* Offset to the traced function's stack frame */ - func_frame_offset = bpf_dummy_frame_size + bpf_frame_size; - -/* Create dummy frame for unwind, store original return value */ +/* Store original return value */ EMIT (PPC_RAW_STL (_R0, _R1,PPC_LR_STKOFF )); -/* Protect red zone where tail call count goes */ -EMIT (PPC_RAW_STLU (_R1, _R1, -bpf_dummy_frame_size));/* Create our stack frame */ EMIT (PPC_RAW_STLU (_R1, _R1, -bpf_frame_size));@@ -891,14 +869,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imif (IS_ENABLED (CONFIG_PPC32 ) && nr_regs <2 )EMIT (PPC_RAW_STL (_R4, _R1, r4_off)); -bpf_trampoline_save_args (image, ctx, func_frame_offset, nr_regs, regs_off); +bpf_trampoline_save_args (image, ctx, bpf_frame_size, nr_regs, regs_off);/* Save our LR/return address */ EMIT (PPC_RAW_MFLR (_R3));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ))EMIT (PPC_RAW_STL (_R3, _R1, alt_lr_off));else -EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_STL (_R3, _R1, retaddr_off));/* * Get IP address of the traced function. @@ -920,9 +898,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_STL (_R3, _R1, ip_off));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) { -/* Fake our LR for unwind */ +/* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ EMIT (PPC_RAW_ADDI (_R3, _R3,4 )); -EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_STL (_R3, _R1, retaddr_off)); }/* Save function arg count -- see bpf_get_func_arg_cnt() */ @@ -961,20 +939,19 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Call the traced function */ if (flags &BPF_TRAMP_F_CALL_ORIG ) {/* -* The address in LR save area points to the correct point in the original function +* retaddr on trampoline stack points to the correct point in the original function * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction * sequence */ -EMIT (PPC_RAW_LL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_LL (_R3, _R1, retaddr_off));EMIT (PPC_RAW_MTCTR (_R3));/* Replicate tail_call_cnt before calling the original BPF prog */ if (flags &BPF_TRAMP_F_TAIL_CALL_CTX ) -bpf_trampoline_setup_tail_call_info (image, ctx, func_frame_offset, - bpf_dummy_frame_size, r4_off); +bpf_trampoline_setup_tail_call_info (image, ctx, bpf_frame_size, r4_off);/* Restore args */ -bpf_trampoline_restore_args_stack (image, ctx, func_frame_offset, nr_regs, regs_off); +bpf_trampoline_restore_args_stack (image, ctx, bpf_frame_size, nr_regs, regs_off);/* Restore TOC for 64-bit */ if (IS_ENABLED (CONFIG_PPC64_ELF_ABI_V2 ) && !IS_ENABLED (CONFIG_PPC_KERNEL_PCREL ))@@ -988,7 +965,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im/* Restore updated tail_call_cnt */ if (flags &BPF_TRAMP_F_TAIL_CALL_CTX ) -bpf_trampoline_restore_tail_call_cnt (image, ctx, func_frame_offset, r4_off); +bpf_trampoline_restore_tail_call_cnt (image, ctx, bpf_frame_size, r4_off);/* Reserve space to patch branch instruction to skip fexit progs */ if (ro_image)/* image is NULL for dummy pass */ @@ -1040,7 +1017,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imEMIT (PPC_RAW_LD (_R2, _R1,24 ));if (flags &BPF_TRAMP_F_SKIP_FRAME ) {/* Skip the traced function and return to parent */ -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));EMIT (PPC_RAW_MTLR (_R0));EMIT (PPC_RAW_BLR ());@@ -1048,13 +1025,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_imif (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) {EMIT (PPC_RAW_LL (_R0, _R1, alt_lr_off));EMIT (PPC_RAW_MTLR (_R0)); -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));
arch/powerpc/net/bpf_jit_comp.c static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im · line 1037
Quoted 10 lines
arch/powerpc/net/bpf_jit_comp.c +2
-2EMIT (PPC_RAW_BLR ()); }else { -EMIT (PPC_RAW_LL (_R0, _R1, bpf_frame_size +PPC_LR_STKOFF )); +EMIT (PPC_RAW_LL (_R0, _R1, retaddr_off));EMIT (PPC_RAW_MTCTR (_R0)); -EMIT (PPC_RAW_ADDI (_R1, _R1, func_frame_offset)); +EMIT (PPC_RAW_ADDI (_R1, _R1, bpf_frame_size));EMIT (PPC_RAW_LL (_R0, _R1,PPC_LR_STKOFF ));EMIT (PPC_RAW_MTLR (_R0));EMIT (PPC_RAW_BCTR ());
Ran the test: tailcalls/tailcall_bpf2bpf_hierarchy_fexit in a loop and issue is not seen. With out this patch, crash is observed.
Regards,
Venkat.
Re: [PATCH v2 2/5] powerpc64/bpf: fix the address returned by bpf_get_func_ip
In reply to Hari BathiniQuoted 28 lines · Hari Bathini
bpf_get_func_ip()helper function returns the address of the traced function. It relies on the IP address stored at ctx - 16 by the bpf trampoline. On 64-bit powerpc, this address is recovered from LR accounting for OOL trampoline. But the address stored here was off by 4-bytes. Ensure the address is the actual start of the traced function.Reported-by Abhishek Dubey <[email protected]>Fixes d243b62b7bd3 ("powerpc64/bpf: Add support for bpf trampolines")Signed-off-by Hari Bathini <[email protected]>---
* No changes since v1.
arch/powerpc/net/bpf_jit_comp.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-)arch/powerpc/net/bpf_jit_comp.c +0
-0@@ -786,8 +786,8 @@ static int __arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void*rw_imarch/powerpc/net/bpf_jit_comp.c +0
-0* [ reg argN ] * [ ... ] * regs_off [ reg_arg1 ] prog ctx
Quoted 1 line
context
prog ctx context/prog_ctx context/prog_ctx, to be in sync with tags below. please refer s390's field tagging
Quoted 28 lines
arch/powerpc/net/bpf_jit_comp.c +1
-2-* nregs_off [ args count ] -* ip_off [ traced function ] +* nregs_off [ args count ] ((u64 *)prog_ctx)[-1]
arch/powerpc/net/bpf_jit_comp.c +1
-0+* ip_off [ traced function ] ((u64 *)prog_ctx)[-2]
arch/powerpc/net/bpf_jit_comp.c +0
-0* [ ... ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] @@ -895,7 +895,7 @@ static int __arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void*rw_imbpf_trampoline_save_args(image, ctx,
func_frame_offset,nr_regs,regs_off);arch/powerpc/net/bpf_jit_comp.c +1
-1-/* Save our return address */ +/* Save our LR/return address */ EMIT (PPC_RAW_MFLR (_R3));if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE ))EMIT (PPC_RAW_STL (_R3, _R1, alt_lr_off));@@ -903,24 +903,29 @@ static int __arch_prepare_bpf_trampoline(struct
bpf_tramp_image*im, void*rw_imarch/powerpc/net/bpf_jit_comp.c +0
-2EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF ));/* -* Save ip address of the traced function. -* We could recover this from LR, but we will need to address for OOL trampoline,
arch/powerpc/net/bpf_jit_comp.c static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im · line 906
Quoted 2 lines
arch/powerpc/net/bpf_jit_comp.c +1
-1-* and optional GEP area. +* Get IP address of the traced function.
Get/Derive
Quoted 10 lines
arch/powerpc/net/bpf_jit_comp.c +2
-0+* In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR +* points to the instruction after the 'bl' instruction in the OOL stub.
arch/powerpc/net/bpf_jit_comp.c +1
-0+* Refer to ftrace_init_ool_stub() and bpf_arch_text_poke() for OOL stub
arch/powerpc/net/bpf_jit_comp.c +3
-0+* of kernel functions and bpf programs respectively. +* Recover kernel function/bpf program address from the unconditional +* branch instruction at the end of OOL stub. */ if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags &
Quoted 2 lines
BPF_TRAMP_F_IP_ARG) {
EMIT(PPC_RAW_LWZ(_R4, _R3, 4));
Please add comment what R4 points to; for easy referencing
Quoted 8 lines
arch/powerpc/net/bpf_jit_comp.c +0
-1EMIT (PPC_RAW_SLWI (_R4, _R4,6 ));EMIT (PPC_RAW_SRAWI (_R4, _R4,6 ));EMIT (PPC_RAW_ADD (_R3, _R3, _R4)); -EMIT (PPC_RAW_ADDI (_R3, _R3,4 )); }if (flags &BPF_TRAMP_F_IP_ARG )EMIT (PPC_RAW_STL (_R3, _R1, ip_off));
arch/powerpc/net/bpf_jit_comp.c static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im · line 931
Quoted 10 lines
arch/powerpc/net/bpf_jit_comp.c +3
-1-if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) +if (IS_ENABLED (CONFIG_PPC_FTRACE_OUT_OF_LINE )) {/* Fake our LR for unwind */ +EMIT (PPC_RAW_ADDI (_R3, _R3,4 ));EMIT (PPC_RAW_STL (_R3, _R1, bpf_frame_size +PPC_LR_STKOFF )); + }/* Save function arg count -- see bpf_get_func_arg_cnt() */ EMIT (PPC_RAW_LI (_R3, nr_regs));
-Abhishek
Re: [PATCH v2 3/5] powerpc64/bpf: use consistent tailcall offset in trampoline
In reply to Hari BathiniQuoted 4 lines · Hari Bathini
Ideally, the offset used to load the tail call info field and to find the pass by reference address for tail call field should be the same. But while setting up the tail call info in the trampoline, this was not followed. This can be misleading and can lead to unpredictable
Quoted 8 lines
results if and when
bpf_has_stack_frame()ends up returning true for trampoline frame. Since commit15513beeb673("powerpc64/bpf:
Movingtail_call_cntto bottom of frame") and commit2ed2d8f6fb38("powerpc64/bpf: Support tailcalls with subprogs") ensured tail call field is at the bottom of the stack frame for BPF programs as well as BPF trampoline, avoid relying onbpf_jit_stack_tailcallinfo_offset()andbpf_has_stack_frame()for trampoline frame and always calculate tail call field offset with reference to older frame.
It's good to add comment about padding field placed after tailcall_info in the trampoline stack layout. Visibly padding is following tailcall_info but tailcall_info is bottom-most field. Clear comment around this will be really helpful.
Quoted 100 lines
Fixes 2ed2d8f6fb38 ("powerpc64/bpf: Support tailcalls with subprogs")Signed-off-by Hari Bathini <[email protected]>---
Changes since v1:
* Fixed spelling error in changelog.
* Fixed a comment inbpf_trampoline_setup_tail_call_info().arch/powerpc/net/bpf_jit.h | 5 ----- arch/powerpc/net/bpf_jit_comp.c | 12 +++++------- arch/powerpc/net/bpf_jit_comp64.c | 5 ++++- 3 files changed, 9 insertions(+), 13 deletions(-)arch/powerpc/net/bpf_jit.h +0
-3@@ -81,9 +81,6 @@#ifdef CONFIG_PPC64 -/* for gpr non volatile registers BPG_REG_6 to 10 */ -#define BPF_PPC_STACK_SAVE (6 * 8) -/* If dummy pass (!image), account for maximum possible instructions */ #define PPC_LI64 (d, i) do { \arch/powerpc/net/bpf_jit.h +0
-2if (!image) \@@ -219,8 +216,6 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage,int pass,struct codegen_context *ctx,int insn_idx,int jmp_off,int dst_reg,u32 code); - -int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx);#endif #endif
arch/powerpc/net/bpf_jit_comp.c +0
-0@@ -642,15 +642,13 @@ static voidbpf_trampoline_setup_tail_call_info(u32 *image, struct
codegen_contearch/powerpc/net/bpf_jit_comp.c +2
-4int bpf_dummy_frame_size, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { - /* See Generated stack layout */ - int tailcallinfo_offset = BPF_PPC_TAILCALL; -/* * func_frame_offset = ...(1) * bpf_dummy_frame_size + trampoline_frame_size */ EMIT (PPC_RAW_LD (_R4 ,_R1 ,func_frame_offset )); -EMIT (PPC_RAW_LD (_R3, _R4, -tailcallinfo_offset)); +/* Refer to trampoline's Generated stack layout */ +EMIT (PPC_RAW_LD (_R3, _R4, -BPF_PPC_TAILCALL ));/* * Setting the tail_call_info in trampoline's frame @@ -658,7 +656,7 @@ static voidbpf_trampoline_setup_tail_call_info(u32 *image, struct
codegen_contearch/powerpc/net/bpf_jit_comp.c +0
-1*/ EMIT (PPC_RAW_CMPLWI (_R3,MAX_TAIL_CALL_CNT ));PPC_BCC_CONST_SHORT (COND_GT ,8 ); - EMIT(PPC_RAW_ADDI(_R3, _R4,
bpf_jit_stack_tailcallinfo_offset(ctx)));arch/powerpc/net/bpf_jit_comp.c +1
-0+EMIT (PPC_RAW_ADDI (_R3, _R4, -BPF_PPC_TAILCALL ));/* * From ...(1) above: * trampoline_frame_bottom = ...(2) @@ -666,14 +664,14 @@ static voidbpf_trampoline_setup_tail_call_info(u32 *image, struct
codegen_contearch/powerpc/net/bpf_jit_comp.c +1
-1* * Using ...(2) derived above: * trampoline_tail_call_info_offset = ...(3) -* trampoline_frame_bottom - tailcallinfo_offset +* trampoline_frame_bottom - BPF_PPC_TAILCALL * * From ...(3): * Use trampoline_tail_call_info_offset to write reference of main's * tail_call_info in trampoline frame. */ EMIT(PPC_RAW_STL(_R3, _R1, (func_frame_offset -
bpf_dummy_frame_size)arch/powerpc/net/bpf_jit_comp.c +1
-1- - tailcallinfo_offset)); + - BPF_PPC_TAILCALL)); }else {/* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT (PPC_RAW_LL (_R4, _R1, r4_off));arch/powerpc/net/bpf_jit_comp64.c +0
-0
arch/powerpc/net/bpf_jit_comp64.c line 46
Quoted 5 lines
arch/powerpc/net/bpf_jit_comp64.c +2
-0@@ -42,6 +42,9 @@* exception boundary. */ +/* BPF non-volatile registers save area size */ +#define BPF_PPC_STACK_SAVE (6*8)
Please maintain space beside *, suggested by Christophe in tailcall reviews
Quoted 13 lines
arch/powerpc/net/bpf_jit_comp64.c +2
-1+/* for bpf JIT code internal usage */ #define BPF_PPC_STACK_LOCALS 24/* @@ -148,7 +151,7 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) } } -int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx) +static int bpf_jit_stack_tailcallinfo_offset (struct codegen_context *ctx) {
Quoted 3 lines
return
bpf_jit_stack_local(ctx)+BPF_PPC_STACK_LOCALS+BPF_PPC_STACK_SAVE;
}
-Abhishek
Re: [PATCH v2 5/5] powerpc64/bpf: fix handling of BPF stack in exception callback
In reply to Hari BathiniQuoted 93 lines · Hari Bathini
Exception callback reuses the stack frame of exception boundary. When exception boundary and exception callback programs have different BPF stack depth, the current stack unwind in exception callback will fail. Adjust the stack frame size of exception callback, in its prologue, if its BPF stack depth is different from that of exception boundary.
Reported-by [email protected]Fixes 11d45eee9f42 ("powerpc64/bpf: Additional NVR handling for bpf_throw")Signed-off-by Hari Bathini <[email protected]>---
Changes since v1:
* Fixed incorrect use ofPPC_RAW_SUB()as pointed out by [email protected].arch/powerpc/net/bpf_jit_comp64.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-)arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -32,14 +32,15 @@* * [ prev sp ] <------------- * [ tail_call_info ] 8 | -* [ nv gpr save area ] 6*8 + (12*8) | +* [ nv gpr save area ] 6*8 | +* [ addl. nv gpr save area] (12*8) | <--- exception boundary/callback program
* [local_tmp_var] 24 |
* fp (r31) --> [ ebpf stack space ] upto 512 |
* [ frame header ] 32/112 |
* sp (r1) ---> [ stack pointer ] --------------
*
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*//* BPF non-volatile registers save area size */
arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -128,12 +129,13 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- * [ tail_call_info ] 8 - * [ nv gpr save area ] 6*8 + (12*8) + * [ nv gpr save area ] 6*8 + * [ addl. nv gpr save area] (12*8) <--- exception boundary/callbackprogram
* [local_tmp_var] 24
* [ unused red zone ] 224
*
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*/
static intbpf_jit_stack_local(struct codegen_context *ctx){arch/powerpc/net/bpf_jit_comp64.c +0
-4@@ -240,10 +242,6 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)if (bpf_has_stack_frame (ctx) && !ctx->exception_cb) {/* -* exception_cb uses boundary frame after stack walk. -* It can simply use redzone, this optimization reduces -* stack walk loop by one level. -* * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions arch/powerpc/net/bpf_jit_comp64.c +4
-0*/ @@ -287,6 +285,22 @@ void bpf_jit_build_prologue(u32 *image, structcodegen_context *ctx) * program(main prog) as third arg */ EMIT (PPC_RAW_MR (_R1, _R5)); +/* +* Exception callback reuses the stack frame of exception boundary. +* But BPF stack depth of exception callback and exception boundary +* don't have to be same. If BPF stack depth is different, adjust the
arch/powerpc/net/bpf_jit_comp64.c +1
-0+* stack frame size considering BPF stack depth of exception callback.
arch/powerpc/net/bpf_jit_comp64.c +5
-0+* The non-volatile register save area remains unchanged. These non- +* volatile registers are restored in exception callback's epilogue. +*/ +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ), _R5,0 )); + EMIT(PPC_RAW_SUB(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_1),
_R1));
arch/powerpc/net/bpf_jit_comp64.c void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) · line 301
Quoted 4 lines
arch/powerpc/net/bpf_jit_comp64.c +4
-0+EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + -BPF_PPC_EXC_STACKFRAME )); +EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_2 ), ctx->stack_size)); +PPC_BCC_CONST_SHORT (COND_EQ ,12 );
Can we avoid resizing when boundary_stack is greater? I think it's safe, NVR will be intact anyway and any reference within bounds of larger size. Any corner case possible?
Quoted 3 lines
arch/powerpc/net/bpf_jit_comp64.c +2
-0+EMIT (PPC_RAW_MR (_R1,bpf_to_ppc (TMP_REG_1 ))); + EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_EXC_STACKFRAME +
ctx->stack_size)));
Do we need to setup FP again? If I get it right, still the FP is pointing to older frame, any reference in callback will resolve to old frame.
arch/powerpc/net/bpf_jit_comp64.c
Quoted 3 lines
arch/powerpc/net/bpf_jit_comp64.c +0
-0}/*
-Abhishek
Re: [PATCH v2 5/5] powerpc64/bpf: fix handling of BPF stack in exception callback
In reply to Hari BathiniQuoted 99 lines · Hari Bathini
Exception callback reuses the stack frame of exception boundary. When exception boundary and exception callback programs have different BPF stack depth, the current stack unwind in exception callback will fail. Adjust the stack frame size of exception callback, in its prologue, if its BPF stack depth is different from that of exception boundary.
Reported-by [email protected]Fixes 11d45eee9f42 ("powerpc64/bpf: Additional NVR handling for bpf_throw")Signed-off-by Hari Bathini <[email protected]>---
Changes since v1:
* Fixed incorrect use ofPPC_RAW_SUB()as pointed out by [email protected].arch/powerpc/net/bpf_jit_comp64.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-)arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -32,14 +32,15 @@* * [ prev sp ] <------------- * [ tail_call_info ] 8 | -* [ nv gpr save area ] 6*8 + (12*8) | +* [ nv gpr save area ] 6*8 | +* [ addl. nv gpr save area] (12*8) | <--- exception boundary/callback program
* [local_tmp_var] 24 |
* fp (r31) --> [ ebpf stack space ] upto 512 |
* [ frame header ] 32/112 |
* sp (r1) ---> [ stack pointer ] --------------
*
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*//* BPF non-volatile registers save area size */
arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -128,12 +129,13 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- * [ tail_call_info ] 8 - * [ nv gpr save area ] 6*8 + (12*8) + * [ nv gpr save area ] 6*8 + * [ addl. nv gpr save area] (12*8) <--- exception boundary/callbackprogram
* [local_tmp_var] 24
* [ unused red zone ] 224
*
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*/
static intbpf_jit_stack_local(struct codegen_context *ctx){arch/powerpc/net/bpf_jit_comp64.c +0
-4@@ -240,10 +242,6 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)if (bpf_has_stack_frame (ctx) && !ctx->exception_cb) {/* -* exception_cb uses boundary frame after stack walk. -* It can simply use redzone, this optimization reduces -* stack walk loop by one level. -* * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions arch/powerpc/net/bpf_jit_comp64.c +4
-0*/ @@ -287,6 +285,22 @@ void bpf_jit_build_prologue(u32 *image, structcodegen_context *ctx) * program(main prog) as third arg */ EMIT (PPC_RAW_MR (_R1, _R5)); +/* +* Exception callback reuses the stack frame of exception boundary. +* But BPF stack depth of exception callback and exception boundary +* don't have to be same. If BPF stack depth is different, adjust the
arch/powerpc/net/bpf_jit_comp64.c +1
-0+* stack frame size considering BPF stack depth of exception callback.
arch/powerpc/net/bpf_jit_comp64.c +5
-0+* The non-volatile register save area remains unchanged. These non- +* volatile registers are restored in exception callback's epilogue. +*/ +EMIT (PPC_RAW_LD (bpf_to_ppc (TMP_REG_1 ), _R5,0 )); + EMIT(PPC_RAW_SUB(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_1),
_R1));arch/powerpc/net/bpf_jit_comp64.c +6
-0+EMIT (PPC_RAW_ADDI (bpf_to_ppc (TMP_REG_2 ),bpf_to_ppc (TMP_REG_2 ), + -BPF_PPC_EXC_STACKFRAME )); +EMIT (PPC_RAW_CMPLDI (bpf_to_ppc (TMP_REG_2 ), ctx->stack_size)); +PPC_BCC_CONST_SHORT (COND_EQ ,12 ); +EMIT (PPC_RAW_MR (_R1,bpf_to_ppc (TMP_REG_1 ))); + EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_EXC_STACKFRAME +
Quoted 1 line
ctx->stack_size)));
For the last comment here:
"Do we need to setup FP again? If I get it right, still the FP is pointing to older frame, any reference in callback will resolve to old frame."
Following inst takes care of above; it was missing for me in rebasing, my bad. EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
STACK_FRAME_MIN_SIZE +ctx->stack_size));
arch/powerpc/net/bpf_jit_comp64.c
Quoted 3 lines
arch/powerpc/net/bpf_jit_comp64.c +0
-0}/*
-Abhishek
Hari Bathini
maintainerRe: [PATCH v2 3/5] powerpc64/bpf: use consistent tailcall offset in trampoline
In reply to adubeyQuoted 15 lines · adubey
Quoted 4 lines · Hari Bathini
Ideally, the offset used to load the tail call info field and to find the pass by reference address for tail call field should be the same. But while setting up the tail call info in the trampoline, this was not followed. This can be misleading and can lead to unpredictable
Quoted 8 lines
results if and when
bpf_has_stack_frame()ends up returning true for trampoline frame. Since commit15513beeb673("powerpc64/bpf:
Movingtail_call_cntto bottom of frame") and commit2ed2d8f6fb38("powerpc64/bpf: Support tailcalls with subprogs") ensured tail call field is at the bottom of the stack frame for BPF programs as well as BPF trampoline, avoid relying onbpf_jit_stack_tailcallinfo_offset()andbpf_has_stack_frame()for trampoline frame and always calculate tail call field offset with reference to older frame.
Quoted 4 lines
It's good to add comment about padding field placed after
tailcall_infoin the trampoline stack layout. Visibly padding is followingtailcall_infobuttailcall_infois bottom-most field. Clear comment around this will be really helpful.
"Generated stack layout:" does capture that clearly. Don't find it relevant to explain it again in this context..
- Hari
Hari Bathini
maintainerRe: [PATCH v2 5/5] powerpc64/bpf: fix handling of BPF stack in exception callback
In reply to adubeyQuoted 96 lines · adubey
Quoted 95 lines · Hari Bathini
Exception callback reuses the stack frame of exception boundary. When exception boundary and exception callback programs have different BPF stack depth, the current stack unwind in exception callback will fail. Adjust the stack frame size of exception callback, in its prologue, if its BPF stack depth is different from that of exception boundary.
Reported-by [email protected]Closes:
https://lore.kernel.org/ bpf/2a310e86a59eb4c44c3ac9e5647814469d9c955580c9c0f1b3d9ca4a44717a34@mail.kernel.org/Fixes 11d45eee9f42 ("powerpc64/bpf: Additional NVR handling for bpf_throw")Signed-off-by Hari Bathini <[email protected]>---
Changes since v1:
* Fixed incorrect use ofPPC_RAW_SUB()as pointed out by [email protected].arch/powerpc/net/bpf_jit_comp64.c | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -32,14 +32,15 @@ * * [ prev sp ] <------------- * [ tail_call_info ] 8 | - * [ nv gpr save area ] 6*8 + (12*8) | + * [ nv gpr save area ] 6*8 | + * [ addl. nv gpr save area] (12*8) | <--- exceptionboundary/callback program
* [ local_tmp_var ] 24 |
* fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 |
* sp (r1) ---> [ stack pointer ] -------------- *
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*//* BPF non-volatile registers save area size */
arch/powerpc/net/bpf_jit_comp64.c +2
-1@@ -128,12 +129,13 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- * [ tail_call_info ] 8 - * [ nv gpr save area ] 6*8 + (12*8) + * [ nv gpr save area ] 6*8 + * [ addl. nv gpr save area] (12*8) <--- exception boundary/callback program
* [ local_tmp_var ] 24
* [ unused red zone ] 224
*
* Additional (12*8) in 'nv gpr save area' only in case of
- * exception boundary.
+ * exception boundary/callback.
*/
static intbpf_jit_stack_local(struct codegen_context *ctx){arch/powerpc/net/bpf_jit_comp64.c +0
-4@@ -240,10 +242,6 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) if (bpf_has_stack_frame(ctx) && !ctx->exception_cb) { /* -* exception_cb uses boundary frame after stack walk. -* It can simply use redzone, this optimization reduces -* stack walk loop by one level. -* * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions */
arch/powerpc/net/bpf_jit_comp64.c +2
-0@@ -287,6 +285,22 @@ void bpf_jit_build_prologue(u32 *image, structcodegen_context *ctx) * program(main prog) as third arg */ EMIT(PPC_RAW_MR(_R1, _R5)); + /* +* Exception callback reuses the stack frame of exception boundary.
+ * But BPF stack depth of exception callback and exception boundary
+ * don't have to be same. If BPF stack depth is different, adjust the
+ * stack frame size considering BPF stack depth of exception callback.
+ * The non-volatile register save area remains unchanged. These non-
+ * volatile registers are restored in exception callback's epilogue.
+ */
+EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R5, 0));
Quoted 8 lines
Quoted 6 lines
+
EMIT(PPC_RAW_SUB(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_1), _R1));
+ EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2),bpf_to_ppc(TMP_REG_2), + -BPF_PPC_EXC_STACKFRAME));
+EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), ctx->stack_size)); +PPC_BCC_CONST_SHORT(COND_EQ, 12);Can we avoid resizing when
boundary_stackis greater? I think it's safe, NVR will be intact anyway and any reference within bounds of larger size. Any corner case possible?
Did think about it but went with this approach to keep the tearing down of the stack consistent for all cases. Addressed the other comments and posted v3.
- Hari