Allocate bpf trampoline on bpf_prog_pack
Song Liu
reviewercover Allocate bpf trampoline on bpf_prog_pack
This set enables allocating bpf trampoline from bpf_prog_pack on x86. The majority of this work, however, is the refactoring of trampoline code. This is needed because we need to handle 4 archs and 2 users (trampoline and struct_ops).
1/8 is a dependency that is already applied to bpf tree. 2/8 through 7/8 refactors trampoline code. A few helpers are added. 8/8 finally let bpf trampoline on x86 use bpf_prog_pack.
Song Liu (8):
s390/bpf: Let arch_prepare_bpf_trampoline return program size
bpf: Let bpf_prog_pack_free handle any pointer
bpf: Adjust argument names of arch_prepare_bpf_trampoline()
bpf: Add helpers for trampoline image management bpf, x86: Adjust arch_prepare_bpf_trampoline return value
bpf: Add arch_bpf_trampoline_size()
bpf: Use arch_bpf_trampoline_size for trampoline x86, bpf: Use bpf_prog_pack for bpf trampoline
arch/arm64/net/bpf_jit_comp.c | 62 +++++++++++----- arch/riscv/net/bpf_jit_comp64.c | 24 ++++-- arch/s390/net/bpf_jit_comp.c | 52 +++++++------ arch/x86/net/bpf_jit_comp.c | 127 +++++++++++++++++++++++++------- include/linux/bpf.h | 12 ++- include/linux/filter.h | 2 +- kernel/bpf/bpf_struct_ops.c | 12 ++- kernel/bpf/core.c | 21 +++--- kernel/bpf/dispatcher.c | 5 +- kernel/bpf/trampoline.c | 89 ++++++++++++++++------ 10 files changed, 285 insertions(+), 121 deletions(-)
--
2.34.1
Song Liu
reviewerPATCH 1/8 s390/bpf: Let arch_prepare_bpf_trampoline return program size
arch_prepare_bpf_trampoline() for s390 currently returns 0 on success. This is not a problem for regular trampoline. However, struct_ops relies on the return value to advance "image" pointer:
bpf_struct_ops_map_update_elem() {
...
for_each_member(i, t, member) {
...
err = bpf_struct_ops_prepare_trampoline();
...
image += err;
}
}When arch_prepare_bpf_trampoline returns 0 on success, all members of the struct_ops will point to the same trampoline (the last one).
Fix this by returning the program size in arch_prepare_bpf_trampoline (on success). This is the same behavior as other architectures.
--- arch/s390/net/bpf_jit_comp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/s390/net/bpf_jit_comp.c +1 -1
@@ -2513,7 +2513,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,return -E2BIG ; } -return ret; +return tjit.common.prg; }bool bpf_jit_supports_subprog_tailcalls (void )
Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 2/8 bpf: Let bpf_prog_pack_free handle any pointer
In reply to Song LiuCurrently, bpf_prog_pack_free only can only free pointer to struct bpf_binary_header, which is not flexible. Add a size argument to bpf_prog_pack_free so that it can handle any pointer.
--- include/linux/filter.h | 2 +- kernel/bpf/core.c | 21 ++++++++++----------- kernel/bpf/dispatcher.c | 5 +---- 3 files changed, 12 insertions(+), 16 deletions(-)
include/linux/filter.h +1 -1
@@ -1073,7 +1073,7 @@ struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);void *bpf_prog_pack_alloc (u32 size,bpf_jit_fill_hole_t bpf_fill_ill_insns); -void bpf_prog_pack_free (struct bpf_binary_header *hdr); +void bpf_prog_pack_free (void *ptr,u32 size);static inline bool bpf_prog_kallsyms_verify_off (const struct bpf_prog *fp) {
kernel/bpf/core.c +10 -11
@@ -928,20 +928,20 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)return ptr; } -void bpf_prog_pack_free (struct bpf_binary_header *hdr) +void bpf_prog_pack_free (void *ptr,u32 size) {struct bpf_prog_pack *pack =NULL , *tmp;unsigned int nbits;unsigned long pos;mutex_lock (&pack_mutex); -if (hdr->size >BPF_PROG_PACK_SIZE ) { -bpf_jit_free_exec (hdr); +if (size >BPF_PROG_PACK_SIZE ) { +bpf_jit_free_exec (ptr); goto out; }list_for_each_entry (tmp, &pack_list, list) { -if ((void *)hdr >= tmp->ptr && (tmp->ptr +BPF_PROG_PACK_SIZE ) > (void *)hdr) { +if (ptr >= tmp->ptr && (tmp->ptr +BPF_PROG_PACK_SIZE ) > ptr) { pack = tmp;break ; }
@@ -950,10 +950,10 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr)if (WARN_ONCE (!pack,"bpf_prog_pack bug\n" )) goto out; - nbits =BPF_PROG_SIZE_TO_NBITS (hdr->size); - pos = ((unsigned long )hdr - (unsigned long )pack->ptr) >>BPF_PROG_CHUNK_SHIFT ; + nbits =BPF_PROG_SIZE_TO_NBITS (size); + pos = ((unsigned long )ptr - (unsigned long )pack->ptr) >>BPF_PROG_CHUNK_SHIFT ; -WARN_ONCE (bpf_arch_text_invalidate (hdr, hdr->size), +WARN_ONCE (bpf_arch_text_invalidate (ptr, size),"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n" );bitmap_clear (pack->bitmap, pos, nbits);
@@ -1100,8 +1100,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, *rw_header =kvmalloc (size,GFP_KERNEL );if (!*rw_header) { -bpf_arch_text_copy (&ro_header->size, &size,sizeof (size)); -bpf_prog_pack_free (ro_header); +bpf_prog_pack_free (ro_header, size);bpf_jit_uncharge_modmem (size);return NULL ; }
@@ -1132,7 +1131,7 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,kvfree (rw_header);if (IS_ERR (ptr)) { -bpf_prog_pack_free (ro_header); +bpf_prog_pack_free (ro_header, ro_header->size);return PTR_ERR (ptr); }return 0 ;
@@ -1153,7 +1152,7 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, {u32 size = ro_header->size; -bpf_prog_pack_free (ro_header); +bpf_prog_pack_free (ro_header, size);kvfree (rw_header);bpf_jit_uncharge_modmem (size); }
kernel/bpf/dispatcher.c +1 -4
@@ -150,10 +150,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, goto out; d->rw_image =bpf_jit_alloc_exec (PAGE_SIZE );if (!d->rw_image) { -u32 size =PAGE_SIZE ; - -bpf_arch_text_copy (d->image, &size,sizeof (size)); -bpf_prog_pack_free ((struct bpf_binary_header *)d->image); +bpf_prog_pack_free (d->image,PAGE_SIZE ); d->image =NULL ; goto out; }
Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 3/8 bpf: Adjust argument names of arch_prepare_bpf_trampoline()
In reply to Song LiuWe are using "im" for "struct bpf_tramp_image" and "tr" for "struct bpf_trampoline" in most of the code base. The only exception is the prototype and fallback version of arch_prepare_bpf_trampoline(). Update them to match the rest of the code base.
We mix "orig_call" and "func_addr" for the argument in different versions of arch_prepare_bpf_trampoline(). s/orig_call/func_addr/g so they match.
--- arch/arm64/net/bpf_jit_comp.c | 10 +++++----- include/linux/bpf.h | 4 ++-- kernel/bpf/trampoline.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-)
arch/arm64/net/bpf_jit_comp.c +5 -5
@@ -1828,7 +1828,7 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nregs)* */ static int prepare_trampoline (struct jit_ctx *ctx,struct bpf_tramp_image *im, -struct bpf_tramp_links *tlinks,void *orig_call, +struct bpf_tramp_links *tlinks,void *func_addr,int nregs,u32 flags) {int i;
@@ -1926,7 +1926,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,if (flags &BPF_TRAMP_F_IP_ARG ) {/* save ip address of the traced function */ -emit_addr_mov_i64 (A64_R (10 ), (const u64 )orig_call, ctx); +emit_addr_mov_i64 (A64_R (10 ), (const u64 )func_addr, ctx);emit (A64_STR64I (A64_R (10 ),A64_SP , ip_off), ctx); }
@@ -2029,7 +2029,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags,struct bpf_tramp_links *tlinks, -void *orig_call) +void *func_addr) {int i, ret;int nregs = m->nr_args;
@@ -2050,7 +2050,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,if (nregs >8 )return -ENOTSUPP ; - ret =prepare_trampoline (&ctx, im, tlinks, orig_call, nregs, flags); + ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags);if (ret <0 )return ret;
@@ -2061,7 +2061,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, ctx.idx =0 ;jit_fill_hole (image, (unsigned int )(image_end - image)); - ret =prepare_trampoline (&ctx, im, tlinks, orig_call, nregs, flags); + ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags);if (ret >0 &&validate_code (&ctx) <0 ) ret = -EINVAL ;
include/linux/bpf.h +2 -2
@@ -1079,10 +1079,10 @@ struct bpf_tramp_run_ctx;* fexit = a set of program to run after original function */ struct bpf_tramp_image ; -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end, +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags,struct bpf_tramp_links *tlinks, -void *orig_call); +void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
kernel/bpf/trampoline.c +2 -2
@@ -1032,10 +1032,10 @@ bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) }int __weak -arch_prepare_bpf_trampoline (struct bpf_tramp_image *tr,void *image,void *image_end, +arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags,struct bpf_tramp_links *tlinks, -void *orig_call) +void *func_addr) {return -ENOTSUPP ; }
Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 4/8 bpf: Add helpers for trampoline image management
In reply to Song LiuAs BPF trampoline of different archs moves from bpf_jit_[alloc|free]_exec() to bpf_prog_pack_[alloc|free](), we need to use different _alloc, _free for different archs during the transition. Add the following helpers for this transition:
void *arch_alloc_bpf_trampoline(int size);
void arch_free_bpf_trampoline(void *image, int size); void arch_protect_bpf_trampoline(void *image, int size); void arch_unprotect_bpf_trampoline(void *image, int size);
The fallback version of these helpers require size <= PAGE_SIZE, but they are only called with size == PAGE_SIZE. They will be called with size < PAGE_SIZE when arch_bpf_trampoline_size() helper is introduced later.
--- include/linux/bpf.h | 5 +++++ kernel/bpf/bpf_struct_ops.c | 12 +++++----- kernel/bpf/trampoline.c | 44 +++++++++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 14 deletions(-)
include/linux/bpf.h +5 -0
@@ -1083,6 +1083,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr); +void *arch_alloc_bpf_trampoline (int size); +void arch_free_bpf_trampoline (void *image,int size); +void arch_protect_bpf_trampoline (void *image,int size); +void arch_unprotect_bpf_trampoline (void *image,int size); + u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
kernel/bpf/bpf_struct_ops.c +5 -7
@@ -515,7 +515,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,if (err) goto reset_unlock; } -set_memory_rox ((long )st_map->image,1 ); +arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE );/* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem().
@@ -524,7 +524,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } -set_memory_rox ((long )st_map->image,1 ); +arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE ); err = st_ops->reg (kdata);if (likely (!err)) {/* This refcnt increment on the map here after
@@ -547,8 +547,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,* there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ -set_memory_nx ((long )st_map->image,1 ); -set_memory_rw ((long )st_map->image,1 ); +arch_unprotect_bpf_trampoline (st_map->image,PAGE_SIZE ); reset_unlock:bpf_struct_ops_map_put_progs (st_map);
@@ -616,7 +615,7 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map)bpf_struct_ops_map_put_progs (st_map);bpf_map_area_free (st_map->links);if (st_map->image) { -bpf_jit_free_exec (st_map->image); +arch_free_bpf_trampoline (st_map->image,PAGE_SIZE );bpf_jit_uncharge_modmem (PAGE_SIZE ); }bpf_map_area_free (st_map->uvalue);
@@ -691,7 +690,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)return ERR_PTR (ret); } - st_map->image =bpf_jit_alloc_exec (PAGE_SIZE ); + st_map->image =arch_alloc_bpf_trampoline (PAGE_SIZE );if (!st_map->image) {/* __bpf_struct_ops_map_free() uses st_map->image as flag * for "charged or not". In this case, we need to unchange
@@ -711,7 +710,6 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) }mutex_init (&st_map->lock); -set_vm_flush_reset_perms (st_map->image);bpf_map_init_from_attr (map, attr);return map;
kernel/bpf/trampoline.c +37 -7
82 lines not on this page · raw message
Signature · 1 line
2.34.1
Re: [PATCH bpf-next 4/8] bpf: Add helpers for trampoline image management
Quoted 14 lines · Song Liu
As BPF trampoline of different archs moves from bpf_jit_[alloc|free]_exec() to bpf_prog_pack_[alloc|free](), we need to use different
_alloc,_freefor different archs during the transition. Add the following helpers for this transition:void *arch_alloc_bpf_trampoline(int size);
voidarch_free_bpf_trampoline(void *image, int size); voidarch_protect_bpf_trampoline(void *image, int size); voidarch_unprotect_bpf_trampoline(void *image, int size);
Quoted 8 lines
The fallback version of these helpers require size <=
PAGE_SIZE, but they are only called with size ==PAGE_SIZE. They will be called with size <PAGE_SIZEwhenarch_bpf_trampoline_size()helper is introduced later.Signed-off-by Song Liu <[email protected]>---
include/linux/bpf.h| 5 +++++kernel/bpf/bpf_struct_ops.c| 12 +++++-----
I just found that we need similar changes in net/bpf/bpf_dummy_struct_ops.c.
Added it to the next version.
Song
Song Liu
reviewerPATCH 5/8 bpf, x86: Adjust arch_prepare_bpf_trampoline return value
In reply to Song Liux86's implementation of arch_prepare_bpf_trampoline() requires BPF_INSN_SAFETY buffer space between end of program and image_end. OTOH, the return value does not include BPF_INSN_SAFETY. This doesn't cause any real issue at the moment. However, "image" of size retval is not enough for arch_prepare_bpf_trampoline(). This will cause confusion when we introduce a new helper arch_bpf_trampoline_size(). To avoid future confusion, adjust the return value to include BPF_INSN_SAFETY.
--- arch/x86/net/bpf_jit_comp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/net/bpf_jit_comp.c +1 -1
@@ -2671,7 +2671,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i ret = -EFAULT ; goto cleanup; } - ret = prog - (u8 *)image; + ret = prog - (u8 *)image +BPF_INSN_SAFETY ; cleanup:kfree (branches);
Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 6/8 bpf: Add arch_bpf_trampoline_size()
In reply to Song LiuThis helper will be used to calculate the size of the trampoline before allocating the memory.
--- arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 6 ++++ 6 files changed, 131 insertions(+), 49 deletions(-)
arch/arm64/net/bpf_jit_comp.c +41 -15
@@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,return ctx->idx; } -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, -void *image_end,const struct btf_func_model *m, -u32 flags,struct bpf_tramp_links *tlinks, -void *func_addr) +static int btf_func_model_nregs (const struct btf_func_model *m) { -int i, ret;int nregs = m->nr_args; -int max_insns = ((long )image_end - (long )image) /AARCH64_INSN_SIZE ; -struct jit_ctx ctx = { - .image =NULL , - .idx =0 , - }; +int i;/* extra registers needed for struct argument */ for (i =0 ; i <MAX_BPF_FUNC_ARGS ; i++) {
@@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] +7 ) /8 -1 ; } +return nregs; +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct jit_ctx ctx = { + .image =NULL , + .idx =0 , + }; +struct bpf_tramp_image im; +int nregs, ret; + + nregs =btf_func_model_nregs (m);/* the first 8 registers are used for arguments */ if (nregs >8 )return -ENOTSUPP ; - ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags); + ret =prepare_trampoline (&ctx, &im, tlinks, func_addr, nregs, flags);if (ret <0 )return ret; -if (ret > max_insns) -return -EFBIG ; +return ret <0 ? ret : ret *AARCH64_INSN_SIZE ; +} - ctx.image = image; - ctx.idx =0 ; +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, +void *image_end,const struct btf_func_model *m, +u32 flags,struct bpf_tramp_links *tlinks, +void *func_addr) +{ +int ret, nregs; +struct jit_ctx ctx = { + .image = image, + .idx =0 , + }; + + nregs =btf_func_model_nregs (m); +/* the first 8 registers are used for arguments */ +if (nregs >8 ) +return -ENOTSUPP ; + + ret =arch_bpf_trampoline_size (m, flags, tlinks, func_addr); +if (ret <0 ) +return ret; + +if (ret > ((long )image_end - (long )image)) +return -EFBIG ;jit_fill_hole (image, (unsigned int )(image_end - image)); ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags);
arch/riscv/net/bpf_jit_comp64.c +16 -8
@@ -1024,6 +1024,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,return ret; } +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +struct rv_jit_context ctx; + + ctx.ninsns =0 ; + ctx.insns =NULL ; + ctx.ro_insns =NULL ; + ret =__arch_prepare_bpf_trampoline (&im, m, tlinks, func_addr, flags, &ctx); + +return ret <0 ? ret :ninsns_rvoff (ctx->ninsns); +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks,
@@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,int ret;struct rv_jit_context ctx; - ctx.ninsns =0 ; - ctx.insns =NULL ; - ctx.ro_insns =NULL ; - ret =__arch_prepare_bpf_trampoline (im, m, tlinks, func_addr, flags, &ctx); -if (ret <0 ) -return ret; - -if (ninsns_rvoff (ret) > (long )image_end - (long )image) + ret =arch_bpf_trampoline_size (im, m, flags, tlinks, func_addr); +if (ret > (long )image_end - (long )image)return -EFBIG ; ctx.ninsns =0 ;
arch/s390/net/bpf_jit_comp.c +30 -22
66 lines not on this page · raw message
arch/x86/net/bpf_jit_comp.c +36 -4
@@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,* add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, -const struct btf_func_model *m,u32 flags, -struct bpf_tramp_links *tlinks, -void *func_addr) +static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) {int i, ret, nr_regs = m->nr_args, stack_size =0 ;int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off;
@@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *ireturn ret; } +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) +{ +return __arch_prepare_bpf_trampoline (im, image, image_end, m, flags, tlinks, func_addr); +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +void *image; +int ret; + +/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). +* This will NOT cause fragmentation in direct map, as we do not +* call set_memory_*() on this buffer. +*/ + image =bpf_jit_alloc_exec (PAGE_SIZE ); +if (!image) +return -ENOMEM ; + + ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , m, flags, + tlinks, func_addr); +bpf_jit_free_exec (image); +if (ret <0 ) +return ret; + +return ret; +} +static int emit_bpf_dispatcher (u8 **pprog,int a,int b,s64 *progs,u8 *image,u8 *buf) {u8 *jg_reloc, *prog = *pprog;
include/linux/bpf.h +2 -0
@@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size);void arch_free_bpf_trampoline (void *image,int size);void arch_protect_bpf_trampoline (void *image,int size);void arch_unprotect_bpf_trampoline (void *image,int size); +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx);
kernel/bpf/trampoline.c +6 -0
@@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size)set_memory_rw ((long )image,1 ); } +int __weak arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +return -ENOTSUPP ; +} +static int __init init_trampolines (void ) {int i;
Signature · 1 line
2.34.1
Re: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
Hi Song,
Quoted 81 lines · Song Liu
This helper will be used to calculate the size of the trampoline before allocating the memory.
Signed-off-by Song Liu <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 6 ++++ 6 files changed, 131 insertions(+), 49 deletions(-)arch/arm64/net/bpf_jit_comp.c +28
-15@@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,return ctx->idx; } -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, -void *image_end,const struct btf_func_model *m, -u32 flags,struct bpf_tramp_links *tlinks, -void *func_addr) +static int btf_func_model_nregs (const struct btf_func_model *m) { -int i, ret;int nregs = m->nr_args; -int max_insns = ((long )image_end - (long )image) /AARCH64_INSN_SIZE ; -struct jit_ctx ctx = { - .image =NULL , - .idx =0 , - }; +int i;/* extra registers needed for struct argument */ for (i =0 ; i <MAX_BPF_FUNC_ARGS ; i++) {@@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] +7 ) /8 -1 ; } +return nregs; +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct jit_ctx ctx = { + .image =NULL , + .idx =0 , + }; +struct bpf_tramp_image im; +int nregs, ret; + + nregs =btf_func_model_nregs (m);/* the first 8 registers are used for arguments */ if (nregs >8 )return -ENOTSUPP ; - ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags); + ret =prepare_trampoline (&ctx, &im, tlinks, func_addr, nregs, flags);if (ret <0 )return ret; -if (ret > max_insns) -return -EFBIG ; +return ret <0 ? ret : ret *AARCH64_INSN_SIZE ; +} - ctx.image = image; - ctx.idx =0 ; +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, +void *image_end,const struct btf_func_model *m, +u32 flags,struct bpf_tramp_links *tlinks, +void *func_addr) +{ +int ret, nregs; +struct jit_ctx ctx = { + .image = image, + .idx =0 ,
arch/arm64/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, · line 2084
Quoted 10 lines
arch/arm64/net/bpf_jit_comp.c +10
-0+ }; + + nregs =btf_func_model_nregs (m); +/* the first 8 registers are used for arguments */ +if (nregs >8 ) +return -ENOTSUPP ; + + ret =arch_bpf_trampoline_size (m, flags, tlinks, func_addr); +if (ret <0 ) +return ret;
Since arch_bpf_trampoline_size was already called before the trampoline image was allocated, it seems this call to arch_bpf_trampoline_size is unnecessary. If this call can be omitted, we can avoid one less dry run.
Quoted 6 lines
arch/arm64/net/bpf_jit_comp.c +3
-0+ +if (ret > ((long )image_end - (long )image)) +return -EFBIG ;jit_fill_hole (image, (unsigned int )(image_end - image)); ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags);
[...]
Re: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
Hi Kuohai,
Quoted 9 lines · Xu Kuohai
Hi Song,
Quoted 4 lines · Song Liu
This helper will be used to calculate the size of the trampoline before allocating the memory.
Signed-off-by Song Liu <[email protected]>---
[...]
Quoted 5 lines
Quoted 5 lines
+
+ nregs =btf_func_model_nregs(m);
+ /* the first 8 registers are used for arguments */
+ if (nregs > 8)
+ return-ENOTSUPP;
Quoted 8 lines
Quoted 4 lines
+
+ ret =arch_bpf_trampoline_size(m, flags, tlinks, func_addr);
+ if (ret < 0)
+ return ret;Since
arch_bpf_trampoline_sizewas already called before the trampoline image was allocated, it seems this call toarch_bpf_trampoline_sizeis unnecessary. If this call can be omitted, we can avoid one less dry run.
Indeed. This set doesn't call arch_bpf_trampoline_size() from struct_ops. But we can add that and then remove the _size() call here.
Thanks,
Song
Quoted 10 lines
Quoted 6 lines
+
+ if (ret > ((long)image_end - (long)image))+ return -EFBIG; jit_fill_hole(image, (unsigned int)(image_end - image));ret =
prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);[...]
Pu Lehui
maintainerRe: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
In reply to Song LiuQuoted 125 lines · Song Liu
This helper will be used to calculate the size of the trampoline before allocating the memory.
Signed-off-by Song Liu <[email protected]>--- arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++--------- arch/riscv/net/bpf_jit_comp64.c | 24 +++++++++----- arch/s390/net/bpf_jit_comp.c | 52 +++++++++++++++++------------- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++--- include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 6 ++++ 6 files changed, 131 insertions(+), 49 deletions(-)arch/arm64/net/bpf_jit_comp.c +41
-15@@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,return ctx->idx; } -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, -void *image_end,const struct btf_func_model *m, -u32 flags,struct bpf_tramp_links *tlinks, -void *func_addr) +static int btf_func_model_nregs (const struct btf_func_model *m) { -int i, ret;int nregs = m->nr_args; -int max_insns = ((long )image_end - (long )image) /AARCH64_INSN_SIZE ; -struct jit_ctx ctx = { - .image =NULL , - .idx =0 , - }; +int i;/* extra registers needed for struct argument */ for (i =0 ; i <MAX_BPF_FUNC_ARGS ; i++) {@@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] +7 ) /8 -1 ; } +return nregs; +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct jit_ctx ctx = { + .image =NULL , + .idx =0 , + }; +struct bpf_tramp_image im; +int nregs, ret; + + nregs =btf_func_model_nregs (m);/* the first 8 registers are used for arguments */ if (nregs >8 )return -ENOTSUPP ; - ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags); + ret =prepare_trampoline (&ctx, &im, tlinks, func_addr, nregs, flags);if (ret <0 )return ret; -if (ret > max_insns) -return -EFBIG ; +return ret <0 ? ret : ret *AARCH64_INSN_SIZE ; +} - ctx.image = image; - ctx.idx =0 ; +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image, +void *image_end,const struct btf_func_model *m, +u32 flags,struct bpf_tramp_links *tlinks, +void *func_addr) +{ +int ret, nregs; +struct jit_ctx ctx = { + .image = image, + .idx =0 , + }; + + nregs =btf_func_model_nregs (m); +/* the first 8 registers are used for arguments */ +if (nregs >8 ) +return -ENOTSUPP ; + + ret =arch_bpf_trampoline_size (m, flags, tlinks, func_addr); +if (ret <0 ) +return ret; + +if (ret > ((long )image_end - (long )image)) +return -EFBIG ;jit_fill_hole (image, (unsigned int )(image_end - image)); ret =prepare_trampoline (&ctx, im, tlinks, func_addr, nregs, flags);arch/riscv/net/bpf_jit_comp64.c +14
-0@@ -1024,6 +1024,20 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,return ret; } +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +struct rv_jit_context ctx; + + ctx.ninsns =0 ; + ctx.insns =NULL ; + ctx.ro_insns =NULL ; + ret =__arch_prepare_bpf_trampoline (&im, m, tlinks, func_addr, flags, &ctx); + +return ret <0 ? ret :ninsns_rvoff (ctx->ninsns); +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks,@@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,int ret;struct rv_jit_context ctx;
arch/riscv/net/bpf_jit_comp64.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, · line 1049
Quoted 10 lines
arch/riscv/net/bpf_jit_comp64.c +1
-8- ctx.ninsns =0 ; - ctx.insns =NULL ; - ctx.ro_insns =NULL ; - ret =__arch_prepare_bpf_trampoline (im, m, tlinks, func_addr, flags, &ctx); -if (ret <0 ) -return ret; - -if (ninsns_rvoff (ret) > (long )image_end - (long )image) + ret =arch_bpf_trampoline_size (im, m, flags, tlinks, func_addr);
Hi Song, there is missing check for negative return values.
Quoted 164 lines
arch/riscv/net/bpf_jit_comp64.c +1
-0+if (ret > (long )image_end - (long )image)return -EFBIG ; ctx.ninsns =0 ;arch/s390/net/bpf_jit_comp.c +30
-22@@ -2483,6 +2483,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,return 0 ; } +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *orig_call) +{ +struct bpf_tramp_image im; +struct bpf_tramp_jit tjit; +int ret; + +memset (&tjit,0 ,sizeof (tjit)); + + ret =__arch_prepare_bpf_trampoline (&im, &tjit, m, flags, + tlinks, orig_call); + +return ret <0 ? ret : tjit.common.prg; +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks,@@ -2490,30 +2505,23 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, {struct bpf_tramp_jit tjit;int ret; -int i; -for (i =0 ; i <2 ; i++) { -if (i ==0 ) { -/* Compute offsets, check whether the code fits. */ -memset (&tjit,0 ,sizeof (tjit)); - }else { -/* Generate the code. */ - tjit.common.prg =0 ; - tjit.common.prg_buf = image; - } - ret =__arch_prepare_bpf_trampoline (im, &tjit, m, flags, - tlinks, func_addr); -if (ret <0 ) -return ret; -if (tjit.common.prg > (char *)image_end - (char *)image) -/* -* Use the same error code as for exceeding -* BPF_MAX_TRAMP_LINKS. -*/ -return -E2BIG ; - } + ret =arch_bpf_trampoline_size (m, flags, tlinks, func_addr); +if (ret <0 ) +return ret; + +if (ret > (char *)image_end - (char *)image) +/* +* Use the same error code as for exceeding +* BPF_MAX_TRAMP_LINKS. +*/ +return -E2BIG ; -return tjit.common.prg; + tjit.common.prg =0 ; + tjit.common.prg_buf = image; + ret =__arch_prepare_bpf_trampoline (im, &tjit, m, flags, + tlinks, func_addr); +return ret <0 ? ret : tjit.common.prg; }bool bpf_jit_supports_subprog_tailcalls (void )arch/x86/net/bpf_jit_comp.c +36
-4@@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,* add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, -const struct btf_func_model *m,u32 flags, -struct bpf_tramp_links *tlinks, -void *func_addr) +static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) {int i, ret, nr_regs = m->nr_args, stack_size =0 ;int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off;@@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *ireturn ret; } +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) +{ +return __arch_prepare_bpf_trampoline (im, image, image_end, m, flags, tlinks, func_addr); +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +void *image; +int ret; + +/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). +* This will NOT cause fragmentation in direct map, as we do not +* call set_memory_*() on this buffer. +*/ + image =bpf_jit_alloc_exec (PAGE_SIZE ); +if (!image) +return -ENOMEM ; + + ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , m, flags, + tlinks, func_addr); +bpf_jit_free_exec (image); +if (ret <0 ) +return ret; + +return ret; +} +static int emit_bpf_dispatcher (u8 **pprog,int a,int b,s64 *progs,u8 *image,u8 *buf) {u8 *jg_reloc, *prog = *pprog;include/linux/bpf.h +2
-0@@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size);void arch_free_bpf_trampoline (void *image,int size);void arch_protect_bpf_trampoline (void *image,int size);void arch_unprotect_bpf_trampoline (void *image,int size); +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx);kernel/bpf/trampoline.c +6
-0@@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size)set_memory_rw ((long )image,1 ); } +int __weak arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +return -ENOTSUPP ; +} +static int __init init_trampolines (void ) {int i;
Song Liu
reviewerRe: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
Quoted 1 line · Pu Lehui
[...]
Quoted 21 lines
Quoted 21 lines
arch/riscv/net/bpf_jit_comp64.c +14
-0+int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +struct rv_jit_context ctx; + + ctx.ninsns =0 ; + ctx.insns =NULL ; + ctx.ro_insns =NULL ; + ret =__arch_prepare_bpf_trampoline (&im, m, tlinks, func_addr, flags, &ctx); + +return ret <0 ? ret :ninsns_rvoff (ctx->ninsns); +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks,@@ -1032,14 +1046,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,int ret;struct rv_jit_context ctx;
Quoted 11 lines
arch/riscv/net/bpf_jit_comp64.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, · line 1049
Quoted 9 lines
arch/riscv/net/bpf_jit_comp64.c +1
-8- ctx.ninsns =0 ; - ctx.insns =NULL ; - ctx.ro_insns =NULL ; - ret =__arch_prepare_bpf_trampoline (im, m, tlinks, func_addr, flags, &ctx); -if (ret <0 ) -return ret; - -if (ninsns_rvoff (ret) > (long )image_end - (long )image) + ret =arch_bpf_trampoline_size (im, m, flags, tlinks, func_addr);Hi Song, there is missing check for negative return values.
Thanks! I will fix it in the next version.
Song
Quoted 11 lines
Quoted 11 lines
arch/riscv/net/bpf_jit_comp64.c +1
-0+if (ret > (long )image_end - (long )image)return -EFBIG ; ctx.ninsns =0 ;arch/s390/net/bpf_jit_comp.c +0
-0@@ -2483,6 +2483,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,return 0 ; }
[...]
Re: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
In reply to Song LiuOn Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote:
SNIP
Quoted 8 lines
bool
bpf_jit_supports_subprog_tailcalls(void)arch/x86/net/bpf_jit_comp.c +0
-0@@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,* add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller
arch/x86/net/bpf_jit_comp.c static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, · line 2429
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +4
-4*/ -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, -const struct btf_func_model *m,u32 flags, -struct bpf_tramp_links *tlinks, -void *func_addr) +static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) {
hum, I dont understand what's the __arch_prepare_bpf_trampoline for, could we have just the arch_prepare_bpf_trampoline?
jirka
Quoted 75 lines
arch/x86/net/bpf_jit_comp.c +32
-0int i, ret, nr_regs = m->nr_args, stack_size =0 ;int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off;@@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *ireturn ret; } +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) +{ +return __arch_prepare_bpf_trampoline (im, image, image_end, m, flags, tlinks, func_addr); +} + +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +void *image; +int ret; + +/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). +* This will NOT cause fragmentation in direct map, as we do not +* call set_memory_*() on this buffer. +*/ + image =bpf_jit_alloc_exec (PAGE_SIZE ); +if (!image) +return -ENOMEM ; + + ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , m, flags, + tlinks, func_addr); +bpf_jit_free_exec (image); +if (ret <0 ) +return ret; + +return ret; +} +static int emit_bpf_dispatcher (u8 **pprog,int a,int b,s64 *progs,u8 *image,u8 *buf) {u8 *jg_reloc, *prog = *pprog;include/linux/bpf.h +2
-0@@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size);void arch_free_bpf_trampoline (void *image,int size);void arch_protect_bpf_trampoline (void *image,int size);void arch_unprotect_bpf_trampoline (void *image,int size); +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx);kernel/bpf/trampoline.c +6
-0@@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size)set_memory_rw ((long )image,1 ); } +int __weak arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +return -ENOTSUPP ; +} +static int __init init_trampolines (void ) {int i;Signature · 1 line
2.34.1
Re: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
Quoted 17 lines · Jiri Olsa
On Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote:
SNIP
Quoted 13 lines
bool
bpf_jit_supports_subprog_tailcalls(void)arch/x86/net/bpf_jit_comp.c +0
-4@@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,* add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, -const struct btf_func_model *m,u32 flags, -struct bpf_tramp_links *tlinks, -void *func_addr)
Quoted 8 lines
arch/x86/net/bpf_jit_comp.c static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, · line 2429
Quoted 5 lines
arch/x86/net/bpf_jit_comp.c +4
-0+static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks, +void *func_addr) {hum, I dont understand what's the
__arch_prepare_bpf_trampolinefor, could we have just thearch_prepare_bpf_trampoline?
ok, in the next patch ;-)
jirka
Re: [PATCH bpf-next 6/8] bpf: Add arch_bpf_trampoline_size()
In reply to Song LiuOn Tue, Sep 19, 2023 at 10:31:56PM -0700, Song Liu wrote:
SNIP
Quoted 12 lines
arch/x86/net/bpf_jit_comp.c +12
-0+int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +struct bpf_tramp_image im; +void *image; +int ret; + +/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). +* This will NOT cause fragmentation in direct map, as we do not +* call set_memory_*() on this buffer. +*/ + image =bpf_jit_alloc_exec (PAGE_SIZE );
arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2710
Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +10
-0+if (!image) +return -ENOMEM ; + + ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , m, flags, + tlinks, func_addr); +bpf_jit_free_exec (image); +if (ret <0 ) +return ret; + +return ret;
this can be just 'return ret;'
jirka
Quoted 39 lines
arch/x86/net/bpf_jit_comp.c +2
-0+} +static int emit_bpf_dispatcher (u8 **pprog,int a,int b,s64 *progs,u8 *image,u8 *buf) {u8 *jg_reloc, *prog = *pprog;include/linux/bpf.h +2
-0@@ -1087,6 +1087,8 @@ void *arch_alloc_bpf_trampoline(int size);void arch_free_bpf_trampoline (void *image,int size);void arch_protect_bpf_trampoline (void *image,int size);void arch_unprotect_bpf_trampoline (void *image,int size); +int arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,struct bpf_tramp_run_ctx *run_ctx);kernel/bpf/trampoline.c +6
-0@@ -1070,6 +1070,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, int size)set_memory_rw ((long )image,1 ); } +int __weak arch_bpf_trampoline_size (const struct btf_func_model *m,u32 flags, +struct bpf_tramp_links *tlinks,void *func_addr) +{ +return -ENOTSUPP ; +} +static int __init init_trampolines (void ) {int i;Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 7/8 bpf: Use arch_bpf_trampoline_size for trampoline
In reply to Song LiuInstead of blindly allocating PAGE_SIZE for each trampoline, check the size of the trampoline with arch_bpf_trampoline_size(). This size is saved in bpf_tramp_image->size, and used for modmem charge/uncharge. The fallback arch_alloc_bpf_trampoline() still allocates a whole page because we need to use set_memory_* to protect the memory.
This change does not cover struct_ops trampoline, which will still use a whole page for multiple trampolines.
--- include/linux/bpf.h | 1 + kernel/bpf/trampoline.c | 41 +++++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 16 deletions(-)
include/linux/bpf.h +1 -0
@@ -1122,6 +1122,7 @@ enum bpf_tramp_prog_type {struct bpf_tramp_image {void *image; +int size;struct bpf_ksym ksym;struct percpu_ref pcref;void *ip_after_call;
kernel/bpf/trampoline.c +25 -16
93 lines not on this page · raw message
Signature · 1 line
2.34.1
Song Liu
reviewerPATCH 8/8 x86, bpf: Use bpf_prog_pack for bpf trampoline
In reply to Song LiuThere are three major changes here:
1. Add arch_[alloc|free]_bpf_trampoline based on bpf_prog_pack;
2. Let arch_prepare_bpf_trampoline handle ROX input image, this requires arch_prepare_bpf_trampoline allocating a temporary RW buffer;
3. Update __arch_prepare_bpf_trampoline() to handle a RW buffer (rw_image) and a ROX buffer (image). This part is similar to the image/rw_image logic in bpf_int_jit_compile().
--- arch/x86/net/bpf_jit_comp.c | 95 +++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 26 deletions(-)
arch/x86/net/bpf_jit_comp.c +69 -26
@@ -2198,7 +2198,8 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog,static int invoke_bpf_prog (const struct btf_func_model *m,u8 **pprog,struct bpf_tramp_link *l,int stack_size, -int run_ctx_off,bool save_ret) +int run_ctx_off,bool save_ret, +void *image,void *rw_image) {u8 *prog = *pprog;u8 *jmp_insn;
@@ -2226,7 +2227,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,else EMIT4 (0x48 ,0x8D ,0x75 , -run_ctx_off); -if (emit_rsb_call (&prog,bpf_trampoline_enter (p), prog)) +if (emit_rsb_call (&prog,bpf_trampoline_enter (p), image + (prog - (u8 *)rw_image)))return -EINVAL ;/* remember prog start time returned by __bpf_prog_enter */ emit_mov_reg (&prog, true,BPF_REG_6 ,BPF_REG_0 );
@@ -2250,7 +2251,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, (long ) p->insnsi >>32 , (u32) (long) p->insnsi);/* call JITed bpf program or interpreter */ -if (emit_rsb_call (&prog, p->bpf_func, prog)) +if (emit_rsb_call (&prog, p->bpf_func, image + (prog - (u8 *)rw_image)))return -EINVAL ;/*
@@ -2277,7 +2278,7 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,EMIT3_off32 (0x48 ,0x8D ,0x95 , -run_ctx_off); else EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); -if (emit_rsb_call (&prog,bpf_trampoline_exit (p), prog)) +if (emit_rsb_call (&prog,bpf_trampoline_exit (p), image + (prog - (u8 *)rw_image)))return -EINVAL ; *pprog = prog;
@@ -2312,14 +2313,15 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)static int invoke_bpf (const struct btf_func_model *m,u8 **pprog,struct bpf_tramp_links *tl,int stack_size, -int run_ctx_off,bool save_ret) +int run_ctx_off,bool save_ret, +void *image,void *rw_image) {int i;u8 *prog = *pprog;for (i =0 ; i < tl->nr_links; i++) {if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, - run_ctx_off, save_ret)) + run_ctx_off, save_ret, image, rw_image))return -EINVAL ; } *pprog = prog;
@@ -2328,7 +2330,8 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,static int invoke_bpf_mod_ret (const struct btf_func_model *m,u8 **pprog,struct bpf_tramp_links *tl,int stack_size, -int run_ctx_off,u8 **branches) +int run_ctx_off,u8 **branches, +void *image,void *rw_image) {u8 *prog = *pprog;int i;
@@ -2339,7 +2342,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,emit_mov_imm32 (&prog, false,BPF_REG_0 ,0 );emit_stx (&prog,BPF_DW ,BPF_REG_FP ,BPF_REG_0 ,-8 );for (i =0 ; i < tl->nr_links; i++) { -if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, run_ctx_off, true)) +if (invoke_bpf_prog (m, &prog, tl->links[i], stack_size, run_ctx_off, true, + image, rw_image))return -EINVAL ;/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -2422,7 +2426,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,* add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end, +static int __arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *rw_image, +void *rw_image_end,void *image,const struct btf_func_model *m,u32 flags,struct bpf_tramp_links *tlinks,void *func_addr)
@@ -2521,7 +2526,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image orig_call +=X86_PATCH_SIZE ; } - prog = image; + prog = rw_image;EMIT_ENDBR ();/*
@@ -2563,7 +2568,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *imageif (flags &BPF_TRAMP_F_CALL_ORIG ) {/* arg1: mov rdi, im */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) im >>32 , (u32 ) (long ) im); -if (emit_rsb_call (&prog, __bpf_tramp_enter, prog)) { +if (emit_rsb_call (&prog, __bpf_tramp_enter, + image + (prog - (u8 *)rw_image))) { ret = -EINVAL ; goto cleanup; }
@@ -2571,7 +2577,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *imageif (fentry->nr_links)if (invoke_bpf (m, &prog, fentry, regs_off, run_ctx_off, - flags &BPF_TRAMP_F_RET_FENTRY_RET )) + flags &BPF_TRAMP_F_RET_FENTRY_RET , image, rw_image))return -EINVAL ;if (fmod_ret->nr_links) {
@@ -2581,7 +2587,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *imagereturn -ENOMEM ;if (invoke_bpf_mod_ret (m, &prog, fmod_ret, regs_off, - run_ctx_off, branches)) { + run_ctx_off, branches, image, rw_image)) { ret = -EINVAL ; goto cleanup; }
@@ -2602,14 +2608,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *imageEMIT2 (0xff ,0xd3 );/* call *rbx */ }else {/* call original function */ -if (emit_rsb_call (&prog, orig_call, prog)) { +if (emit_rsb_call (&prog, orig_call, image + (prog - (u8 *)rw_image))) { ret = -EINVAL ; goto cleanup; } }/* remember return value in a stack for bpf prog to access */ emit_stx (&prog,BPF_DW ,BPF_REG_FP ,BPF_REG_0 ,-8 ); - im->ip_after_call = prog; + im->ip_after_call = image + (prog - (u8 *)rw_image);memcpy (prog, x86_nops[5 ],X86_PATCH_SIZE ); prog +=X86_PATCH_SIZE ; }
@@ -2625,12 +2631,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image* aligned address of do_fexit. */ for (i =0 ; i < fmod_ret->nr_links; i++) -emit_cond_near_jump (&branches[i], prog, branches[i], -X86_JNE ); +emit_cond_near_jump (&branches[i], image + (prog - (u8 *)rw_image), + image + (branches[i] - (u8 *)rw_image),X86_JNE ); }if (fexit->nr_links) -if (invoke_bpf (m, &prog, fexit, regs_off, run_ctx_off, false)) { +if (invoke_bpf (m, &prog, fexit, regs_off, run_ctx_off, + false, image, rw_image)) { ret = -EINVAL ; goto cleanup; }
@@ -2643,10 +2650,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image* restored to R0. */ if (flags &BPF_TRAMP_F_CALL_ORIG ) { - im->ip_epilogue = prog; + im->ip_epilogue = image + (prog - (u8 *)rw_image);/* arg1: mov rdi, im */ emit_mov_imm64 (&prog,BPF_REG_1 , (long ) im >>32 , (u32 ) (long ) im); -if (emit_rsb_call (&prog, __bpf_tramp_exit, prog)) { +if (emit_rsb_call (&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { ret = -EINVAL ; goto cleanup; }
@@ -2665,25 +2672,61 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *imageif (flags &BPF_TRAMP_F_SKIP_FRAME )/* skip our return address and return to parent */ EMIT4 (0x48 ,0x83 ,0xC4 ,8 );/* add rsp, 8 */ -emit_return (&prog, prog); +emit_return (&prog, image + (prog - (u8 *)rw_image));/* Make sure the trampoline generation logic doesn't overflow */ -if (WARN_ON_ONCE (prog > (u8 *)image_end -BPF_INSN_SAFETY )) { +if (WARN_ON_ONCE (prog > (u8 *)rw_image_end -BPF_INSN_SAFETY )) { ret = -EFAULT ; goto cleanup; } - ret = prog - (u8 *)image +BPF_INSN_SAFETY ; + ret = prog - (u8 *)rw_image +BPF_INSN_SAFETY ; cleanup:kfree (branches);return ret; } +void *arch_alloc_bpf_trampoline (int size) +{ +return bpf_prog_pack_alloc (size, jit_fill_hole); +} + +void arch_free_bpf_trampoline (void *image,int size) +{ +bpf_prog_pack_free (image, size); +} + +void arch_protect_bpf_trampoline (void *image,int size) +{ +} + +void arch_unprotect_bpf_trampoline (void *image,int size) +{ +} +int arch_prepare_bpf_trampoline (struct bpf_tramp_image *im,void *image,void *image_end,const struct btf_func_model *m,u32 flags,struct bpf_tramp_links *tlinks,void *func_addr) { -return __arch_prepare_bpf_trampoline (im, image, image_end, m, flags, tlinks, func_addr); +void *rw_image, *tmp; +int ret; +u32 size = image_end - image; + + rw_image =bpf_jit_alloc_exec (size); +if (!rw_image) +return -ENOMEM ; + + ret =__arch_prepare_bpf_trampoline (im, rw_image, rw_image + size, image, m, + flags, tlinks, func_addr); +if (ret <0 ) + goto out; + + tmp =bpf_arch_text_copy (image, rw_image, size); +if (IS_ERR (tmp)) + ret =PTR_ERR (tmp); +out: +bpf_jit_free_exec (rw_image); +return ret; } int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
@@ -2701,8 +2744,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,if (!image)return -ENOMEM ; - ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , m, flags, - tlinks, func_addr); + ret =__arch_prepare_bpf_trampoline (&im, image, image +PAGE_SIZE , image, + m, flags, tlinks, func_addr);bpf_jit_free_exec (image);if (ret <0 )return ret;
Signature · 1 line
2.34.1