bpf: Support 64-bit pointers to kfuncs
Subsystems (4)
Ilya Leoshkevich
maintainerPATCH v5 1/1 bpf: Support 64-bit pointers to kfuncs
test_ksyms_module fails to emit a kfunc call targeting a module on s390x, because the verifier stores the difference between kfunc address and __bpf_call_base in bpf_insn.imm, which is s32, and modules are roughly (1 << 42) bytes away from the kernel on s390x.
Fix by keeping BTF id in bpf_insn.imm for BPF_PSEUDO_KFUNC_CALLs, and storing the absolute address in bpf_kfunc_desc.
Introduce bpf_jit_supports_far_kfunc_call() in order to limit this new behavior to the s390x JIT. Otherwise other JITs need to be modified, which is not desired.
Introduce bpf_get_kfunc_addr() instead of exposing both find_kfunc_desc() and struct bpf_kfunc_desc.
In addition to sorting kfuncs by imm, also sort them by offset, in order to handle conflicting imms from different modules. Do this on all architectures in order to simplify code.
Factor out resolving specialized kfuncs (XPD and dynptr) from fixup_kfunc_call(). This was required in the first place, because fixup_kfunc_call() uses find_kfunc_desc(), which returns a const pointer, so it's not possible to modify kfunc addr without stripping const, which is not nice. It also removes repetition of code like:
if (bpf_jit_supports_far_kfunc_call()) desc->addr = func; else insn->imm = BPF_CALL_IMM(func);
and separates kfunc_desc_tab fixups from kfunc_call fixups.
---
v4: https://lore.kernel.org/bpf/[email protected]/
v4 -> v5: Fix issues identified by Andrii:
- Improve bpf_get_kfunc_addr() argument naming.
- Do overflow check only in !bpf_jit_supports_far_kfunc_call() case.
- Fix kfunc_desc_cmp_by_imm_off() bug when passing huge values.
- Update fixup_kfunc_call() comment to reflect the new logic.
v3: https://lore.kernel.org/bpf/[email protected]/
v3 -> v4: Use Jiri's proposal and make it work on s390x.
arch/s390/net/bpf_jit_comp.c | 5 ++ include/linux/bpf.h | 2 + include/linux/filter.h | 1 + kernel/bpf/core.c | 11 +++ kernel/bpf/verifier.c | 137 +++++++++++++++++++++++++---------- 5 files changed, 116 insertions(+), 40 deletions(-)
arch/s390/net/bpf_jit_comp.c +5 -0
@@ -2001,6 +2001,11 @@ bool bpf_jit_supports_kfunc_call(void)return true; } +bool bpf_jit_supports_far_kfunc_call (void ) +{ +return true; +} +int bpf_arch_text_poke (void *ip,enum bpf_text_poke_type t,void *old_addr,void *new_addr) {
include/linux/bpf.h +2 -0
@@ -2295,6 +2295,8 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);const struct btf_func_model *bpf_jit_find_kfunc_model (const struct bpf_prog *prog,const struct bpf_insn *insn); +int bpf_get_kfunc_addr (const struct bpf_prog *prog,u32 func_id, +u16 btf_fd_idx,u8 **func_addr);struct bpf_core_ctx {struct bpf_verifier_log *log;const struct btf *btf;
include/linux/filter.h +1 -0
@@ -920,6 +920,7 @@ void bpf_jit_compile(struct bpf_prog *prog);bool bpf_jit_needs_zext (void );bool bpf_jit_supports_subprog_tailcalls (void );bool bpf_jit_supports_kfunc_call (void ); +bool bpf_jit_supports_far_kfunc_call (void );bool bpf_helper_changes_pkt_data (void *func);static inline bool bpf_dump_raw_ok (const struct cred *cred)
kernel/bpf/core.c +11 -0
@@ -1187,6 +1187,7 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,s16 off = insn->off;s32 imm = insn->imm;u8 *addr; +int err; *func_addr_fixed = insn->src_reg !=BPF_PSEUDO_CALL ;if (!*func_addr_fixed) {
@@ -1201,6 +1202,11 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog, addr = (u8 *)prog->aux->func[off]->bpf_func; else return -EINVAL; + }else if (insn->src_reg ==BPF_PSEUDO_KFUNC_CALL && +bpf_jit_supports_far_kfunc_call ()) { + err =bpf_get_kfunc_addr (prog, insn->imm, insn->off, &addr); +if (err) +return err; }else {/* Address of a BPF helper call. Since part of the core * kernel, it's always at a fixed location. __bpf_call_base
@@ -2732,6 +2738,11 @@ bool __weak bpf_jit_supports_kfunc_call(void)return false; } +bool __weak bpf_jit_supports_far_kfunc_call (void ) +{ +return false; +} +/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call * skb_copy_bits(), so provide a weak definition of it for NET-less config. */
kernel/bpf/verifier.c +97 -40
240 lines not on this page · raw message
Signature · 1 line
2.39.2
kernel test robot
build botkernel test robot noticed the following build errors:
Hi Ilya,
kernel test robot noticed the following build errors:
[auto build test ERROR on bpf-next/master]
url: https://github.com/intel-lab-lkp/linux/commits/Ilya-Leoshkevich/bpf-Support-64-bit-pointers-to-kfuncs/20230405-221738 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master patch link: https://lore.kernel.org/r/20230405141407.172357-1-iii%40linux.ibm.com patch subject: [PATCH bpf-next v5] bpf: Support 64-bit pointers to kfuncs config: i386-randconfig-r022-20230403 (https://download.01.org/0day-ci/archive/20230406/[email protected]/config) compiler: gcc-11 (Debian 11.3.0-8) 11.3.0
reproduce (this is a W=1 build):
# https://github.com/intel-lab-lkp/linux/commit/43119fecff7ac0771f037a6dc128ceb791917466
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Ilya-Leoshkevich/bpf-Support-64-bit-pointers-to-kfuncs/20230405-221738 git checkout 43119fecff7ac0771f037a6dc128ceb791917466
# save the config file
mkdir build_dir && cp config build_dir/.config
make W=1 O=build_dir ARCH=i386 olddefconfig
make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash kernel/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <[email protected]>
| Link: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
kernel/bpf/core.c: In function 'bpf_jit_get_func_addr':
>> kernel/bpf/core.c:1207:23: error: implicit declaration of function 'bpf_get_kfunc_addr'; did you mean 'bpf_jit_get_func_addr'? [-Werror=implicit-function-declaration]
1207 | err = bpf_get_kfunc_addr(prog, insn->imm, insn->off, &addr);
| ^~~~~~~~~~~~~~~~~~
| bpf_jit_get_func_addr kernel/bpf/core.c: At top level:
kernel/bpf/core.c:1638:12: warning: no previous prototype for 'bpf_probe_read_kernel' [-Wmissing-prototypes]
1638 | u64 __weak bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr)
| ^~~~~~~~~~~~~~~~~~~~~
kernel/bpf/core.c:2075:6: warning: no previous prototype for 'bpf_patch_call_args' [-Wmissing-prototypes]
2075 | void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
| ^~~~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
vim +1207 kernel/bpf/core.c
1182
1183 int bpf_jit_get_func_addr(const struct bpf_prog *prog,
1184 const struct bpf_insn *insn, bool extra_pass,
1185 u64 *func_addr, bool *func_addr_fixed)
1186 {
1187 s16 off = insn->off;
1188 s32 imm = insn->imm;
1189 u8 *addr;
1190 int err;
1191
1192 *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL;
1193 if (!*func_addr_fixed) {
1194 /* Place-holder address till the last pass has collected
1195 * all addresses for JITed subprograms in which case we
1196 * can pick them up from prog->aux.
1197 */
1198 if (!extra_pass)
1199 addr = NULL;
1200 else if (prog->aux->func &&
1201 off >= 0 && off < prog->aux->func_cnt)
1202 addr = (u8 *)prog->aux->func[off]->bpf_func;
1203 else
1204 return -EINVAL;
1205 } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
1206 bpf_jit_supports_far_kfunc_call()) {
> 1207 err = bpf_get_kfunc_addr(prog, insn->imm, insn->off, &addr);
1208 if (err)
1209 return err;
1210 } else {
1211 /* Address of a BPF helper call. Since part of the core
1212 * kernel, it's always at a fixed location. __bpf_call_base
1213 * and the helper with imm relative to it are both in core
1214 * kernel.
1215 */
1216 addr = (u8 *)__bpf_call_base + imm;
1217 }
1218
1219 *func_addr = (unsigned long)addr;
1220 return 0;
1221 }
1222
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests