Skip to contentSkip to conversation

bpf: Support 64-bit pointers to kfuncs

PATCH Superseded Ilya Leoshkevich 4 participants 21 messages Feb 22, 2023 – Mar 28, 2023

Subsystems (11)
Arm (32-bit) x86 Kselftest BPF +7 more subsystems

v3 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#5 lore raw

PATCH v3 5/12 bpf: sparc64: Fix jumping to the first insn

In reply to Ilya Leoshkevich

Jumping to the first insn causes emit_compare_and_branch() to access ctx->offsets[-1]. Currently ctx->offsets[] stores instruction end addresses; change it to store start addresses instead. We never need the end address of the last instruction.

---
 arch/sparc/net/bpf_jit_comp_64.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +7 -11
index 59d2d9953aa7..0a3c18e39199 100644
@@ -1168,7 +1168,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 
 	/* JUMP off */
 	case BPF_JMP | BPF_JA:
-		emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
+		emit_branch(BA, ctx->idx, ctx->offset[i + off + 1], ctx);
 		emit_nop(ctx);
 		break;
 	/* IF (dst COND src) JUMP off */
@@ -1185,7 +1185,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_JMP | BPF_JSET | BPF_X: {
 		int err;
 
-		err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
+		err = emit_compare_and_branch(code, dst, src, 0, false, i + off + 1, ctx);
 		if (err)
 			return err;
 		break;
@@ -1204,7 +1204,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_JMP | BPF_JSET | BPF_K: {
 		int err;
 
-		err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
+		err = emit_compare_and_branch(code, dst, 0, imm, true, i + off + 1, ctx);
 		if (err)
 			return err;
 		break;
@@ -1453,16 +1453,12 @@ static int build_body(struct jit_ctx *ctx)
 		const struct bpf_insn *insn = &prog->insnsi[i];
 		int ret;
 
-		ret = build_insn(insn, ctx);
-
-		if (ret > 0) {
-			i++;
-			ctx->offset[i] = ctx->idx;
-			continue;
-		}
 		ctx->offset[i] = ctx->idx;
-		if (ret)
+		ret = build_insn(insn, ctx);
+		if (ret < 0)
 			return ret;
+
+		i += ret;
 	}
 	return 0;
 }
Signature · 1 line
2.39.1
#8 lore raw

PATCH v3 8/12 bpf: sparc64: Use bpf_jit_get_func_addr()

In reply to Ilya Leoshkevich

Preparation for moving kfunc address from bpf_insn.imm.

---
 arch/sparc/net/bpf_jit_comp_64.c | 20 ++++++++++++++------
 arch/x86/net/bpf_jit_comp32.c    |  7 +++++++
 2 files changed, 21 insertions(+), 6 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +14 -6
index 6c482685dc6c..b23083776718 100644
@@ -893,7 +893,8 @@ static void emit_tail_call(struct jit_ctx *ctx)
 	emit_nop(ctx);
 }
 
-static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
+static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
+		      bool extra_pass)
 {
 	const u8 code = insn->code;
 	const u8 dst = bpf2sparc[insn->dst_reg];
@@ -1214,7 +1215,14 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	/* function call */
 	case BPF_JMP | BPF_CALL:
 	{
-		u8 *func = ((u8 *)__bpf_call_base) + imm;
+		bool func_addr_fixed;
+		u64 func;
+		int err;
+
+		err = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
+					    &func, &func_addr_fixed);
+		if (err)
+			return err;
 
 		ctx->saw_call = true;
 
@@ -1445,7 +1453,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	return 0;
 }
 
-static int build_body(struct jit_ctx *ctx)
+static int build_body(struct jit_ctx *ctx, bool extra_pass)
 {
 	const struct bpf_prog *prog = ctx->prog;
 	int i;
@@ -1455,7 +1463,7 @@ static int build_body(struct jit_ctx *ctx)
 		int ret;
 
 		ctx->offset[i] = ctx->idx;
-		ret = build_insn(insn, ctx);
+		ret = build_insn(insn, ctx, extra_pass);
 		if (ret < 0)
 			return ret;
 
@@ -1549,7 +1557,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		ctx.idx = 0;
 
 		build_prologue(&ctx);
-		if (build_body(&ctx)) {
+		if (build_body(&ctx, extra_pass)) {
 			prog = orig_prog;
 			goto out_off;
 		}
@@ -1586,7 +1594,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
 	build_prologue(&ctx);
 
-	if (build_body(&ctx)) {
+	if (build_body(&ctx, extra_pass)) {
 		bpf_jit_binary_free(header);
 		prog = orig_prog;
 		goto out_off;
arch/x86/net/bpf_jit_comp32.c +7 -0
index 429a89c5468b..0abb4d6c9dec 100644
@@ -2091,6 +2091,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			if (insn->src_reg == BPF_PSEUDO_CALL)
 				goto notyet;
 
+			err = bpf_jit_get_func_addr(bpf_prog, insn, extra_pass,
+						    &func_addr,
+						    &func_addr_fixed);
+			if (err)
+				return err;
+			func = (u8 *)(unsigned long)func_addr;
+
 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 				int err;
 
Signature · 1 line
2.39.1
#12 lore raw

PATCH v3 11/12 bpf: Support 64-bit pointers to kfuncs

In reply to Ilya Leoshkevich

test_ksyms_module fails to emit a kfunc call targeting a module on s390x, because the verifier stores the difference between kfunc address and __bpf_call_base in bpf_insn.imm, which is s32, and modules are roughly (1 << 42) bytes away from the kernel on s390x.

Fix by keeping BTF id in bpf_insn.imm for BPF_PSEUDO_KFUNC_CALLs, and storing the absolute address in bpf_kfunc_desc, which JITs retrieve as usual by calling bpf_jit_get_func_addr().

Introduce bpf_get_kfunc_addr() instead of exposing both find_kfunc_desc() and struct bpf_kfunc_desc.

This also fixes the problem with XDP metadata functions outlined in the description of commit 63d7b53ab59f ("s390/bpf: Implement bpf_jit_supports_kfunc_call()") by replacing address lookups with BTF id lookups. This eliminates the inconsistency between "abstract" XDP metadata functions' BTF ids and their concrete addresses.

---
 include/linux/bpf.h   |  2 ++
 kernel/bpf/core.c     | 21 ++++++++++--
 kernel/bpf/verifier.c | 79 +++++++++++++------------------------------
 3 files changed, 44 insertions(+), 58 deletions(-)
include/linux/bpf.h +2 -0
index 520b238abd5a..e521eae334ea 100644
@@ -2234,6 +2234,8 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
 const struct btf_func_model *
 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 			 const struct bpf_insn *insn);
+int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 offset,
+		       u8 **func_addr);
 struct bpf_core_ctx {
 	struct bpf_verifier_log *log;
 	const struct btf *btf;
kernel/bpf/core.c +18 -3
index 933869983e2a..4d51782f17ab 100644
@@ -1185,10 +1185,12 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 {
 	s16 off = insn->off;
 	s32 imm = insn->imm;
+	bool fixed;
 	u8 *addr;
+	int err;
 
-	*func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL;
-	if (!*func_addr_fixed) {
+	switch (insn->src_reg) {
+	case BPF_PSEUDO_CALL:
 		/* Place-holder address till the last pass has collected
 		 * all addresses for JITed subprograms in which case we
 		 * can pick them up from prog->aux.
@@ -1200,15 +1202,28 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 			addr = (u8 *)prog->aux->func[off]->bpf_func;
 		else
 			return -EINVAL;
-	} else {
+		fixed = false;
+		break;
+	case 0:
 		/* Address of a BPF helper call. Since part of the core
 		 * kernel, it's always at a fixed location. __bpf_call_base
 		 * and the helper with imm relative to it are both in core
 		 * kernel.
 		 */
 		addr = (u8 *)__bpf_call_base + imm;
+		fixed = true;
+		break;
+	case BPF_PSEUDO_KFUNC_CALL:
+		err = bpf_get_kfunc_addr(prog, imm, off, &addr);
+		if (err)
+			return err;
+		fixed = true;
+		break;
+	default:
+		return -EINVAL;
 	}
 
+	*func_addr_fixed = fixed;
 	*func_addr = (unsigned long)addr;
 	return 0;
 }
kernel/bpf/verifier.c +24 -55

156 lines not on this page · raw message

Signature · 1 line
2.39.1
#2 lore raw

PATCH v3 2/12 selftests/bpf: Fix test_verifier on 32-bit systems

In reply to Ilya Leoshkevich

bpf_fentry_test_t.a, bpf_iter_meta.seq, bpf_map.ops and prog_test_ref_kfunc.next are pointers, so access them using BPF_W on 32-bit systems.

---
 tools/testing/selftests/bpf/test_verifier.c             | 5 +++++
 tools/testing/selftests/bpf/verifier/atomic_fetch_add.c | 2 +-
 tools/testing/selftests/bpf/verifier/bpf_get_stack.c    | 2 +-
 tools/testing/selftests/bpf/verifier/calls.c            | 4 ++--
 tools/testing/selftests/bpf/verifier/map_kptr.c         | 2 +-
 tools/testing/selftests/bpf/verifier/map_ptr.c          | 2 +-
 6 files changed, 11 insertions(+), 6 deletions(-)
tools/testing/selftests/bpf/test_verifier.c +5 -0
index 8b9949bb833d..81a8aef7362c 100644
@@ -62,6 +62,11 @@
 #define MAX_FUNC_INFOS	8
 #define MAX_BTF_STRINGS	256
 #define MAX_BTF_TYPES	256
+#if BITS_PER_LONG == 32
+#define BPF_PTR BPF_W
+#else
+#define BPF_PTR BPF_DW
+#endif
 
 #define INSN_OFF_MASK	((__s16)0xFFFF)
 #define INSN_IMM_MASK	((__s32)0xFFFFFFFF)
tools/testing/selftests/bpf/verifier/atomic_fetch_add.c +1 -1
index a91de8cd9def..6ed98eb217b3 100644
@@ -93,7 +93,7 @@
 		 * because it's kernel memory.
 		 */
 		BPF_MOV64_IMM(BPF_REG_3, 1),
-		BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, BPF_REG_2, BPF_REG_3, 0),
+		BPF_ATOMIC_OP(BPF_PTR, BPF_ADD | BPF_FETCH, BPF_REG_2, BPF_REG_3, 0),
 		/* Done */
 		BPF_MOV64_IMM(BPF_REG_0, 0),
 		BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/bpf_get_stack.c +1 -1
index 3e024c891178..e48b409d1759 100644
@@ -46,7 +46,7 @@
 	"bpf_get_task_stack return R0 range is refined",
 	.insns = {
 	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0),
-	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_6, 0), // ctx->meta->seq
+	BPF_LDX_MEM(BPF_PTR, BPF_REG_6, BPF_REG_6, 0), // ctx->meta->seq
 	BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_1, 8), // ctx->task
 	BPF_LD_MAP_FD(BPF_REG_1, 0), // fixup_map_array_48b
 	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
tools/testing/selftests/bpf/verifier/calls.c +2 -2
index 9d993926bf0e..a59759db8c78 100644
@@ -168,7 +168,7 @@
 	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
 	BPF_EXIT_INSN(),
 	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
-	BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 16),
+	BPF_LDX_MEM(BPF_PTR, BPF_REG_1, BPF_REG_1, 16),
 	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -4),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
 	BPF_MOV64_IMM(BPF_REG_0, 0),
@@ -230,7 +230,7 @@
 	BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
 	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
-	BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 16),
+	BPF_LDX_MEM(BPF_PTR, BPF_REG_1, BPF_REG_6, 16),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
 	BPF_MOV64_IMM(BPF_REG_0, 0),
 	BPF_EXIT_INSN(),
tools/testing/selftests/bpf/verifier/map_kptr.c +1 -1
index 6914904344c0..b8ea6b0b536d 100644
@@ -235,7 +235,7 @@
 	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0),
 	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
 	BPF_EXIT_INSN(),
-	BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 16),
+	BPF_LDX_MEM(BPF_PTR, BPF_REG_1, BPF_REG_0, 16),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_this_cpu_ptr),
 	BPF_EXIT_INSN(),
 	},
tools/testing/selftests/bpf/verifier/map_ptr.c +1 -1
index 17ee84dc7766..b8e308d678b7 100644
@@ -51,7 +51,7 @@
 	.insns = {
 	BPF_MOV64_IMM(BPF_REG_6, 0),
 	BPF_LD_MAP_FD(BPF_REG_1, 0),
-	BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0),
+	BPF_LDX_MEM(BPF_PTR, BPF_REG_6, BPF_REG_1, 0),
 	BPF_MOV64_IMM(BPF_REG_0, 1),
 	BPF_EXIT_INSN(),
 	},
Signature · 1 line
2.39.1
#1 lore raw

PATCH v3 1/12 selftests/bpf: Finish folding after BPF_FUNC_csum_diff

In reply to Ilya Leoshkevich

bpf_csum_diff() may return non-folded checksum, and the arm implementation actually does this. Finish folding in the test prog.

---
 tools/testing/selftests/bpf/verifier/array_access.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
tools/testing/selftests/bpf/verifier/array_access.c +9 -1
index 1b138cd2b187..e570d6a95702 100644
@@ -241,7 +241,7 @@
 	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
 	BPF_LD_MAP_FD(BPF_REG_1, 0),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
-	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 6),
+	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 14),
 
 	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
 	BPF_MOV64_IMM(BPF_REG_2, 4),
@@ -250,7 +250,15 @@
 	BPF_MOV64_IMM(BPF_REG_5, 0),
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
 		     BPF_FUNC_csum_diff),
+	/* csum_partial() is allowed to return both 0xffffffe3 and 0x1ffe2 */
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_ALU64_IMM(BPF_RSH, BPF_REG_1, 16),
+	BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffff),
+	BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_ALU64_IMM(BPF_RSH, BPF_REG_1, 16),
 	BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffff),
+	BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
 	BPF_EXIT_INSN(),
 	},
 	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
Signature · 1 line
2.39.1
#11 lore raw

PATCH v3 10/12 bpf, x86_32: Use bpf_jit_get_func_addr()

In reply to Ilya Leoshkevich

Preparation for moving kfunc address from bpf_insn.imm.

---
 arch/x86/net/bpf_jit_comp32.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
arch/x86/net/bpf_jit_comp32.c +14 -6
index 0abb4d6c9dec..998d9bebe4ae 100644
@@ -1567,7 +1567,7 @@ static u8 get_cond_jmp_opcode(const u8 op, bool is_cmp_lo)
  *	- 0-2 jit-insns (3 bytes each) to handle the return value.
  */
 static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr,
-			   const struct bpf_insn *insn, u8 **pprog)
+			   const struct bpf_insn *insn, u8 *func, u8 **pprog)
 {
 	const u8 arg_regs[] = { IA32_EAX, IA32_EDX, IA32_ECX };
 	int i, cnt = 0, first_stack_regno, last_stack_regno;
@@ -1628,7 +1628,7 @@ static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr,
 	if (fm->ret_size)
 		end_addr -= 3;
 
-	jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr;
+	jmp_offset = func - end_addr;
 	if (!is_simm32(jmp_offset)) {
 		pr_err("unsupported BPF kernel function jmp_offset:%lld\n",
 		       jmp_offset);
@@ -1657,7 +1657,7 @@ static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr,
 }
 
 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
-		  int oldproglen, struct jit_context *ctx)
+		  int oldproglen, struct jit_context *ctx, bool extra_pass)
 {
 	struct bpf_insn *insn = bpf_prog->insnsi;
 	int insn_cnt = bpf_prog->len;
@@ -2087,6 +2087,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			const u8 *r3 = bpf2ia32[BPF_REG_3];
 			const u8 *r4 = bpf2ia32[BPF_REG_4];
 			const u8 *r5 = bpf2ia32[BPF_REG_5];
+			bool func_addr_fixed;
+			u64 func_addr;
+			int err;
 
 			if (insn->src_reg == BPF_PSEUDO_CALL)
 				goto notyet;
@@ -2103,14 +2106,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 
 				err = emit_kfunc_call(bpf_prog,
 						      image + addrs[i],
-						      insn, &prog);
+						      insn, func, &prog);
 
 				if (err)
 					return err;
 				break;
 			}
 
-			func = (u8 *) __bpf_call_base + imm32;
 			jmp_offset = func - (image + addrs[i]);
 
 			if (!imm32 || !is_simm32(jmp_offset)) {
@@ -2533,6 +2535,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	struct jit_context ctx = {};
 	bool tmp_blinded = false;
 	u8 *image = NULL;
+	bool extra_pass;
 	int *addrs;
 	int pass;
 	int i;
@@ -2552,6 +2555,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = tmp;
 	}
 
+	extra_pass = prog->aux->jit_data;
+	if (!extra_pass)
+		prog->aux->jit_data = bpf_int_jit_compile;
+
 	addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);
 	if (!addrs) {
 		prog = orig_prog;
@@ -2575,7 +2582,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 * pass to emit the final image.
 	 */
 	for (pass = 0; pass < 20 || image; pass++) {
-		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
+		proglen = do_jit(prog, addrs, image, oldproglen, &ctx,
+				 extra_pass);
 		if (proglen <= 0) {
 out_image:
 			image = NULL;
Signature · 1 line
2.39.1
#15 lore raw

PATCH v3 12/12 selftests/bpf: Trim DENYLIST.s390x

In reply to Ilya Leoshkevich

With commit 1a280f48c0e4 ("s390/kprobes: replace kretprobe with rethook") and commit 2213d44e140f ("s390/syscalls: get rid of system call alias functions") merged, and kfunc range issues fixed, only two known test_progs failures remain on s390x.

---
 tools/testing/selftests/bpf/DENYLIST.s390x | 20 --------------------
 1 file changed, 20 deletions(-)
tools/testing/selftests/bpf/DENYLIST.s390x +0 -20
index b89eb87034e4..a17baf8c6fd7 100644
@@ -1,24 +1,4 @@
 # TEMPORARY
 # Alphabetical order
-bloom_filter_map                         # failed to find kernel BTF type ID of '__x64_sys_getpgid': -3                (?)
-bpf_cookie                               # failed to open_and_load program: -524 (trampoline)
-bpf_loop                                 # attaches to __x64_sys_nanosleep
-cgrp_local_storage                       # prog_attach unexpected error: -524                                          (trampoline)
-fexit_sleep                              # fexit_skel_load fexit skeleton failed                                       (trampoline)
 get_stack_raw_tp                         # user_stack corrupted user stack                                             (no backchain userspace)
-kprobe_multi_bench_attach                # bpf_program__attach_kprobe_multi_opts unexpected error: -95
-kprobe_multi_test                        # relies on fentry
-ksyms_module                             # test_ksyms_module__open_and_load unexpected error: -9                       (?)
-ksyms_module_libbpf                      # JIT does not support calling kernel function                                (kfunc)
-ksyms_module_lskel                       # test_ksyms_module_lskel__open_and_load unexpected error: -9                 (?)
-module_attach                            # skel_attach skeleton attach failed: -524                                    (trampoline)
-ringbuf                                  # skel_load skeleton load failed                                              (?)
 stacktrace_build_id                      # compare_map_keys stackid_hmap vs. stackmap err -2 errno 2                   (?)
-test_lsm                                 # attach unexpected error: -524                                               (trampoline)
-trace_printk                             # trace_printk__load unexpected error: -2 (errno 2)                           (?)
-trace_vprintk                            # trace_vprintk__open_and_load unexpected error: -9                           (?)
-unpriv_bpf_disabled                      # fentry
-user_ringbuf                             # failed to find kernel BTF type ID of '__s390x_sys_prctl': -3                (?)
-verif_stats                              # trace_vprintk__open_and_load unexpected error: -9                           (?)
-xdp_bonding                              # failed to auto-attach program 'trace_on_entry': -524                        (trampoline)
-xdp_metadata                             # JIT does not support calling kernel function                                (kfunc)
Signature · 1 line
2.39.1
#10 lore raw

PATCH v3 9/12 bpf: x86: Use bpf_jit_get_func_addr()

In reply to Ilya Leoshkevich

Preparation for moving kfunc address from bpf_insn.imm.

---
 arch/x86/net/bpf_jit_comp.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
arch/x86/net/bpf_jit_comp.c +12 -3
index 1056bbf55b17..f722f431ba6f 100644
@@ -964,7 +964,8 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)
 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
 
 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
-		  int oldproglen, struct jit_context *ctx, bool jmp_padding)
+		  int oldproglen, struct jit_context *ctx, bool jmp_padding,
+		  bool extra_pass)
 {
 	bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
 	struct bpf_insn *insn = bpf_prog->insnsi;
@@ -1000,9 +1001,11 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
 		const s32 imm32 = insn->imm;
 		u32 dst_reg = insn->dst_reg;
 		u32 src_reg = insn->src_reg;
+		bool func_addr_fixed;
 		u8 b2 = 0, b3 = 0;
 		u8 *start_of_ldx;
 		s64 jmp_offset;
+		u64 func_addr;
 		s16 insn_off;
 		u8 jmp_cond;
 		u8 *func;
@@ -1536,7 +1539,12 @@ st:			if (is_imm8(insn->off))
 		case BPF_JMP | BPF_CALL: {
 			int offs;
 
-			func = (u8 *) __bpf_call_base + imm32;
+			err = bpf_jit_get_func_addr(bpf_prog, insn, extra_pass,
+						    &func_addr,
+						    &func_addr_fixed);
+			if (err < 0)
+				return err;
+			func = (u8 *)(unsigned long)func_addr;
 			if (tail_call_reachable) {
 				/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
 				EMIT3_off32(0x48, 0x8B, 0x85,
@@ -2518,7 +2526,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	for (pass = 0; pass < MAX_PASSES || image; pass++) {
 		if (!padding && pass >= PADDING_PASSES)
 			padding = true;
-		proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);
+		proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx,
+				 padding, extra_pass);
 		if (proglen <= 0) {
 out_image:
 			image = NULL;
Signature · 1 line
2.39.1
#3 lore raw

PATCH v3 3/12 sparc: Update maximum errno

In reply to Ilya Leoshkevich

When the bpf syscall returns -ENOTSUPP, the kernel does not set psr/csr. glibc's syscall() then thinks everything is fine and skips updating errno, causing all kinds of confusion in bpf selftests.

sparc decides whether to set psr/csr by comparing syscall return value with ERESTART_RESTARTBLOCK, which is smaller than ENOTSUPP. Fix by introducing EMAXERRNO (like mips) and comparing with that insead.

---
 arch/sparc/include/asm/errno.h | 10 ++++++++++
 arch/sparc/kernel/entry.S      |  2 +-
 arch/sparc/kernel/process.c    |  6 +++---
 arch/sparc/kernel/syscalls.S   |  2 +-
 4 files changed, 15 insertions(+), 5 deletions(-)
 create mode 100644 arch/sparc/include/asm/errno.h
arch/sparc/include/asm/errno.h new file +10 -0
new file mode 100644
index 000000000000..0e0a790b8ea8
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SPARC_ERRNO_H
+#define _ASM_SPARC_ERRNO_H
+
+#include <uapi/asm/errno.h>
+
+/* The biggest error number defined here or in <linux/errno.h>. */
+#define EMAXERRNO	1133
+
+#endif /* _ASM_SPARC_ERRNO_H */
arch/sparc/kernel/entry.S +1 -1
index a269ad2fe6df..a24b46ad7b20 100644
@@ -1004,7 +1004,7 @@ do_syscall:
 
 ret_sys_call:
 	ld	[%curptr + TI_FLAGS], %l5
-	cmp	%o0, -ERESTART_RESTARTBLOCK
+	cmp	%o0, -EMAXERRNO
 	ld	[%sp + STACKFRAME_SZ + PT_PSR], %g3
 	set	PSR_C, %g2
 	bgeu	1f
arch/sparc/kernel/process.c +3 -3
index 0442ab00518d..582933d16f9f 100644
@@ -32,7 +32,7 @@ asmlinkage long sparc_fork(struct pt_regs *regs)
 	 * the parent's %o1.  So detect that case and restore it
 	 * here.
 	 */
-	if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK)
+	if ((unsigned long)ret >= -EMAXERRNO)
 		regs->u_regs[UREG_I1] = orig_i1;
 
 	return ret;
@@ -57,7 +57,7 @@ asmlinkage long sparc_vfork(struct pt_regs *regs)
 	 * the parent's %o1.  So detect that case and restore it
 	 * here.
 	 */
-	if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK)
+	if ((unsigned long)ret >= -EMAXERRNO)
 		regs->u_regs[UREG_I1] = orig_i1;
 
 	return ret;
@@ -103,7 +103,7 @@ asmlinkage long sparc_clone(struct pt_regs *regs)
 	 * the parent's %o1.  So detect that case and restore it
 	 * here.
 	 */
-	if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK)
+	if ((unsigned long)ret >= -EMAXERRNO)
 		regs->u_regs[UREG_I1] = orig_i1;
 
 	return ret;
arch/sparc/kernel/syscalls.S +1 -1
index 0e8ab0602c36..9064304f6a3a 100644
@@ -262,7 +262,7 @@ ret_sys_call:
 	mov	%ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2
 	sllx	%g2, 32, %g2
 
-	cmp	%o0, -ERESTART_RESTARTBLOCK
+	cmp	%o0, -EMAXERRNO
 	bgeu,pn	%xcc, 1f
 	 andcc	%l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
 	ldx	[%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc
Signature · 1 line
2.39.1
#0 lore raw

v3 cover bpf: Support 64-bit pointers to kfuncs

v2: https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
Drop the merged check_subprogs() cleanup.
Adjust arm, sparc and i386 JITs.
Fix a few portability issues in test_verifier.
Fix a few sparc64 issues.
Trim s390x denylist.

v1: https://lore.kernel.org/bpf/[email protected]/T/#t
v1 -> v2: Add BPF_HELPER_CALL (Stanislav).
Add check_subprogs() cleanup - noticed while reviewing the code for BPF_HELPER_CALL.
Drop WARN_ON_ONCE (Stanislav, Alexei).
Add bpf_jit_get_func_addr() to x86_64 JIT.

Hi,

This series solves the problems outlined in [1, 2, 3]. The main problem is that kfuncs in modules do not fit into bpf_insn.imm on s390x; the secondary problem is that there is a conflict between "abstract" XDP metadata function BTF ids and their "concrete" addresses.

The solution is to keep fkunc BTF ids in bpf_insn.imm, and put the addresses into bpf_kfunc_desc, which does not have size restrictions.

Regtested with test_verifier and test_progs on x86_64 and s390x. Regtested with test_verifier only on arm, sparc64 and i386.

[1] https://lore.kernel.org/bpf/Y9%2FyrKZkBK6yzXp+@krava/
[2] https://lore.kernel.org/bpf/[email protected]/
[3] https://lore.kernel.org/bpf/[email protected]/

Best regards,
Ilya

Ilya Leoshkevich (12):
selftests/bpf: Finish folding after BPF_FUNC_csum_diff selftests/bpf: Fix test_verifier on 32-bit systems
sparc: Update maximum errno
bpf: sparc64: Emit fixed-length instructions for BPF_PSEUDO_FUNC
bpf: sparc64: Fix jumping to the first insn
bpf: sparc64: Use 32-bit tail call index
bpf, arm: Use bpf_jit_get_func_addr()
bpf: sparc64: Use bpf_jit_get_func_addr()
bpf: x86: Use bpf_jit_get_func_addr()
bpf, x86_32: Use bpf_jit_get_func_addr()
bpf: Support 64-bit pointers to kfuncs
selftests/bpf: Trim DENYLIST.s390x

 arch/arm/net/bpf_jit_32.c                     | 27 +++++--
 arch/sparc/include/asm/errno.h                | 10 +++
 arch/sparc/kernel/entry.S                     |  2 +-
 arch/sparc/kernel/process.c                   |  6 +-
 arch/sparc/kernel/syscalls.S                  |  2 +-
 arch/sparc/net/bpf_jit_comp_64.c              | 50 +++++++-----
 arch/x86/net/bpf_jit_comp.c                   | 15 +++-
 arch/x86/net/bpf_jit_comp32.c                 | 27 +++++--
 include/linux/bpf.h                           |  2 +
 kernel/bpf/core.c                             | 21 ++++-
 kernel/bpf/verifier.c                         | 79 ++++++-------------
 tools/testing/selftests/bpf/DENYLIST.s390x    | 20 -----
 tools/testing/selftests/bpf/test_verifier.c   |  5 ++
 .../selftests/bpf/verifier/array_access.c     | 10 ++-
 .../selftests/bpf/verifier/atomic_fetch_add.c |  2 +-
 .../selftests/bpf/verifier/bpf_get_stack.c    |  2 +-
 tools/testing/selftests/bpf/verifier/calls.c  |  4 +-
 .../testing/selftests/bpf/verifier/map_kptr.c |  2 +-
 .../testing/selftests/bpf/verifier/map_ptr.c  |  2 +-
 19 files changed, 164 insertions(+), 124 deletions(-)
 create mode 100644 arch/sparc/include/asm/errno.h
Signature · 1 line
2.39.1
#7 lore raw

PATCH v3 7/12 bpf, arm: Use bpf_jit_get_func_addr()

Preparation for moving kfunc address from bpf_insn.imm.

---
 arch/arm/net/bpf_jit_32.c | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)
arch/arm/net/bpf_jit_32.c +20 -7
index 6a1c9fca5260..b9a7f7bba811 100644
@@ -1345,7 +1345,8 @@ static void build_epilogue(struct jit_ctx *ctx)
  *	>0 - Successfully JITed a 16-byte eBPF instruction
  *	<0 - Failed to JIT.
  */
-static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
+static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
+		      bool extra_pass)
 {
 	const u8 code = insn->code;
 	const s8 *dst = bpf2a32[insn->dst_reg];
@@ -1783,7 +1784,14 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		const s8 *r3 = bpf2a32[BPF_REG_3];
 		const s8 *r4 = bpf2a32[BPF_REG_4];
 		const s8 *r5 = bpf2a32[BPF_REG_5];
-		const u32 func = (u32)__bpf_call_base + (u32)imm;
+		bool func_addr_fixed;
+		u64 func;
+		int err;
+
+		err = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
+					    &func, &func_addr_fixed);
+		if (err)
+			return err;
 
 		emit_a32_mov_r64(true, r0, r1, ctx);
 		emit_a32_mov_r64(true, r1, r2, ctx);
@@ -1791,7 +1799,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		emit_push_r64(r4, ctx);
 		emit_push_r64(r3, ctx);
 
-		emit_a32_mov_i(tmp[1], func, ctx);
+		emit_a32_mov_i(tmp[1], (u32)func, ctx);
 		emit_blx_r(tmp[1], ctx);
 
 		emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
@@ -1826,7 +1834,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	return 0;
 }
 
-static int build_body(struct jit_ctx *ctx)
+static int build_body(struct jit_ctx *ctx, bool extra_pass)
 {
 	const struct bpf_prog *prog = ctx->prog;
 	unsigned int i;
@@ -1835,7 +1843,7 @@ static int build_body(struct jit_ctx *ctx)
 		const struct bpf_insn *insn = &(prog->insnsi[i]);
 		int ret;
 
-		ret = build_insn(insn, ctx);
+		ret = build_insn(insn, ctx, extra_pass);
 
 		/* It's used with loading the 64 bit immediate value. */
 		if (ret > 0) {
@@ -1880,6 +1888,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	struct jit_ctx ctx;
 	unsigned int tmp_idx;
 	unsigned int image_size;
+	bool extra_pass;
 	u8 *image_ptr;
 
 	/* If BPF JIT was not enabled then we must fall back to
@@ -1901,6 +1910,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = tmp;
 	}
 
+	extra_pass = prog->aux->jit_data;
+	if (!extra_pass)
+		prog->aux->jit_data = bpf_int_jit_compile;
+
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.prog = prog;
 	ctx.cpu_architecture = cpu_architecture();
@@ -1924,7 +1937,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 * being successful in the second pass, so just fall back
 	 * to the interpreter.
 	 */
-	if (build_body(&ctx)) {
+	if (build_body(&ctx, extra_pass)) {
 		prog = orig_prog;
 		goto out_off;
 	}
@@ -1982,7 +1995,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	/* If building the body of the JITed code fails somehow,
 	 * we fall back to the interpretation.
 	 */
-	if (build_body(&ctx) < 0) {
+	if (build_body(&ctx, extra_pass) < 0) {
 		image_ptr = NULL;
 		bpf_jit_binary_free(header);
 		prog = orig_prog;
Signature · 1 line
2.39.1
#4 lore raw

PATCH v3 4/12 bpf: sparc64: Emit fixed-length instructions for BPF_PSEUDO_FUNC

In reply to Ilya Leoshkevich

This is the same as commit b54b6003612a ("riscv, bpf: Emit fixed-length instructions for BPF_PSEUDO_FUNC"), but for sparc64. The code sequence is borrowed from sparc64-linux-gnu-gcc -Os.

---
 arch/sparc/net/bpf_jit_comp_64.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +11 -2
index fa0759bfe498..59d2d9953aa7 100644
@@ -1243,10 +1243,19 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_LD | BPF_IMM | BPF_DW:
 	{
 		const struct bpf_insn insn1 = insn[1];
+		const u8 tmp = bpf2sparc[TMP_REG_1];
 		u64 imm64;
 
-		imm64 = (u64)insn1.imm << 32 | (u32)imm;
-		emit_loadimm64(imm64, dst, ctx);
+		if (bpf_pseudo_func(insn)) {
+			/* fixed-length insns for extra jit pass */
+			emit_set_const(dst, insn1.imm, ctx);
+			emit_set_const(tmp, imm, ctx);
+			emit_alu_K(SLLX, dst, 32, ctx);
+			emit_alu3(ADD, dst, tmp, dst, ctx);
+		} else {
+			imm64 = (u64)insn1.imm << 32 | (u32)imm;
+			emit_loadimm64(imm64, dst, ctx);
+		}
 
 		return 1;
 	}
Signature · 1 line
2.39.1
#6 lore raw

PATCH v3 6/12 bpf: sparc64: Use 32-bit tail call index

In reply to Ilya Leoshkevich

The interpreter does the following:

  u32 index = BPF_R3;

and JITs should do the same.

---
 arch/sparc/net/bpf_jit_comp_64.c | 1 +
 1 file changed, 1 insertion(+)
arch/sparc/net/bpf_jit_comp_64.c +1 -0
index 0a3c18e39199..6c482685dc6c 100644
@@ -856,6 +856,7 @@ static void emit_tail_call(struct jit_ctx *ctx)
 
 	ctx->saw_tail_call = true;
 
+	emit_alu_K(SRL, bpf_index, 0, ctx);
 	off = offsetof(struct bpf_array, map.max_entries);
 	emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
 	emit_cmp(bpf_index, tmp, ctx);
Signature · 1 line
2.39.1
Acked-by on v3 11/12
#13 lore raw
In reply to Ilya Leoshkevich
Quoted 13 lines · Ilya Leoshkevich

test_ksyms_module fails to emit a kfunc call targeting a module on s390x, because the verifier stores the difference between kfunc address and __bpf_call_base in bpf_insn.imm, which is s32, and modules are roughly (1 << 42) bytes away from the kernel on s390x.

Fix by keeping BTF id in bpf_insn.imm for BPF_PSEUDO_KFUNC_CALLs, and storing the absolute address in bpf_kfunc_desc, which JITs retrieve as usual by calling bpf_jit_get_func_addr().

Introduce bpf_get_kfunc_addr() instead of exposing both find_kfunc_desc() and struct bpf_kfunc_desc.

Quoted 7 lines

This also fixes the problem with XDP metadata functions outlined in the description of commit 63d7b53ab59f ("s390/bpf: Implement bpf_jit_supports_kfunc_call()") by replacing address lookups with BTF id lookups. This eliminates the inconsistency between "abstract" XDP metadata functions' BTF ids and their concrete addresses.

With a nit below (and an unrelated question).

I'll wait a bit for the buildbots to finish until ack'ing the rest. But the jit (except sparc quirks) and selftest changes also make sense to me.

Quoted 27 lines
---
 include/linux/bpf.h   |  2 ++
 kernel/bpf/core.c     | 21 ++++++++++--
 kernel/bpf/verifier.c | 79 +++++++++++++------------------------------
 3 files changed, 44 insertions(+), 58 deletions(-)
include/linux/bpf.h +2 -0
index 520b238abd5a..e521eae334ea 100644
@@ -2234,6 +2234,8 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
 const struct btf_func_model *
 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
                         const struct bpf_insn *insn);
+int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 offset,
+                      u8 **func_addr);
 struct bpf_core_ctx {
        struct bpf_verifier_log *log;
        const struct btf *btf;
kernel/bpf/core.c +0 -0
index 933869983e2a..4d51782f17ab 100644
@@ -1185,10 +1185,12 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 {
        s16 off = insn->off;
        s32 imm = insn->imm;

[..]

Quoted 1 line

+ bool fixed;

nit: do we really need that extra fixed bool? Why not directly *func_addr_fixes = true/false in all the places?

Quoted 50 lines
        u8 *addr;
kernel/bpf/core.c +17 -3
+       int err;
 
-       *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL;
-       if (!*func_addr_fixed) {
+       switch (insn->src_reg) {
+       case BPF_PSEUDO_CALL:
                /* Place-holder address till the last pass has collected
                 * all addresses for JITed subprograms in which case we
                 * can pick them up from prog->aux.
@@ -1200,15 +1202,28 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
                        addr = (u8 *)prog->aux->func[off]->bpf_func;
                else
                        return -EINVAL;
-       } else {
+               fixed = false;
+               break;
+       case 0:
                /* Address of a BPF helper call. Since part of the core
                 * kernel, it's always at a fixed location. __bpf_call_base
                 * and the helper with imm relative to it are both in core
                 * kernel.
                 */
                addr = (u8 *)__bpf_call_base + imm;
+               fixed = true;
+               break;
+       case BPF_PSEUDO_KFUNC_CALL:
+               err = bpf_get_kfunc_addr(prog, imm, off, &addr);
+               if (err)
+                       return err;
+               fixed = true;
+               break;
+       default:
+               return -EINVAL;
        }
 
+       *func_addr_fixed = fixed;
        *func_addr = (unsigned long)addr;
        return 0;
 }
kernel/bpf/verifier.c +0 -1
index 574d2dfc6ada..6d4632476c9c 100644
@@ -2115,8 +2115,8 @@ static int add_subprog(struct bpf_verifier_env *env, int off)
 struct bpf_kfunc_desc {
        struct btf_func_model func_model;
        u32 func_id;
-       s32 imm;
        u16 offset;

[..]

Quoted 1 line

+ unsigned long addr;

Do we have some canonical type to store the address? I was using void
* in bpf_dev_bound_resolve_kfunc, but we are doing ulong here. We seem to be doing u64/void */unsigned long inconsistently.

Also, maybe move it up a bit? To turn u32+u16+gap+u64 into u64+u32+u16+padding ?

Quoted 162 lines

};

struct bpf_kfunc_btf {

kernel/bpf/verifier.c +23 -55
@@ -2166,6 +2166,19 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
                       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
 }
 
+int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 offset,
+                      u8 **func_addr)
+{
+       const struct bpf_kfunc_desc *desc;
+
+       desc = find_kfunc_desc(prog, func_id, offset);
+       if (!desc)
+               return -EFAULT;
+
+       *func_addr = (u8 *)desc->addr;
+       return 0;
+}
+
 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
                                         s16 offset)
 {
@@ -2261,8 +2274,8 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
        struct bpf_kfunc_desc *desc;
        const char *func_name;
        struct btf *desc_btf;
-       unsigned long call_imm;
        unsigned long addr;
+       void *xdp_kfunc;
        int err;
 
        prog_aux = env->prog->aux;
@@ -2346,24 +2359,21 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
                return -EINVAL;
        }
 
-       call_imm = BPF_CALL_IMM(addr);
-       /* Check whether or not the relative offset overflows desc->imm */
-       if ((unsigned long)(s32)call_imm != call_imm) {
-               verbose(env, "address of kernel function %s is out of range\n",
-                       func_name);
-               return -EINVAL;
-       }
-
        if (bpf_dev_bound_kfunc_id(func_id)) {
                err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
                if (err)
                        return err;
+
+               xdp_kfunc = bpf_dev_bound_resolve_kfunc(env->prog, func_id);
+               if (xdp_kfunc)
+                       addr = (unsigned long)xdp_kfunc;
+               /* fallback to default kfunc when not supported by netdev */
        }
 
        desc = &tab->descs[tab->nr_descs++];
        desc->func_id = func_id;
-       desc->imm = call_imm;
        desc->offset = offset;
+       desc->addr = addr;
        err = btf_distill_func_proto(&env->log, desc_btf,
                                     func_proto, func_name,
                                     &desc->func_model);
@@ -2373,30 +2383,6 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
        return err;
 }
 
-static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
-{
-       const struct bpf_kfunc_desc *d0 = a;
-       const struct bpf_kfunc_desc *d1 = b;
-
-       if (d0->imm > d1->imm)
-               return 1;
-       else if (d0->imm < d1->imm)
-               return -1;
-       return 0;
-}
-
-static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
-{
-       struct bpf_kfunc_desc_tab *tab;
-
-       tab = prog->aux->kfunc_tab;
-       if (!tab)
-               return;
-
-       sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
-            kfunc_desc_cmp_by_imm, NULL);
-}
-
 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
 {
        return !!prog->aux->kfunc_tab;
@@ -2407,14 +2393,15 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
                         const struct bpf_insn *insn)
 {
        const struct bpf_kfunc_desc desc = {
-               .imm = insn->imm,
+               .func_id = insn->imm,
+               .offset = insn->off,
        };
        const struct bpf_kfunc_desc *res;
        struct bpf_kfunc_desc_tab *tab;
 
        tab = prog->aux->kfunc_tab;
        res = bsearch(&desc, tab->descs, tab->nr_descs,
-                     sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
+                     sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
 
        return res ? &res->func_model : NULL;
 }
@@ -16267,7 +16254,6 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                            struct bpf_insn *insn_buf, int insn_idx, int *cnt)
 {
        const struct bpf_kfunc_desc *desc;
-       void *xdp_kfunc;
 
        if (!insn->imm) {
                verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
@@ -16275,20 +16261,6 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
        }
 
        *cnt = 0;
-
-       if (bpf_dev_bound_kfunc_id(insn->imm)) {
-               xdp_kfunc = bpf_dev_bound_resolve_kfunc(env->prog, insn->imm);
-               if (xdp_kfunc) {
-                       insn->imm = BPF_CALL_IMM(xdp_kfunc);
-                       return 0;
-               }
-
-               /* fallback to default kfunc when not supported by netdev */
-       }
-
-       /* insn->imm has the btf func_id. Replace it with
-        * an address (relative to __bpf_call_base).
-        */
        desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
        if (!desc) {
                verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
@@ -16296,7 +16268,6 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                return -EFAULT;
        }
 
-       insn->imm = desc->imm;
        if (insn->off)
                return 0;
        if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) {
@@ -16850,8 +16821,6 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
                }
        }
 
-       sort_kfunc_descs_by_imm(env->prog);
-
        return 0;
 }
 
--

2.39.1

#14 lore raw
Quoted 69 lines · Stanislav Fomichev
Quoted 23 lines · Ilya Leoshkevich

test_ksyms_module fails to emit a kfunc call targeting a module on s390x, because the verifier stores the difference between kfunc address and __bpf_call_base in bpf_insn.imm, which is s32, and modules
are roughly (1 << 42) bytes away from the kernel on s390x.

Fix by keeping BTF id in bpf_insn.imm for BPF_PSEUDO_KFUNC_CALLs, and storing the absolute address in bpf_kfunc_desc, which JITs retrieve
as usual by calling bpf_jit_get_func_addr().

Introduce bpf_get_kfunc_addr() instead of exposing both find_kfunc_desc() and struct bpf_kfunc_desc.

This also fixes the problem with XDP metadata functions outlined in the description of commit 63d7b53ab59f ("s390/bpf: Implement bpf_jit_supports_kfunc_call()") by replacing address lookups with BTF
id lookups. This eliminates the inconsistency between "abstract" XDP
metadata functions' BTF ids and their concrete addresses.

Quoted 1 line

With a nit below (and an unrelated question).

I'll wait a bit for the buildbots to finish until ack'ing the rest. But the jit (except sparc quirks) and selftest changes also make sense to me.

Quoted 31 lines

---
 include/linux/bpf.h   |  2 ++
 kernel/bpf/core.c     | 21 ++++++++++-- kernel/bpf/verifier.c | 79 +++++++++++++--------------------------
----

 3 files changed, 44 insertions(+), 58 deletions(-)
include/linux/bpf.h +0 -0
index 520b238abd5a..e521eae334ea 100644
@@ -2234,6 +2234,8 @@ bool bpf_prog_has_kfunc_call(const struct
 bpf_prog *prog);

 const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); +int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 offset,
+                      u8 **func_addr);
 struct bpf_core_ctx {
        struct bpf_verifier_log *log;
        const struct btf *btf;

kernel/bpf/core.c +0 -0
index 933869983e2a..4d51782f17ab 100644
@@ -1185,10 +1185,12 @@ int bpf_jit_get_func_addr(const struct
 bpf_prog *prog,
  {
         s16 off = insn->off;
         s32 imm = insn->imm;

[..]

Quoted 4 lines
Quoted 1 line

+       bool fixed;

nit: do we really need that extra fixed bool? Why not directly *func_addr_fixes = true/false in all the places?

I introduced it in order to avoid touching func_addr_fixed if there is an error, but actually that's not necessary - it's assigned after all checks. I will drop it in v4.

Quoted 61 lines
Quoted 58 lines

        u8 *addr;
+       int err;

-       *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; -       if (!*func_addr_fixed) {
+       switch (insn->src_reg) {
+       case BPF_PSEUDO_CALL:
                /* Place-holder address till the last pass has collected
                 * all addresses for JITed subprograms in which case we
                 * can pick them up from prog->aux.

kernel/bpf/core.c +0 -0
@@ -1200,15 +1202,28 @@ int bpf_jit_get_func_addr(const struct
 bpf_prog *prog,

                        addr = (u8 *)prog->aux->func[off]-

Quoted 1 line

bpf_func;

                else
                        return -EINVAL;
-       } else {
+               fixed = false;
+               break;
+       case 0:
                /* Address of a BPF helper call. Since part of the core
                 * kernel, it's always at a fixed location. __bpf_call_base
                 * and the helper with imm relative to it are both in core
                 * kernel.
                 */
                addr = (u8 *)__bpf_call_base + imm;
+               fixed = true;
+               break;
+       case BPF_PSEUDO_KFUNC_CALL:
+               err = bpf_get_kfunc_addr(prog, imm, off, &addr); +               if (err)
+                       return err;
+               fixed = true;
+               break;
+       default:
+               return -EINVAL;
        }

+       *func_addr_fixed = fixed;
        *func_addr = (unsigned long)addr;
        return 0;
 }

kernel/bpf/verifier.c +0 -1
index 574d2dfc6ada..6d4632476c9c 100644
@@ -2115,8 +2115,8 @@ static int add_subprog(struct
 bpf_verifier_env *env, int off)
  struct bpf_kfunc_desc {
         struct btf_func_model func_model;
         u32 func_id;
-       s32 imm;
         u16 offset;

[..]

Quoted 6 lines
Quoted 1 line

+       unsigned long addr;

Do we have some canonical type to store the address? I was using void
* in bpf_dev_bound_resolve_kfunc, but we are doing ulong here. We seem
to be doing u64/void */unsigned long inconsistently.

IIUC u64 is for BPF progs [1]. I've seen unsigned long in a number of places, e.g. kallsyms. My personal heuristic is that if we don't dereference it on the C side, it can be unsigned long. But I don't have a strong opinion on this.

Quoted 2 lines

Also, maybe move it up a bit? To turn u32+u16+gap+u64 into u64+u32+u16+padding ?

You are right, we can do better here w.r.t. space efficiency:

struct bpf_kfunc_desc {
struct btf_func_model func_model; /* 0 27
*/

/* XXX 1 byte hole, try to pack */

u32 func_id; /* 28 4
*/
u16 offset; /* 32 2
*/

/* XXX 6 bytes hole, try to pack */

long unsigned int addr; /* 40 8
*/

[1]
https://lore.kernel.org/bpf/CAEf4BzaQJfB0Qh2Wn5wd9H0ZSURbzWBfKkav8xbkhozqTWXndw@mail.gmail.com/

Best regards,
Ilya

#9 lore raw

Re: [PATCH bpf-next v3 08/12] bpf: sparc64: Use bpf_jit_get_func_addr()

In reply to Ilya Leoshkevich
Quoted 17 lines · Ilya Leoshkevich

Preparation for moving kfunc address from bpf_insn.imm.

---
 arch/sparc/net/bpf_jit_comp_64.c | 20 ++++++++++++++------
 arch/x86/net/bpf_jit_comp32.c    |  7 +++++++
 2 files changed, 21 insertions(+), 6 deletions(-)
arch/sparc/net/bpf_jit_comp_64.c +0 -0
index 6c482685dc6c..b23083776718 100644
@@ -893,7 +893,8 @@ static void emit_tail_call(struct jit_ctx *ctx)
 	emit_nop(ctx);
 }
 

SNIP

Quoted 7 lines
arch/x86/net/bpf_jit_comp32.c +0 -0
index 429a89c5468b..0abb4d6c9dec 100644
@@ -2091,6 +2091,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			if (insn->src_reg == BPF_PSEUDO_CALL)
 				goto notyet;

arch/x86/net/bpf_jit_comp32.c static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · line 2102

Quoted 10 lines
arch/x86/net/bpf_jit_comp32.c +7 -0
 
+			err = bpf_jit_get_func_addr(bpf_prog, insn, extra_pass,
+						    &func_addr,
+						    &func_addr_fixed);
+			if (err)
+				return err;
+			func = (u8 *)(unsigned long)func_addr;
+
 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 				int err;

looks like this hunk should be in:
bpf, x86_32: Use bpf_jit_get_func_addr

jirka

#16 lore raw
In reply to Ilya Leoshkevich
Quoted 8 lines · Ilya Leoshkevich

v2: https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
Drop the merged check_subprogs() cleanup.
Adjust arm, sparc and i386 JITs.
Fix a few portability issues in test_verifier.
Fix a few sparc64 issues.
Trim s390x denylist.

I don't think it's a good idea to change a bunch of JITs that you cannot test just to address the s390 issue. Please figure out an approach that none of the JITs need changes.

#17 lore raw
Quoted 12 lines · Alexei Starovoitov
Quoted 9 lines · Ilya Leoshkevich

v2:
https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
          Drop the merged check_subprogs() cleanup. Adjust arm, sparc and i386 JITs.
          Fix a few portability issues in test_verifier. Fix a few sparc64 issues.
          Trim s390x denylist.

Quoted 3 lines

I don't think it's a good idea to change a bunch of JITs that you cannot test just to address the s390 issue. Please figure out an approach that none of the JITs need changes.

What level of testing for these JITs would you find acceptable?

#18 lore raw
Quoted 14 lines · Ilya Leoshkevich
Quoted 12 lines · Alexei Starovoitov
Quoted 9 lines · Ilya Leoshkevich

v2:
https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
Drop the merged check_subprogs() cleanup. Adjust arm, sparc and i386 JITs.
Fix a few portability issues in test_verifier. Fix a few sparc64 issues.
Trim s390x denylist.

Quoted 5 lines
Quoted 3 lines

I don't think it's a good idea to change a bunch of JITs that you cannot test just to address the s390 issue. Please figure out an approach that none of the JITs need changes.

What level of testing for these JITs would you find acceptable?

Just find a way to avoid changing them.

#19 lore raw
Quoted 17 lines · Alexei Starovoitov
Quoted 15 lines · Ilya Leoshkevich
Quoted 13 lines · Alexei Starovoitov
Quoted 9 lines · Ilya Leoshkevich

v2:
https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
          Drop the merged check_subprogs() cleanup. Adjust arm, sparc and i386 JITs.
          Fix a few portability issues in test_verifier. Fix a few sparc64 issues.
          Trim s390x denylist.

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines

I don't think it's a good idea to change a bunch of JITs that you cannot test just to address the s390 issue. Please figure out an approach that none of the JITs need changes.

What level of testing for these JITs would you find acceptable?

Just find a way to avoid changing them.

Ok. But please take a look at patches 1-6. They fix existing issues, which were found by running test_verifier on arm and sparc64.

#20 lore raw
In reply to Alexei Starovoitov
Quoted 15 lines · Alexei Starovoitov
Quoted 14 lines · Ilya Leoshkevich
Quoted 12 lines · Alexei Starovoitov
Quoted 9 lines · Ilya Leoshkevich

v2:
https://lore.kernel.org/bpf/[email protected]/
v2 -> v3: Drop BPF_HELPER_CALL (Alexei).
Drop the merged check_subprogs() cleanup. Adjust arm, sparc and i386 JITs.
Fix a few portability issues in test_verifier. Fix a few sparc64 issues.
Trim s390x denylist.

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines

I don't think it's a good idea to change a bunch of JITs that you cannot test just to address the s390 issue. Please figure out an approach that none of the JITs need changes.

What level of testing for these JITs would you find acceptable?

Just find a way to avoid changing them.

hi,
sending another stub on this

the idea is to use 'func_id' in insn->imm for s390 arch and keep other archs to use the current BPF_CALL_IMM(addr) value

this way the s390 arch is able to lookup the kfunc_desc and use the stored kfunc address

I added insn->off to the kfunc_desc sorting, which is not needed for !__s390__ case, but it won't hurt... we can have that separated as well if needed

the patch below is completely untested on s390x of course, but it does not seem to break x86 ;-)

I think we could have config option for that instead of using __s390x__

thoughts?

thanks,
jirka

---

include/linux/bpf.h +2 -0
index 2d8f3f639e68..b60945e135ee 100644
@@ -2296,6 +2296,8 @@ bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
 const struct btf_func_model *
 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 			 const struct bpf_insn *insn);
+int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 offset,
+		       u8 **func_addr);
 struct bpf_core_ctx {
 	struct bpf_verifier_log *log;
 	const struct btf *btf;
kernel/bpf/core.c +18 -3
index b297e9f60ca1..06459df0a8c0 100644
@@ -1186,10 +1186,12 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 {
 	s16 off = insn->off;
 	s32 imm = insn->imm;
+	bool fixed;
 	u8 *addr;
+	int err;
 
-	*func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL;
-	if (!*func_addr_fixed) {
+	switch (insn->src_reg) {
+	case BPF_PSEUDO_CALL:
 		/* Place-holder address till the last pass has collected
 		 * all addresses for JITed subprograms in which case we
 		 * can pick them up from prog->aux.
@@ -1201,15 +1203,28 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 			addr = (u8 *)prog->aux->func[off]->bpf_func;
 		else
 			return -EINVAL;
-	} else {
+		fixed = false;
+		break;
+	case 0:
 		/* Address of a BPF helper call. Since part of the core
 		 * kernel, it's always at a fixed location. __bpf_call_base
 		 * and the helper with imm relative to it are both in core
 		 * kernel.
 		 */
 		addr = (u8 *)__bpf_call_base + imm;
+		fixed = true;
+		break;
+	case BPF_PSEUDO_KFUNC_CALL:
+		err = bpf_get_kfunc_addr(prog, imm, off, &addr);
+		if (err)
+			return err;
+		fixed = true;
+		break;
+	default:
+		return -EINVAL;
 	}
 
+	*func_addr_fixed = fixed;
 	*func_addr = (unsigned long)addr;
 	return 0;
 }
kernel/bpf/verifier.c +36 -10

117 lines not on this page · raw message