Skip to contentSkip to conversation

Attach a cookie to a tracing program.

PATCH Superseded Kui-Feng Lee 3 participants 19 messages May 8, 2022 – May 10, 2022

Subsystems (12)
Networking x86 Kselftest BPF +8 more subsystems

v7 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

v7 cover Attach a cookie to a tracing program.

Allow users to attach a 64-bits cookie to a bpf_link of fentry, fexit, or fmod_ret.

This patchset includes several major changes.

- Define struct bpf_tramp_links to replace bpf_tramp_prog. struct bpf_tramp_links collects bpf_links of a trampoline

- Generate a trampoline to call bpf_progs of given bpf_links.

- Trampolines always set/reset bpf_run_ctx before/after calling/leaving a tracing program.

- Attach a cookie to a bpf_link of fentry/fexit/fmod_ret/lsm. The value will be available when running the associated bpf_prog.

Th major differences from v6:

- bpf_link_create() can create links of BPF_LSM_MAC attach type.

- Add a test for lsm.

- Add function proto of bpf_get_attach_cookie() for lsm.

- Check BPF_LSM_MAC in bpf_prog_has_trampoline().

- Adapt to the changes of LINK_CREATE made by Andrii.

v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/
v4: https://lore.kernel.org/bpf/[email protected]/
v5: https://lore.kernel.org/bpf/[email protected]/
v6: https://lore.kernel.org/bpf/[email protected]/

Kui-Feng Lee (5):
bpf, x86: Generate trampolines from bpf_tramp_links bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.
libbpf: Assign cookies to links in libbpf. selftest/bpf: The test cses of BPF cookie for
fentry/fexit/fmod_ret/lsm.

 arch/x86/net/bpf_jit_comp.c                   |  82 +++++++++++---
 include/linux/bpf.h                           |  54 +++++++---
 include/linux/bpf_types.h                     |   1 +
 include/uapi/linux/bpf.h                      |  10 ++
 kernel/bpf/bpf_lsm.c                          |  17 +++
 kernel/bpf/bpf_struct_ops.c                   |  69 ++++++++----
 kernel/bpf/syscall.c                          |  42 ++++----
 kernel/bpf/trampoline.c                       | 100 +++++++++++-------
 kernel/trace/bpf_trace.c                      |  17 +++
 net/bpf/bpf_dummy_struct_ops.c                |  36 ++++++-
 tools/bpf/bpftool/link.c                      |   1 +
 tools/include/uapi/linux/bpf.h                |  10 ++
 tools/lib/bpf/bpf.c                           |   8 ++
 tools/lib/bpf/bpf.h                           |   3 +
 tools/lib/bpf/libbpf.c                        |  32 ++++++
 tools/lib/bpf/libbpf.h                        |  12 +++
 tools/lib/bpf/libbpf.map                      |   1 +
 .../selftests/bpf/prog_tests/bpf_cookie.c     |  89 ++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     |  52 +++++++--
 19 files changed, 509 insertions(+), 127 deletions(-)
Signature · 1 line
2.30.2
#1 lore raw

PATCH v7 1/5 bpf, x86: Generate trampolines from bpf_tramp_links

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++--------
 include/linux/bpf.h            | 36 +++++++++++------
 include/linux/bpf_types.h      |  1 +
 include/uapi/linux/bpf.h       |  1 +
 kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++----------
 kernel/bpf/syscall.c           | 23 ++++-------
 kernel/bpf/trampoline.c        | 73 +++++++++++++++++++---------------
 net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++---
 tools/bpf/bpftool/link.c       |  1 +
 tools/include/uapi/linux/bpf.h |  1 +
 10 files changed, 174 insertions(+), 103 deletions(-)
arch/x86/net/bpf_jit_comp.c +19 -17
index 8fe35ed11fd6..4dcc0b1ac770 100644
@@ -1761,10 +1761,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 }
 
 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
-			   struct bpf_prog *p, int stack_size, bool save_ret)
+			   struct bpf_tramp_link *l, int stack_size,
+			   bool save_ret)
 {
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
+	struct bpf_prog *p = l->link.prog;
 
 	/* arg1: mov rdi, progs[i] */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
@@ -1849,14 +1851,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
 }
 
 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
-		      struct bpf_tramp_progs *tp, int stack_size,
+		      struct bpf_tramp_links *tl, int stack_size,
 		      bool save_ret)
 {
 	int i;
 	u8 *prog = *pprog;
 
-	for (i = 0; i < tp->nr_progs; i++) {
-		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,
+	for (i = 0; i < tl->nr_links; i++) {
+		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
 				    save_ret))
 			return -EINVAL;
 	}
@@ -1865,7 +1867,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 }
 
 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
-			      struct bpf_tramp_progs *tp, int stack_size,
+			      struct bpf_tramp_links *tl, int stack_size,
 			      u8 **branches)
 {
 	u8 *prog = *pprog;
@@ -1876,8 +1878,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
 	 */
 	emit_mov_imm32(&prog, false, BPF_REG_0, 0);
 	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
-	for (i = 0; i < tp->nr_progs; i++) {
-		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))
+	for (i = 0; i < tl->nr_links; i++) {
+		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))
 			return -EINVAL;
 
 		/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1979,14 +1981,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)
  */
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
 				const struct btf_func_model *m, u32 flags,
-				struct bpf_tramp_progs *tprogs,
+				struct bpf_tramp_links *tlinks,
 				void *orig_call)
 {
 	int ret, i, nr_args = m->nr_args;
 	int regs_off, ip_off, args_off, stack_size = nr_args * 8;
-	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
-	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
-	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
+	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
+	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
+	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
 	u8 **branches = NULL;
 	u8 *prog;
 	bool save_ret;
@@ -2077,13 +2079,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		}
 	}
 
-	if (fentry->nr_progs)
+	if (fentry->nr_links)
 		if (invoke_bpf(m, &prog, fentry, regs_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET))
 			return -EINVAL;
 
-	if (fmod_ret->nr_progs) {
-		branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),
+	if (fmod_ret->nr_links) {
+		branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),
 				   GFP_KERNEL);
 		if (!branches)
 			return -ENOMEM;
@@ -2110,7 +2112,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		prog += X86_PATCH_SIZE;
 	}
 
-	if (fmod_ret->nr_progs) {
+	if (fmod_ret->nr_links) {
 		/* From Intel 64 and IA-32 Architectures Optimization
 		 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
 		 * Coding Rule 11: All branch targets should be 16-byte
@@ -2120,12 +2122,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		/* Update the branches saved in invoke_bpf_mod_ret with the
 		 * aligned address of do_fexit.
 		 */
-		for (i = 0; i < fmod_ret->nr_progs; i++)
+		for (i = 0; i < fmod_ret->nr_links; i++)
 			emit_cond_near_jump(&branches[i], prog, branches[i],
 					    X86_JNE);
 	}
 
-	if (fexit->nr_progs)
+	if (fexit->nr_links)
 		if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
 			ret = -EINVAL;
 			goto cleanup;
include/linux/bpf.h +24 -12
index be94833d390a..77258a34ec20 100644
@@ -723,11 +723,11 @@ struct btf_func_model {
 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
  * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2
  */
-#define BPF_MAX_TRAMP_PROGS 38
+#define BPF_MAX_TRAMP_LINKS 38
 
-struct bpf_tramp_progs {
-	struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
-	int nr_progs;
+struct bpf_tramp_links {
+	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
+	int nr_links;
 };
 
 /* Different use cases for BPF trampoline:
@@ -753,7 +753,7 @@ struct bpf_tramp_progs {
 struct bpf_tramp_image;
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
 				const struct btf_func_model *m, u32 flags,
-				struct bpf_tramp_progs *tprogs,
+				struct bpf_tramp_links *tlinks,
 				void *orig_call);
 /* these two functions are called from generated trampoline */
 u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
 {
 	return bpf_func(ctx, insnsi);
 }
+
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
 					  struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 					   struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
 					     struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
@@ -1009,7 +1010,6 @@ struct bpf_prog_aux {
 	bool tail_call_reachable;
 	bool xdp_has_frags;
 	bool use_bpf_prog_pack;
-	struct hlist_node tramp_hlist;
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
 	const struct btf_type *attach_func_proto;
 	/* function name for valid attach_btf_id */
@@ -1096,6 +1096,18 @@ struct bpf_link_ops {
 			      struct bpf_link_info *info);
 };
 
+struct bpf_tramp_link {
+	struct bpf_link link;
+	struct hlist_node tramp_hlist;
+};
+
+struct bpf_tracing_link {
+	struct bpf_tramp_link link;
+	enum bpf_attach_type attach_type;
+	struct bpf_trampoline *trampoline;
+	struct bpf_prog *tgt_prog;
+};
+
 struct bpf_link_primer {
 	struct bpf_link *link;
 	struct file *file;
@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata);
 void bpf_struct_ops_put(const void *kdata);
 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
 				       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-				      struct bpf_prog *prog,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+				      struct bpf_tramp_link *link,
 				      const struct btf_func_model *model,
 				      void *image, void *image_end);
 static inline bool bpf_try_module_get(const void *data, struct module *owner)
include/linux/bpf_types.h +1 -0
index 3e24ad0c4b3c..2b9112b80171 100644
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
 BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
 #endif
 BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
+BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)
include/uapi/linux/bpf.h +1 -0
index 444fe6f1cf35..ff9af73859ca 100644
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
 	BPF_LINK_TYPE_XDP = 6,
 	BPF_LINK_TYPE_PERF_EVENT = 7,
 	BPF_LINK_TYPE_KPROBE_MULTI = 8,
+	BPF_LINK_TYPE_STRUCT_OPS = 9,
 
 	MAX_BPF_LINK_TYPE,
 };
kernel/bpf/bpf_struct_ops.c +47 -22

153 lines not on this page · raw message

kernel/bpf/syscall.c +8 -15
index e0aead17dff4..3c1853e1c715 100644
@@ -2864,19 +2864,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd)
 }
 EXPORT_SYMBOL(bpf_link_get_from_fd);
 
-struct bpf_tracing_link {
-	struct bpf_link link;
-	enum bpf_attach_type attach_type;
-	struct bpf_trampoline *trampoline;
-	struct bpf_prog *tgt_prog;
-};
-
 static void bpf_tracing_link_release(struct bpf_link *link)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
-	WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
+	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
 						tr_link->trampoline));
 
 	bpf_trampoline_put(tr_link->trampoline);
@@ -2889,7 +2882,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
 static void bpf_tracing_link_dealloc(struct bpf_link *link)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	kfree(tr_link);
 }
@@ -2898,7 +2891,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
 					 struct seq_file *seq)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	seq_printf(seq,
 		   "attach_type:\t%d\n",
@@ -2909,7 +2902,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
 					   struct bpf_link_info *info)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	info->tracing.attach_type = tr_link->attach_type;
 	bpf_trampoline_unpack_key(tr_link->trampoline->key,
@@ -2990,7 +2983,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 		err = -ENOMEM;
 		goto out_put_prog;
 	}
-	bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
+	bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
 		      &bpf_tracing_link_lops, prog);
 	link->attach_type = prog->expected_attach_type;
 
@@ -3060,11 +3053,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 		tgt_prog = prog->aux->dst_prog;
 	}
 
-	err = bpf_link_prime(&link->link, &link_primer);
+	err = bpf_link_prime(&link->link.link, &link_primer);
 	if (err)
 		goto out_unlock;
 
-	err = bpf_trampoline_link_prog(prog, tr);
+	err = bpf_trampoline_link_prog(&link->link, tr);
 	if (err) {
 		bpf_link_cleanup(&link_primer);
 		link = NULL;
kernel/bpf/trampoline.c +41 -32

171 lines not on this page · raw message

net/bpf/bpf_dummy_struct_ops.c +31 -5
index d0e54e30658a..41552d6f1d23 100644
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args)
 		    args->args[3], args->args[4]);
 }
 
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+	kfree(link);
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+	.release = bpf_struct_ops_link_release,
+	.dealloc = bpf_struct_ops_link_dealloc,
+};
+
 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 			    union bpf_attr __user *uattr)
 {
 	const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;
 	const struct btf_type *func_proto;
 	struct bpf_dummy_ops_test_args *args;
-	struct bpf_tramp_progs *tprogs;
+	struct bpf_tramp_links *tlinks;
+	struct bpf_tramp_link *link = NULL;
 	void *image = NULL;
 	unsigned int op_idx;
 	int prog_ret;
@@ -92,8 +107,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 	if (IS_ERR(args))
 		return PTR_ERR(args);
 
-	tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
-	if (!tprogs) {
+	tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+	if (!tlinks) {
 		err = -ENOMEM;
 		goto out;
 	}
@@ -105,8 +120,17 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 	}
 	set_vm_flush_reset_perms(image);
 
+	link = kzalloc(sizeof(*link), GFP_USER);
+	if (!link) {
+		err = -ENOMEM;
+		goto out;
+	}
+	/* prog doesn't take the ownership of the reference from caller */
+	bpf_prog_inc(prog);
+	bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
+
 	op_idx = prog->expected_attach_type;
-	err = bpf_struct_ops_prepare_trampoline(tprogs, prog,
+	err = bpf_struct_ops_prepare_trampoline(tlinks, link,
 						&st_ops->func_models[op_idx],
 						image, image + PAGE_SIZE);
 	if (err < 0)
@@ -124,7 +148,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 out:
 	kfree(args);
 	bpf_jit_free_exec(image);
-	kfree(tprogs);
+	if (link)
+		bpf_link_put(&link->link);
+	kfree(tlinks);
 	return err;
 }
 
tools/bpf/bpftool/link.c +1 -0
index 8fb0116f9136..6353a789322b 100644
@@ -23,6 +23,7 @@ static const char * const link_type_name[] = {
 	[BPF_LINK_TYPE_XDP]			= "xdp",
 	[BPF_LINK_TYPE_PERF_EVENT]		= "perf_event",
 	[BPF_LINK_TYPE_KPROBE_MULTI]		= "kprobe_multi",
+	[BPF_LINK_TYPE_STRUCT_OPS]               = "struct_ops",
 };
 
 static struct hashmap *link_table;
tools/include/uapi/linux/bpf.h +1 -0
index 444fe6f1cf35..ff9af73859ca 100644
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
 	BPF_LINK_TYPE_XDP = 6,
 	BPF_LINK_TYPE_PERF_EVENT = 7,
 	BPF_LINK_TYPE_KPROBE_MULTI = 8,
+	BPF_LINK_TYPE_STRUCT_OPS = 9,
 
 	MAX_BPF_LINK_TYPE,
 };
Signature · 1 line
2.30.2
Acked-by on v7 1/5
#2 lore raw

Re: [PATCH bpf-next v7 1/5] bpf, x86: Generate trampolines from bpf_tramp_links

Quoted 12 lines · Kui-Feng Lee

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

Quoted 13 lines
---
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++--------
 include/linux/bpf.h            | 36 +++++++++++------
 include/linux/bpf_types.h      |  1 +
 include/uapi/linux/bpf.h       |  1 +
 kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++----------
 kernel/bpf/syscall.c           | 23 ++++-------
 kernel/bpf/trampoline.c        | 73 +++++++++++++++++++---------------
 net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++---
 tools/bpf/bpftool/link.c       |  1 +
 tools/include/uapi/linux/bpf.h |  1 +
 10 files changed, 174 insertions(+), 103 deletions(-)

Two things that can be done as a follow up, otherwise LGTM:

[...]

kernel/bpf/bpf_struct_ops.c static int check_zero_holes(const struct btf_type *t, void *data) · line 325

Quoted 9 lines
kernel/bpf/bpf_struct_ops.c +7 -2
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog,
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+       kfree(link);

This works by accident because struct bpf_link is at the top of struct bpf_tramp_link. But to do this properly you'd need container_of() to get struct bpf_tramp_link and then free that. I don't think it needs a respin just for this, but please send a follow-up fix.

Quoted 7 lines
kernel/bpf/bpf_struct_ops.c +7 -0
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+       .release = bpf_struct_ops_link_release,
+       .dealloc = bpf_struct_ops_link_dealloc,
+};
+

[...]

Quoted 12 lines
net/bpf/bpf_dummy_struct_ops.c +4 -0
index d0e54e30658a..41552d6f1d23 100644
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args)
                    args->args[3], args->args[4]);
 }
 
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+

net/bpf/bpf_dummy_struct_ops.c static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) · line 88

Quoted 10 lines
net/bpf/bpf_dummy_struct_ops.c +10 -0
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+       kfree(link);
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+       .release = bpf_struct_ops_link_release,
+       .dealloc = bpf_struct_ops_link_dealloc,
+};
+

You already defined this ops struct and release/dealloc implementation in kernel/bpf/bpf_struct_ops.c, we need to reuse it here. Just make the bpf_struct_ops.c's non-static and declare it in include/linux/bpf.h. Again, don't think we need a respin just for this, it's mostly code hygiene.

Quoted 12 lines
net/bpf/bpf_dummy_struct_ops.c +2 -1
 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
                            union bpf_attr __user *uattr)
 {
        const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;
        const struct btf_type *func_proto;
        struct bpf_dummy_ops_test_args *args;
-       struct bpf_tramp_progs *tprogs;
+       struct bpf_tramp_links *tlinks;
+       struct bpf_tramp_link *link = NULL;
        void *image = NULL;
        unsigned int op_idx;
        int prog_ret;

[...]

#3 lore raw

Re: [PATCH bpf-next v7 1/5] bpf, x86: Generate trampolines from bpf_tramp_links

Quoted 45 lines · Andrii Nakryiko
Quoted 26 lines · Kui-Feng Lee

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline.  struct bpf_tramp_link extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++-------- include/linux/bpf.h            | 36 +++++++++++------ include/linux/bpf_types.h      |  1 + include/uapi/linux/bpf.h       |  1 + kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++-------- --
 kernel/bpf/syscall.c           | 23 ++++------- kernel/bpf/trampoline.c        | 73 +++++++++++++++++++-----------
----
 net/bpf/bpf_dummy_struct_ops.c | 36 ++++++++++++++--- tools/bpf/bpftool/link.c       |  1 + tools/include/uapi/linux/bpf.h |  1 +

Quoted 2 lines
 10 files changed, 174 insertions(+), 103 deletions(-)

Two things that can be done as a follow up, otherwise LGTM:

[...]

Quoted 9 lines

-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog,

kernel/bpf/bpf_struct_ops.c +6 -0
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
Quoted 9 lines
Quoted 1 line

+       kfree(link);

This works by accident because struct bpf_link is at the top of struct
bpf_tramp_link. But to do this properly you'd need container_of() to get struct bpf_tramp_link and then free that. I don't think it needs a
respin just for this, but please send a follow-up fix.

Fixed!

Quoted 33 lines
Quoted 7 lines
kernel/bpf/bpf_struct_ops.c +7 -0
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+       .release = bpf_struct_ops_link_release,
+       .dealloc = bpf_struct_ops_link_dealloc,
+};
+

[...]

Quoted 23 lines
net/bpf/bpf_dummy_struct_ops.c +13 -0
index d0e54e30658a..41552d6f1d23 100644
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image,
 struct bpf_dummy_ops_test_args *args)
                     args->args[3], args->args[4]);
  }
 
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+       kfree(link);
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+       .release = bpf_struct_ops_link_release,
+       .dealloc = bpf_struct_ops_link_dealloc,
+};
Quoted 8 lines

net/bpf/bpf_dummy_struct_ops.c static int dummy_ops_call_op(void *image, · line 89

Quoted 1 line
net/bpf/bpf_dummy_struct_ops.c +1 -0
+

You already defined this ops struct and release/dealloc implementation
in kernel/bpf/bpf_struct_ops.c, we need to reuse it here. Just make the bpf_struct_ops.c's non-static and declare it in include/linux/bpf.h. Again, don't think we need a respin just for this, it's mostly code hygiene.

Fixed!

#4 lore raw

PATCH v7 2/5 bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

In reply to Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---
 arch/x86/net/bpf_jit_comp.c | 38 +++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++++++----
 kernel/bpf/syscall.c        |  7 +++++--
 kernel/bpf/trampoline.c     | 20 +++++++++++++++----
 4 files changed, 72 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +38 -0
index 4dcc0b1ac770..bf4576a6938c 100644
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 {
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
+	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = l->link.prog;
 
+	/* mov rdi, 0 */
+	emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+
+	/* Prepare struct bpf_tramp_run_ctx.
+	 *
+	 * bpf_tramp_run_ctx is already preserved by
+	 * arch_prepare_bpf_trampoline().
+	 *
+	 * mov QWORD PTR [rsp + ctx_cookie_off], rdi
+	 */
+	EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off);
+
 	/* arg1: mov rdi, progs[i] */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
+	/* arg2: mov rsi, rsp (struct bpf_run_ctx *) */
+	EMIT3(0x48, 0x89, 0xE6);
+
 	if (emit_call(&prog,
 		      p->aux->sleepable ? __bpf_prog_enter_sleepable :
 		      __bpf_prog_enter, prog))
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
 	/* arg2: mov rsi, rbx <- start time in nsec */
 	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+	/* arg3: mov rdx, rsp (struct bpf_run_ctx *) */
+	EMIT3(0x48, 0x89, 0xE2);
 	if (emit_call(&prog,
 		      p->aux->sleepable ? __bpf_prog_exit_sleepable :
 		      __bpf_prog_exit, prog))
@@ -2079,6 +2097,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		}
 	}
 
+	/* Prepare struct bpf_tramp_run_ctx.
+	 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+	 */
+	EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+
 	if (fentry->nr_links)
 		if (invoke_bpf(m, &prog, fentry, regs_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET))
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		/* pop struct bpf_tramp_run_ctx
+		 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
 		restore_regs(m, &prog, nr_args, regs_off);
 
 		/* call original function */
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		im->ip_after_call = prog;
 		memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
 		prog += X86_PATCH_SIZE;
+
+		/* Prepare struct bpf_tramp_run_ctx.
+		 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
 	}
 
 	if (fmod_ret->nr_links) {
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 			goto cleanup;
 		}
 
+	/* pop struct bpf_tramp_run_ctx
+	 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+	 */
+	EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
 	if (flags & BPF_TRAMP_F_RESTORE_REGS)
 		restore_regs(m, &prog, nr_args, regs_off);
 
include/linux/bpf.h +13 -4
index 77258a34ec20..29c3188195a6 100644
@@ -730,6 +730,8 @@ struct bpf_tramp_links {
 	int nr_links;
 };
 
+struct bpf_tramp_run_ctx;
+
 /* Different use cases for BPF trampoline:
  * 1. replace nop at the function entry (kprobe equivalent)
  *    flags = BPF_TRAMP_F_RESTORE_REGS
@@ -756,10 +758,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *i
 				struct bpf_tramp_links *tlinks,
 				void *orig_call);
 /* these two functions are called from generated trampoline */
-u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
-u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
-void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
+u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);
+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
+				       struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
 
@@ -1351,6 +1354,12 @@ struct bpf_trace_run_ctx {
 	u64 bpf_cookie;
 };
 
+struct bpf_tramp_run_ctx {
+	struct bpf_run_ctx run_ctx;
+	u64 bpf_cookie;
+	struct bpf_run_ctx *saved_run_ctx;
+};
+
 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
 {
 	struct bpf_run_ctx *old_ctx = NULL;
kernel/bpf/syscall.c +5 -2
index 3c1853e1c715..5ed9a15daaee 100644
@@ -5001,6 +5001,7 @@ static bool syscall_prog_is_valid_access(int off, int size,
 BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 {
 	struct bpf_prog * __maybe_unused prog;
+	struct bpf_tramp_run_ctx __maybe_unused run_ctx;
 
 	switch (cmd) {
 	case BPF_MAP_CREATE:
@@ -5028,13 +5029,15 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 			return -EINVAL;
 		}
 
-		if (!__bpf_prog_enter_sleepable(prog)) {
+		run_ctx.bpf_cookie = 0;
+		run_ctx.saved_run_ctx = NULL;
+		if (!__bpf_prog_enter_sleepable(prog, &run_ctx)) {
 			/* recursion detected */
 			bpf_prog_put(prog);
 			return -EBUSY;
 		}
 		attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
-		__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */);
+		__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */, &run_ctx);
 		bpf_prog_put(prog);
 		return 0;
 #endif
kernel/bpf/trampoline.c +16 -4
index d5e6bc5517cb..baf1b65d523e 100644
@@ -568,11 +568,14 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)
  * [2..MAX_U64] - execute bpf prog and record execution time.
  *     This is start time.
  */
-u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
+u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 	__acquires(RCU)
 {
 	rcu_read_lock();
 	migrate_disable();
+
+	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+
 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
 		inc_misses_counter(prog);
 		return 0;
@@ -602,29 +605,38 @@ static void notrace update_prog_stats(struct bpf_prog *prog,
 	}
 }
 
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)
 	__releases(RCU)
 {
+	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+
 	update_prog_stats(prog, start);
 	__this_cpu_dec(*(prog->active));
 	migrate_enable();
 	rcu_read_unlock();
 }
 
-u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 {
 	rcu_read_lock_trace();
 	migrate_disable();
 	might_fault();
+
 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
 		inc_misses_counter(prog);
 		return 0;
 	}
+
+	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+
 	return bpf_prog_start_time();
 }
 
-void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
+				       struct bpf_tramp_run_ctx *run_ctx)
 {
+	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+
 	update_prog_stats(prog, start);
 	__this_cpu_dec(*(prog->active));
 	migrate_enable();
Signature · 1 line
2.30.2
Acked-by on v7 2/5
#5 lore raw

Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 7 lines · Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---

Please preserve received Acked-by/Reviewed-by/etc tags that you got on previous iterations, unless you feel like you did some major changes that might invalidate reviewer's "approval".

Still looks good to me:

Quoted 6 lines
 arch/x86/net/bpf_jit_comp.c | 38 +++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++++++----
 kernel/bpf/syscall.c        |  7 +++++--
 kernel/bpf/trampoline.c     | 20 +++++++++++++++----
 4 files changed, 72 insertions(+), 10 deletions(-)

[...]

#6 lore raw

Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

In reply to Kui-Feng Lee
Quoted 34 lines · Kui-Feng Lee
arch/x86/net/bpf_jit_comp.c +15 -0
+	/* Prepare struct bpf_tramp_run_ctx.
+	 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+	 */
+	EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+
 	if (fentry->nr_links)
 		if (invoke_bpf(m, &prog, fentry, regs_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET))
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		/* pop struct bpf_tramp_run_ctx
+		 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
 		restore_regs(m, &prog, nr_args, regs_off);
 
 		/* call original function */
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		im->ip_after_call = prog;
 		memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
 		prog += X86_PATCH_SIZE;
+
+		/* Prepare struct bpf_tramp_run_ctx.
+		 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
 	}
 
 	if (fmod_ret->nr_links) {

arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2173

Quoted 8 lines
arch/x86/net/bpf_jit_comp.c +5 -0
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 			goto cleanup;
 		}
 
+	/* pop struct bpf_tramp_run_ctx
+	 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+	 */
+	EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+

What is the point of all of these additional sub/add rsp ?
It seems unconditionally increasing stack_size by sizeof(struct bpf_tramp_run_ctx) will achieve the same and above 4 extra insns won't be needed.

#7 lore raw

Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 45 lines · Alexei Starovoitov
Quoted 44 lines · Kui-Feng Lee


+       /* Prepare struct bpf_tramp_run_ctx.
+        * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+        */
+       EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx)); +
        if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))

(quoted diff) +0 -0
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
        }

        if (flags & BPF_TRAMP_F_CALL_ORIG) {
+               /* pop struct bpf_tramp_run_ctx
+                * add rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
                restore_regs(m, &prog, nr_args, regs_off);
                /* call original function */

(quoted diff) +0 -0
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i im->ip_after_call = prog; memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE;
+
+               /* Prepare struct bpf_tramp_run_ctx. +                * sub rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
        }

        if (fmod_ret->nr_links) {

(quoted diff) +0 -0
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i goto cleanup;
                }

+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx)

Quoted 8 lines
Quoted 3 lines

+        */
+       EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx)); +

What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing stack_size by sizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.

I think you are right.

#8 lore raw

Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 58 lines · Kui-Feng Lee
Quoted 48 lines · Alexei Starovoitov
Quoted 47 lines · Kui-Feng Lee


+       /* Prepare struct bpf_tramp_run_ctx.
+        * sub rsp, sizeof(struct bpf_tramp_run_ctx) +        */
+       EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+
        if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))

(quoted diff) +0 -0
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
        }

        if (flags & BPF_TRAMP_F_CALL_ORIG) {
+               /* pop struct bpf_tramp_run_ctx
+                * add rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
                restore_regs(m, &prog, nr_args, regs_off);
                /* call original function */

(quoted diff) +0 -0
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i im->ip_after_call = prog; memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE;
+
+               /* Prepare struct bpf_tramp_run_ctx. +                * sub rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
        }

        if (fmod_ret->nr_links) {

(quoted diff) +0 -0
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i goto cleanup;
                }

+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx) +        */

Quoted 8 lines
Quoted 3 lines

+       EMIT4(0x48, 0x83, 0xC4, sizeof(struct
bpf_tramp_run_ctx));
+

What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing stack_size by sizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.

Quoted 2 lines

I think you are right.

The reason that I don't change stack_size is that we access arguments or saved registers basing on stack_size. Once the stack_size is changed, all these offsets should be changed too.

#9 lore raw

Re: [PATCH bpf-next v7 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 60 lines · Kui-Feng Lee
Quoted 58 lines · Kui-Feng Lee
Quoted 56 lines · Alexei Starovoitov
Quoted 48 lines · Kui-Feng Lee
arch/x86/net/bpf_jit_comp.c +4 -0
+       /* Prepare struct bpf_tramp_run_ctx.
+        * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+        */
+       EMIT4(0x48, 0x83, 0xEC, sizeof(struct

bpf_tramp_run_ctx));
+
if (fentry->nr_links)
if (invoke_bpf(m, &prog, fentry, regs_off,
flags &
BPF_TRAMP_F_RET_FENTRY_RET))

(quoted diff) +0 -0
@@ -2098,6 +2121,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i

        }

        if (flags & BPF_TRAMP_F_CALL_ORIG) {
arch/x86/net/bpf_jit_comp.c +4 -0
+               /* pop struct bpf_tramp_run_ctx
+                * add rsp, sizeof(struct bpf_tramp_run_ctx)
+                */
+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct

bpf_tramp_run_ctx));

arch/x86/net/bpf_jit_comp.c +1 -0
+
                restore_regs(m, &prog, nr_args, regs_off);
 
                /* call original function */
@@ -2110,6 +2138,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i

                im->ip_after_call = prog;
                memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
                prog += X86_PATCH_SIZE;
arch/x86/net/bpf_jit_comp.c +5 -0
+
+               /* Prepare struct bpf_tramp_run_ctx.
+                * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct
bpf_tramp_run_ctx));
        }

        if (fmod_ret->nr_links) {
(quoted diff) +0 -0
@@ -2133,6 +2166,11 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i

                        goto cleanup;
                }
arch/x86/net/bpf_jit_comp.c +4 -0
+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx)
+        */
+       EMIT4(0x48, 0x83, 0xC4, sizeof(struct
Quoted 2 lines

bpf_tramp_run_ctx));
+

What is the point of all of these additional sub/add rsp ? It seems unconditionally increasing stack_size by sizeof(struct bpf_tramp_run_ctx)
will achieve the same and above 4 extra insns won't be needed.

Quoted 6 lines
Quoted 2 lines

I think you are right.

The reason that I don't change stack_size is that we access arguments or saved registers basing on stack_size. Once the stack_size is changed, all these offsets should be changed too.

That should be trivial.
keep regs_off = stack_size;
and increase stack_size right after.
or some other math.
Maybe worth introducing another _off
in addition to int regs_off, ip_off, args_off. Definitely update 'Generated trampoline stack layout' comment and explain where bpf_tramp_run_ctx is in relation to regs_off. Maybe keeping regs_off (without new _off) is enough.

#10 lore raw

PATCH v7 3/5 bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.

In reply to Kui-Feng Lee

Pass a cookie along with BPF_LINK_CREATE requests.

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie. The cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

---
 arch/x86/net/bpf_jit_comp.c    | 12 ++++++++++--
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       |  9 +++++++++
 kernel/bpf/bpf_lsm.c           | 17 +++++++++++++++++
 kernel/bpf/syscall.c           | 12 ++++++++----
 kernel/bpf/trampoline.c        |  7 +++++--
 kernel/trace/bpf_trace.c       | 17 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  9 +++++++++
 8 files changed, 76 insertions(+), 8 deletions(-)
arch/x86/net/bpf_jit_comp.c +10 -2
index bf4576a6938c..52a5eba2d5e8 100644
@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 			   struct bpf_tramp_link *l, int stack_size,
 			   bool save_ret)
 {
+	u64 cookie = 0;
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
 	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = l->link.prog;
 
-	/* mov rdi, 0 */
-	emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+	if (l->link.type == BPF_LINK_TYPE_TRACING) {
+		struct bpf_tracing_link *tr_link =
+			container_of(l, struct bpf_tracing_link, link);
+
+		cookie = tr_link->cookie;
+	}
+
+	/* mov rdi, cookie */
+	emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);
 
 	/* Prepare struct bpf_tramp_run_ctx.
 	 *
include/linux/bpf.h +1 -0
index 29c3188195a6..13d80a4aa45b 100644
@@ -1109,6 +1109,7 @@ struct bpf_tracing_link {
 	enum bpf_attach_type attach_type;
 	struct bpf_trampoline *trampoline;
 	struct bpf_prog *tgt_prog;
+	u64 cookie;
 };
 
 struct bpf_link_primer {
include/uapi/linux/bpf.h +9 -0
index ff9af73859ca..a70e1fd3b3a1 100644
@@ -1490,6 +1490,15 @@ union bpf_attr {
 				__aligned_u64	addrs;
 				__aligned_u64	cookies;
 			} kprobe_multi;
+			struct {
+				/* this is overliad with the target_btf_id above. */
+				__u32		target_btf_id;
+				/* black box user-provided value passed through
+				 * to BPF program at the execution time and
+				 * accessible through bpf_get_attach_cookie() BPF helper
+				 */
+				__u64		cookie;
+			} tracing;
 		};
 	} link_create;
 
kernel/bpf/bpf_lsm.c +17 -0
index 064eccba641d..c1351df9f7ee 100644
@@ -117,6 +117,21 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = {
 	.allowed	= bpf_ima_inode_hash_allowed,
 };
 
+BPF_CALL_1(bpf_get_attach_cookie, void *, ctx)
+{
+	struct bpf_trace_run_ctx *run_ctx;
+
+	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
+	return run_ctx->bpf_cookie;
+}
+
+static const struct bpf_func_proto bpf_get_attach_cookie_proto = {
+	.func		= bpf_get_attach_cookie,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
+
 static const struct bpf_func_proto *
 bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -141,6 +156,8 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return prog->aux->sleepable ? &bpf_ima_inode_hash_proto : NULL;
 	case BPF_FUNC_ima_file_hash:
 		return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;
+	case BPF_FUNC_get_attach_cookie:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;
 	default:
 		return tracing_prog_func_proto(func_id, prog);
 	}
kernel/bpf/syscall.c +8 -4
index 5ed9a15daaee..dd0f4d51bcf6 100644
@@ -2921,7 +2921,8 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
 
 static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 				   int tgt_prog_fd,
-				   u32 btf_id)
+				   u32 btf_id,
+				   u64 bpf_cookie)
 {
 	struct bpf_link_primer link_primer;
 	struct bpf_prog *tgt_prog = NULL;
@@ -2986,6 +2987,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 	bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
 		      &bpf_tracing_link_lops, prog);
 	link->attach_type = prog->expected_attach_type;
+	link->cookie = bpf_cookie;
 
 	mutex_lock(&prog->aux->dst_mutex);
 
@@ -3271,7 +3273,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
 			tp_name = prog->aux->attach_func_name;
 			break;
 		}
-		return bpf_tracing_prog_attach(prog, 0, 0);
+		return bpf_tracing_prog_attach(prog, 0, 0, 0);
 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
 	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
 		if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
@@ -4524,7 +4526,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
 	case BPF_PROG_TYPE_EXT:
 		ret = bpf_tracing_prog_attach(prog,
 					      attr->link_create.target_fd,
-					      attr->link_create.target_btf_id);
+					      attr->link_create.target_btf_id,
+					      attr->link_create.tracing.cookie);
 		break;
 	case BPF_PROG_TYPE_LSM:
 	case BPF_PROG_TYPE_TRACING:
@@ -4539,7 +4542,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
 		else
 			ret = bpf_tracing_prog_attach(prog,
 						      attr->link_create.target_fd,
-						      attr->link_create.target_btf_id);
+						      attr->link_create.target_btf_id,
+						      attr->link_create.tracing.cookie);
 		break;
 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
 	case BPF_PROG_TYPE_SK_LOOKUP:
kernel/bpf/trampoline.c +5 -2
index baf1b65d523e..0e9b3aefc34a 100644
@@ -30,9 +30,12 @@ static DEFINE_MUTEX(trampoline_mutex);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
 {
 	enum bpf_attach_type eatype = prog->expected_attach_type;
+	enum bpf_prog_type ptype = prog->type;
 
-	return eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
-	       eatype == BPF_MODIFY_RETURN;
+	return (ptype == BPF_PROG_TYPE_TRACING &&
+		(eatype == BPF_TRACE_FENTRY || eatype == BPF_TRACE_FEXIT ||
+		 eatype == BPF_MODIFY_RETURN)) ||
+		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
 }
 
 void *bpf_jit_alloc_exec_page(void)
kernel/trace/bpf_trace.c +17 -0
index f15b826f9899..6377ed23e17f 100644
@@ -1091,6 +1091,21 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = {
 	.arg1_type	= ARG_PTR_TO_CTX,
 };
 
+BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx)
+{
+	struct bpf_trace_run_ctx *run_ctx;
+
+	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
+	return run_ctx->bpf_cookie;
+}
+
+static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = {
+	.func		= bpf_get_attach_cookie_tracing,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
+
 BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags)
 {
 #ifndef CONFIG_X86
@@ -1719,6 +1734,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
 	case BPF_FUNC_get_func_arg_cnt:
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
+	case BPF_FUNC_get_attach_cookie:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
 	default:
 		fn = raw_tp_prog_func_proto(func_id, prog);
 		if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)
tools/include/uapi/linux/bpf.h +9 -0
index ff9af73859ca..a70e1fd3b3a1 100644
@@ -1490,6 +1490,15 @@ union bpf_attr {
 				__aligned_u64	addrs;
 				__aligned_u64	cookies;
 			} kprobe_multi;
+			struct {
+				/* this is overliad with the target_btf_id above. */
+				__u32		target_btf_id;
+				/* black box user-provided value passed through
+				 * to BPF program at the execution time and
+				 * accessible through bpf_get_attach_cookie() BPF helper
+				 */
+				__u64		cookie;
+			} tracing;
 		};
 	} link_create;
 
Signature · 1 line
2.30.2
Acked-by on v7 3/5
#11 lore raw

Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.

Quoted 12 lines · Kui-Feng Lee

Pass a cookie along with BPF_LINK_CREATE requests.

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie. The cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

Quoted 11 lines
---
 arch/x86/net/bpf_jit_comp.c    | 12 ++++++++++--
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       |  9 +++++++++
 kernel/bpf/bpf_lsm.c           | 17 +++++++++++++++++
 kernel/bpf/syscall.c           | 12 ++++++++----
 kernel/bpf/trampoline.c        |  7 +++++--
 kernel/trace/bpf_trace.c       | 17 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  9 +++++++++
 8 files changed, 76 insertions(+), 8 deletions(-)

LGTM with a suggestion for some follow up clean up.

Quoted 7 lines
arch/x86/net/bpf_jit_comp.c +0 -0
index bf4576a6938c..52a5eba2d5e8 100644
@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
                           struct bpf_tramp_link *l, int stack_size,
                           bool save_ret)

arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1773

Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +2 -2
 {
+       u64 cookie = 0;
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;
 
-       /* mov rdi, 0 */
-       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+       if (l->link.type == BPF_LINK_TYPE_TRACING) {

It would probably be nicer to put cookie field into struct bpf_tramp_link instead so that the JIT compiler doesn't have to do this special handling. It also makes sense that struct bpf_trampoline *trampoline is moved into struct bpf_tramp_link itself (given trampoline is always there for bpf_tramp_link).

Quoted 11 lines
arch/x86/net/bpf_jit_comp.c +8 -0
+               struct bpf_tracing_link *tr_link =
+                       container_of(l, struct bpf_tracing_link, link);
+
+               cookie = tr_link->cookie;
+       }
+
+       /* mov rdi, cookie */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);
 
        /* Prepare struct bpf_tramp_run_ctx.
         *

[...]

#12 lore raw

Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.

Quoted 48 lines · Andrii Nakryiko
Quoted 22 lines · Kui-Feng Lee

Pass a cookie along with BPF_LINK_CREATE requests.

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie.
The cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

---
 arch/x86/net/bpf_jit_comp.c    | 12 ++++++++++-- include/linux/bpf.h            |  1 + include/uapi/linux/bpf.h       |  9 +++++++++ kernel/bpf/bpf_lsm.c           | 17 +++++++++++++++++ kernel/bpf/syscall.c           | 12 ++++++++---- kernel/bpf/trampoline.c        |  7 +++++-- kernel/trace/bpf_trace.c       | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h |  9 +++++++++

Quoted 2 lines
 8 files changed, 76 insertions(+), 8 deletions(-)

LGTM with a suggestion for some follow up clean up.

Quoted 18 lines
arch/x86/net/bpf_jit_comp.c +0 -0
index bf4576a6938c..52a5eba2d5e8 100644
@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,

                           struct bpf_tramp_link *l, int stack_size,
                           bool save_ret)
 {
+       u64 cookie = 0;
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;

Quoted 9 lines
Quoted 3 lines

-       /* mov rdi, 0 */
-       emit_mov_imm64(&prog, BPF_REG_1, 0, 0); +       if (l->link.type == BPF_LINK_TYPE_TRACING) {

It would probably be nicer to put cookie field into struct bpf_tramp_link instead so that the JIT compiler doesn't have to do this special handling. It also makes sense that struct bpf_trampoline *trampoline is moved into struct bpf_tramp_link itself (given trampoline is always there for bpf_tramp_link).

It will increase the size of bpf_tramp_link a little bit, but they are not used by bpf_struct_ops.

Quoted 16 lines
Quoted 14 lines

+               struct bpf_tracing_link *tr_link =
+                       container_of(l, struct bpf_tracing_link, link);

arch/x86/net/bpf_jit_comp.c +6 -0
+
+               cookie = tr_link->cookie;
+       }
+
+       /* mov rdi, cookie */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32)

(long) cookie);

        /* Prepare struct bpf_tramp_run_ctx.
         *

[...]

#13 lore raw

Re: [PATCH bpf-next v7 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret/lsm.

Quoted 60 lines · Kui-Feng Lee
Quoted 48 lines · Andrii Nakryiko
Quoted 13 lines · Kui-Feng Lee

Pass a cookie along with BPF_LINK_CREATE requests.

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie.
The cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

Quoted 11 lines
---
 arch/x86/net/bpf_jit_comp.c    | 12 ++++++++++--
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       |  9 +++++++++
 kernel/bpf/bpf_lsm.c           | 17 +++++++++++++++++
 kernel/bpf/syscall.c           | 12 ++++++++----
 kernel/bpf/trampoline.c        |  7 +++++--
 kernel/trace/bpf_trace.c       | 17 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  9 +++++++++
 8 files changed, 76 insertions(+), 8 deletions(-)

LGTM with a suggestion for some follow up clean up.

Quoted 18 lines
arch/x86/net/bpf_jit_comp.c +1 -0
index bf4576a6938c..52a5eba2d5e8 100644
@@ -1764,13 +1764,21 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,
                           struct bpf_tramp_link *l, int
 stack_size,
                           bool save_ret)
 {
+       u64 cookie = 0;
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx,
 bpf_cookie);
        struct bpf_prog *p = l->link.prog;
 
Quoted 9 lines

arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1773

Quoted 3 lines
arch/x86/net/bpf_jit_comp.c +1 -2
-       /* mov rdi, 0 */
-       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+       if (l->link.type == BPF_LINK_TYPE_TRACING) {

It would probably be nicer to put cookie field into struct bpf_tramp_link instead so that the JIT compiler doesn't have to do this special handling. It also makes sense that struct bpf_trampoline *trampoline is moved into struct bpf_tramp_link itself (given trampoline is always there for bpf_tramp_link).

Quoted 3 lines

It will increase the size of bpf_tramp_link a little bit, but they are not used by bpf_struct_ops.

It feels like the right tradeoff to keep architecture-specific trampoline code oblivious to these details. Some day structs_ops might support cookies as well. And either way 8 bytes for struct_ops link isn't a big deal.

Quoted 17 lines
Quoted 16 lines
Quoted 14 lines
arch/x86/net/bpf_jit_comp.c +2 -0
+               struct bpf_tracing_link *tr_link =
+                       container_of(l, struct bpf_tracing_link,

link);

arch/x86/net/bpf_jit_comp.c +6 -0
+
+               cookie = tr_link->cookie;
+       }
+
+       /* mov rdi, cookie */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32)

(long) cookie);

        /* Prepare struct bpf_tramp_run_ctx.
         *

[...]

#14 lore raw

PATCH v7 4/5 libbpf: Assign cookies to links in libbpf.

In reply to Kui-Feng Lee

Add a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.

---
 tools/lib/bpf/bpf.c      |  8 ++++++++
 tools/lib/bpf/bpf.h      |  3 +++
 tools/lib/bpf/libbpf.c   | 32 ++++++++++++++++++++++++++++++++
 tools/lib/bpf/libbpf.h   | 12 ++++++++++++
 tools/lib/bpf/libbpf.map |  1 +
 5 files changed, 56 insertions(+)
tools/lib/bpf/bpf.c +8 -0
index a9d292c106c2..5660268e103f 100644
@@ -863,6 +863,14 @@ int bpf_link_create(int prog_fd, int target_fd,
 		if (!OPTS_ZEROED(opts, kprobe_multi))
 			return libbpf_err(-EINVAL);
 		break;
+	case BPF_TRACE_FENTRY:
+	case BPF_TRACE_FEXIT:
+	case BPF_MODIFY_RETURN:
+	case BPF_LSM_MAC:
+		attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0);
+		if (!OPTS_ZEROED(opts, tracing))
+			return libbpf_err(-EINVAL);
+		break;
 	default:
 		if (!OPTS_ZEROED(opts, flags))
 			return libbpf_err(-EINVAL);
tools/lib/bpf/bpf.h +3 -0
index f4b4afb6d4ba..34af2232928c 100644
@@ -420,6 +420,9 @@ struct bpf_link_create_opts {
 			const unsigned long *addrs;
 			const __u64 *cookies;
 		} kprobe_multi;
+		struct {
+			__u64 cookie;
+		} tracing;
 	};
 	size_t :0;
 };
tools/lib/bpf/libbpf.c +32 -0
index 73a5192defb3..df9be47d67bc 100644
@@ -11440,6 +11440,38 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
 	return bpf_program__attach_btf_id(prog);
 }
 
+struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
+						const struct bpf_trace_opts *opts)
+{
+	char errmsg[STRERR_BUFSIZE];
+	struct bpf_link *link;
+	int prog_fd, pfd;
+	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+	prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
+
+	link = calloc(1, sizeof(*link));
+	if (!link)
+		return libbpf_err_ptr(-ENOMEM);
+	link->detach = &bpf_link__detach_fd;
+
+	link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0);
+	pfd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &link_opts);
+	if (pfd < 0) {
+		pfd = -errno;
+		free(link);
+		pr_warn("prog '%s': failed to attach: %s\n",
+			prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
+		return libbpf_err_ptr(pfd);
+	}
+	link->fd = pfd;
+	return link;
+}
+
 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
 {
 	return bpf_program__attach_btf_id(prog);
tools/lib/bpf/libbpf.h +12 -0
index cdbfee60ea3e..62af394247eb 100644
@@ -603,8 +603,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
 LIBBPF_API struct bpf_link *
 bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
 				   const char *tp_name);
+
+struct bpf_trace_opts {
+	/* size of this struct, for forward/backward compatibility */
+	size_t sz;
+	/* custom user-provided value fetchable through bpf_get_attach_cookie() */
+	__u64 cookie;
+};
+#define bpf_trace_opts__last_field cookie
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_trace(const struct bpf_program *prog);
+LIBBPF_API struct bpf_link *
+bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts);
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_lsm(const struct bpf_program *prog);
 LIBBPF_API struct bpf_link *
tools/lib/bpf/libbpf.map +1 -0
index 82f6d62176dd..245a0e8677c9 100644
@@ -444,6 +444,7 @@ LIBBPF_0.8.0 {
 	global:
 		bpf_object__destroy_subskeleton;
 		bpf_object__open_subskeleton;
+		bpf_program__attach_trace_opts;
 		bpf_program__attach_usdt;
 		libbpf_register_prog_handler;
 		libbpf_unregister_prog_handler;
Signature · 1 line
2.30.2
Acked-by on v7 4/5
#15 lore raw

Re: [PATCH bpf-next v7 4/5] libbpf: Assign cookies to links in libbpf.

Quoted 4 lines · Kui-Feng Lee

Add a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.

Quoted 9 lines
---
 tools/lib/bpf/bpf.c      |  8 ++++++++
 tools/lib/bpf/bpf.h      |  3 +++
 tools/lib/bpf/libbpf.c   | 32 ++++++++++++++++++++++++++++++++
 tools/lib/bpf/libbpf.h   | 12 ++++++++++++
 tools/lib/bpf/libbpf.map |  1 +
 5 files changed, 56 insertions(+)

I have a gripe with better code reuse, but that's internal change so we can do it in a follow up.

tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11444

Quoted 10 lines
tools/lib/bpf/libbpf.c +2 -0
index 73a5192defb3..df9be47d67bc 100644
@@ -11440,6 +11440,38 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
        return bpf_program__attach_btf_id(prog);
 }
 
+struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
+                                               const struct bpf_trace_opts *opts)

there is bpf_program__attach_btf_id() that does all of this except for the cookie. It would be nicer to extend bpf_program__attach_btf_id(), which won't break any API because it's an internal helper, add optional bpf_trace_opts to it and then just redirect bpf_program__attach_trace_opts() to bpf_program__attach_btf_id and update all the existing callers with just passing NULL for opts.

We can do that as a follow up, given your patch set seems to be pretty much ready to be landed.

Quoted 6 lines
tools/lib/bpf/libbpf.c +6 -0
+{
+       char errmsg[STRERR_BUFSIZE];
+       struct bpf_link *link;
+       int prog_fd, pfd;
+       LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+

[...]

#16 lore raw

Re: [PATCH bpf-next v7 4/5] libbpf: Assign cookies to links in libbpf.

Quoted 29 lines · Andrii Nakryiko
Quoted 4 lines · Kui-Feng Lee

Add a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.

Quoted 9 lines

---
 tools/lib/bpf/bpf.c      |  8 ++++++++ tools/lib/bpf/bpf.h      |  3 +++ tools/lib/bpf/libbpf.c   | 32 ++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf.h   | 12 ++++++++++++ tools/lib/bpf/libbpf.map |  1 +

 5 files changed, 56 insertions(+)

I have a gripe with better code reuse, but that's internal change so we can do it in a follow up.

Quoted 9 lines
tools/lib/bpf/libbpf.c +0 -0
index 73a5192defb3..df9be47d67bc 100644
@@ -11440,6 +11440,38 @@ struct bpf_link
 *bpf_program__attach_trace(const struct bpf_program *prog)
         return bpf_program__attach_btf_id(prog);
  }
 
Quoted 12 lines

tools/lib/bpf/libbpf.c struct bpf_link · line 11447

Quoted 4 lines
tools/lib/bpf/libbpf.c +2 -0
+struct bpf_link *bpf_program__attach_trace_opts(const struct
 bpf_program *prog,
+                                               const struct
 bpf_trace_opts *opts)

there is bpf_program__attach_btf_id() that does all of this except for
the cookie. It would be nicer to extend bpf_program__attach_btf_id(), which won't break any API because it's an internal helper, add optional bpf_trace_opts to it and then just redirect bpf_program__attach_trace_opts() to bpf_program__attach_btf_id and update all the existing callers with just passing NULL for opts.

Fixed! I copied the code from bpf_program__attach_btf_id() for testing, and forgot to refactor it.

Quoted 13 lines

We can do that as a follow up, given your patch set seems to be pretty
much ready to be landed.

Quoted 6 lines
tools/lib/bpf/libbpf.c +6 -0
+{
+       char errmsg[STRERR_BUFSIZE];
+       struct bpf_link *link;
+       int prog_fd, pfd;
+       LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+

[...]

#17 lore raw

PATCH v7 5/5 selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret/lsm.

In reply to Kui-Feng Lee

Make sure BPF cookies are correct for fentry/fexit/fmod_ret/lsm.

---
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 89 +++++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 52 +++++++++--
 2 files changed, 133 insertions(+), 8 deletions(-)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +89 -0
index 923a6139b2d8..83ef55e3caa4 100644
@@ -4,8 +4,11 @@
 #include <pthread.h>
 #include <sched.h>
 #include <sys/syscall.h>
+#include <sys/mman.h>
 #include <unistd.h>
 #include <test_progs.h>
+#include <network_helpers.h>
+#include <bpf/btf.h>
 #include "test_bpf_cookie.skel.h"
 #include "kprobe_multi.skel.h"
 
@@ -410,6 +413,88 @@ static void pe_subtest(struct test_bpf_cookie *skel)
 	bpf_link__destroy(link);
 }
 
+static void tracing_subtest(struct test_bpf_cookie *skel)
+{
+	__u64 cookie;
+	int prog_fd;
+	int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1;
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+	skel->bss->fentry_res = 0;
+	skel->bss->fexit_res = 0;
+
+	cookie = 0x10000000000000L;
+	prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+	link_opts.tracing.cookie = cookie;
+	fentry_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY, &link_opts);
+	if (!ASSERT_GE(fentry_fd, 0, "fentry.link_create"))
+		goto cleanup;
+
+	cookie = 0x20000000000000L;
+	prog_fd = bpf_program__fd(skel->progs.fexit_test1);
+	link_opts.tracing.cookie = cookie;
+	fexit_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FEXIT, &link_opts);
+	if (!ASSERT_GE(fexit_fd, 0, "fexit.link_create"))
+		goto cleanup;
+
+	cookie = 0x30000000000000L;
+	prog_fd = bpf_program__fd(skel->progs.fmod_ret_test);
+	link_opts.tracing.cookie = cookie;
+	fmod_ret_fd = bpf_link_create(prog_fd, 0, BPF_MODIFY_RETURN, &link_opts);
+	if (!ASSERT_GE(fmod_ret_fd, 0, "fmod_ret.link_create"))
+		goto cleanup;
+
+	prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+	bpf_prog_test_run_opts(prog_fd, &opts);
+
+	prog_fd = bpf_program__fd(skel->progs.fmod_ret_test);
+	bpf_prog_test_run_opts(prog_fd, &opts);
+
+	ASSERT_EQ(skel->bss->fentry_res, 0x10000000000000L, "fentry_res");
+	ASSERT_EQ(skel->bss->fexit_res, 0x20000000000000L, "fexit_res");
+	ASSERT_EQ(skel->bss->fmod_ret_res, 0x30000000000000L, "fmod_ret_res");
+
+cleanup:
+	if (fentry_fd >= 0)
+		close(fentry_fd);
+	if (fexit_fd >= 0)
+		close(fexit_fd);
+	if (fmod_ret_fd >= 0)
+		close(fmod_ret_fd);
+}
+
+int stack_mprotect(void);
+
+static void lsm_subtest(struct test_bpf_cookie *skel)
+{
+	__u64 cookie;
+	int prog_fd;
+	int lsm_fd = -1;
+	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+	skel->bss->lsm_res = 0;
+
+	cookie = 0x90000000000090L;
+	prog_fd = bpf_program__fd(skel->progs.test_int_hook);
+	link_opts.tracing.cookie = cookie;
+	lsm_fd = bpf_link_create(prog_fd, 0, BPF_LSM_MAC, &link_opts);
+	if (!ASSERT_GE(lsm_fd, 0, "lsm.link_create"))
+		goto cleanup;
+
+	stack_mprotect();
+	if (!ASSERT_EQ(errno, EPERM, "stack_mprotect"))
+		goto cleanup;
+
+	usleep(1);
+
+	ASSERT_EQ(skel->bss->lsm_res, 0x90000000000090L, "fentry_res");
+
+cleanup:
+	if (lsm_fd >= 0)
+		close(lsm_fd);
+}
+
 void test_bpf_cookie(void)
 {
 	struct test_bpf_cookie *skel;
@@ -432,6 +517,10 @@ void test_bpf_cookie(void)
 		tp_subtest(skel);
 	if (test__start_subtest("perf_event"))
 		pe_subtest(skel);
+	if (test__start_subtest("trampoline"))
+		tracing_subtest(skel);
+	if (test__start_subtest("lsm"))
+		lsm_subtest(skel);
 
 	test_bpf_cookie__destroy(skel);
 }
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +44 -8

66 lines not on this page · raw message

Signature · 1 line
2.30.2
Acked-by on v7 5/5
#18 lore raw

Re: [PATCH bpf-next v7 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret/lsm.

Quoted 9 lines · Kui-Feng Lee

Make sure BPF cookies are correct for fentry/fexit/fmod_ret/lsm.

---
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 89 +++++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 52 +++++++++--
 2 files changed, 133 insertions(+), 8 deletions(-)

Looks great.

[...]