Skip to contentSkip to conversation

Attach a cookie to a tracing program.

PATCH Superseded Kui-Feng Lee 2 participants 17 messages Apr 12, 2022 – Apr 13, 2022

Subsystems (12)
Networking x86 Kselftest BPF +8 more subsystems

v5 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

v5 cover Attach a cookie to a tracing program.

Allow users to attach a 64-bits cookie to a bpf_link of fentry, fexit, or fmod_ret.

This patchset includes several major changes.

- Define struct bpf_tramp_links to replace bpf_tramp_prog. struct bpf_tramp_links collects bpf_links of a trampoline

- Generate a trampoline to call bpf_progs of given bpf_links.

- Trampolines always set/reset bpf_run_ctx before/after calling/leaving a tracing program.

- Attach a cookie to a bpf_link of fentry/fexit/fmod_ret. The value will be available when running the associated bpf_prog.

The major differences from v4:

- Remove condition checking in __bpf_prog_{enter,exit)

The major differences from v3:

- rebase v3 to latest tip.

The major differences from v2:

- Move the allocations of run_ctx (struct bpf_tramp_run_ctx) out of invoke_bpf_prog().

- Move hlist_node out of bpf_link and introduce struct bpf_tramp_link to own hlist_node.

- Store cookies at struct bpf_tracing_link.

- Use SIB byte to reduce the number of instructions to set cookie values. (Use RSP directly)

v1: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/bpf/[email protected]/
v3: https://lore.kernel.org/bpf/[email protected]/
v4: https://lore.kernel.org/bpf/[email protected]/

Kui-Feng Lee (5):
bpf, x86: Generate trampolines from bpf_tramp_links
bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack bpf, x86: Attach a cookie to fentry/fexit/fmod_ret. lib/bpf: Assign cookies to links in libbpf.
selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.

 arch/x86/net/bpf_jit_comp.c                   | 98 +++++++++++++++----
 include/linux/bpf.h                           | 56 ++++++++---
 include/linux/bpf_types.h                     |  1 +
 include/uapi/linux/bpf.h                      |  9 ++
 kernel/bpf/bpf_struct_ops.c                   | 69 ++++++++-----
 kernel/bpf/syscall.c                          | 57 +++++++----
 kernel/bpf/trampoline.c                       | 93 +++++++++++-------
 kernel/trace/bpf_trace.c                      | 17 ++++
 net/bpf/bpf_dummy_struct_ops.c                | 35 ++++++-
 tools/bpf/bpftool/link.c                      |  1 +
 tools/include/uapi/linux/bpf.h                |  8 ++
 tools/lib/bpf/bpf.c                           |  7 ++
 tools/lib/bpf/bpf.h                           |  3 +
 tools/lib/bpf/libbpf.c                        | 33 +++++++
 tools/lib/bpf/libbpf.h                        | 12 +++
 tools/lib/bpf/libbpf.map                      |  1 +
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 52 ++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 24 +++++
 18 files changed, 459 insertions(+), 117 deletions(-)
Signature · 1 line
2.30.2
#1 lore raw

PATCH v5 1/5 bpf, x86: Generate trampolines from bpf_tramp_links

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++--------
 include/linux/bpf.h            | 38 ++++++++++++------
 include/linux/bpf_types.h      |  1 +
 include/uapi/linux/bpf.h       |  1 +
 kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++----------
 kernel/bpf/syscall.c           | 23 ++++-------
 kernel/bpf/trampoline.c        | 73 +++++++++++++++++++---------------
 net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++---
 tools/bpf/bpftool/link.c       |  1 +
 tools/include/uapi/linux/bpf.h |  1 +
 10 files changed, 175 insertions(+), 103 deletions(-)
arch/x86/net/bpf_jit_comp.c +19 -17
index 8fe35ed11fd6..4dcc0b1ac770 100644
@@ -1761,10 +1761,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 }
 
 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
-			   struct bpf_prog *p, int stack_size, bool save_ret)
+			   struct bpf_tramp_link *l, int stack_size,
+			   bool save_ret)
 {
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
+	struct bpf_prog *p = l->link.prog;
 
 	/* arg1: mov rdi, progs[i] */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
@@ -1849,14 +1851,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
 }
 
 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
-		      struct bpf_tramp_progs *tp, int stack_size,
+		      struct bpf_tramp_links *tl, int stack_size,
 		      bool save_ret)
 {
 	int i;
 	u8 *prog = *pprog;
 
-	for (i = 0; i < tp->nr_progs; i++) {
-		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,
+	for (i = 0; i < tl->nr_links; i++) {
+		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
 				    save_ret))
 			return -EINVAL;
 	}
@@ -1865,7 +1867,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 }
 
 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
-			      struct bpf_tramp_progs *tp, int stack_size,
+			      struct bpf_tramp_links *tl, int stack_size,
 			      u8 **branches)
 {
 	u8 *prog = *pprog;
@@ -1876,8 +1878,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
 	 */
 	emit_mov_imm32(&prog, false, BPF_REG_0, 0);
 	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
-	for (i = 0; i < tp->nr_progs; i++) {
-		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))
+	for (i = 0; i < tl->nr_links; i++) {
+		if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))
 			return -EINVAL;
 
 		/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1979,14 +1981,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)
  */
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
 				const struct btf_func_model *m, u32 flags,
-				struct bpf_tramp_progs *tprogs,
+				struct bpf_tramp_links *tlinks,
 				void *orig_call)
 {
 	int ret, i, nr_args = m->nr_args;
 	int regs_off, ip_off, args_off, stack_size = nr_args * 8;
-	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
-	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
-	struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
+	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
+	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
+	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
 	u8 **branches = NULL;
 	u8 *prog;
 	bool save_ret;
@@ -2077,13 +2079,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		}
 	}
 
-	if (fentry->nr_progs)
+	if (fentry->nr_links)
 		if (invoke_bpf(m, &prog, fentry, regs_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET))
 			return -EINVAL;
 
-	if (fmod_ret->nr_progs) {
-		branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),
+	if (fmod_ret->nr_links) {
+		branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),
 				   GFP_KERNEL);
 		if (!branches)
 			return -ENOMEM;
@@ -2110,7 +2112,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		prog += X86_PATCH_SIZE;
 	}
 
-	if (fmod_ret->nr_progs) {
+	if (fmod_ret->nr_links) {
 		/* From Intel 64 and IA-32 Architectures Optimization
 		 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
 		 * Coding Rule 11: All branch targets should be 16-byte
@@ -2120,12 +2122,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		/* Update the branches saved in invoke_bpf_mod_ret with the
 		 * aligned address of do_fexit.
 		 */
-		for (i = 0; i < fmod_ret->nr_progs; i++)
+		for (i = 0; i < fmod_ret->nr_links; i++)
 			emit_cond_near_jump(&branches[i], prog, branches[i],
 					    X86_JNE);
 	}
 
-	if (fexit->nr_progs)
+	if (fexit->nr_links)
 		if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
 			ret = -EINVAL;
 			goto cleanup;
include/linux/bpf.h +26 -12
index bdb5298735ce..2f7909eb51ec 100644
@@ -674,11 +674,11 @@ struct btf_func_model {
 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
  * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2
  */
-#define BPF_MAX_TRAMP_PROGS 38
+#define BPF_MAX_TRAMP_LINKS 38
 
-struct bpf_tramp_progs {
-	struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
-	int nr_progs;
+struct bpf_tramp_links {
+	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
+	int nr_links;
 };
 
 /* Different use cases for BPF trampoline:
@@ -704,7 +704,7 @@ struct bpf_tramp_progs {
 struct bpf_tramp_image;
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
 				const struct btf_func_model *m, u32 flags,
-				struct bpf_tramp_progs *tprogs,
+				struct bpf_tramp_links *tlinks,
 				void *orig_call);
 /* these two functions are called from generated trampoline */
 u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
 {
 	return bpf_func(ctx, insnsi);
 }
+
+struct bpf_link;
+
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
 					  struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 					   struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
 					     struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
@@ -960,7 +963,6 @@ struct bpf_prog_aux {
 	bool tail_call_reachable;
 	bool xdp_has_frags;
 	bool use_bpf_prog_pack;
-	struct hlist_node tramp_hlist;
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
 	const struct btf_type *attach_func_proto;
 	/* function name for valid attach_btf_id */
@@ -1047,6 +1049,18 @@ struct bpf_link_ops {
 			      struct bpf_link_info *info);
 };
 
+struct bpf_tramp_link {
+	struct bpf_link link;
+	struct hlist_node tramp_hlist;
+};
+
+struct bpf_tracing_link {
+	struct bpf_tramp_link link;
+	enum bpf_attach_type attach_type;
+	struct bpf_trampoline *trampoline;
+	struct bpf_prog *tgt_prog;
+};
+
 struct bpf_link_primer {
 	struct bpf_link *link;
 	struct file *file;
@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);
 void bpf_struct_ops_put(const void *kdata);
 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
 				       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-				      struct bpf_prog *prog,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+				      struct bpf_tramp_link *link,
 				      const struct btf_func_model *model,
 				      void *image, void *image_end);
 static inline bool bpf_try_module_get(const void *data, struct module *owner)
include/linux/bpf_types.h +1 -0
index 3e24ad0c4b3c..2b9112b80171 100644
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
 BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
 #endif
 BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
+BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)
include/uapi/linux/bpf.h +1 -0
index d14b10b85e51..a4f557338af7 100644
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
 	BPF_LINK_TYPE_XDP = 6,
 	BPF_LINK_TYPE_PERF_EVENT = 7,
 	BPF_LINK_TYPE_KPROBE_MULTI = 8,
+	BPF_LINK_TYPE_STRUCT_OPS = 9,
 
 	MAX_BPF_LINK_TYPE,
 };
kernel/bpf/bpf_struct_ops.c +47 -22

153 lines not on this page · raw message

kernel/bpf/syscall.c +8 -15
index cdaa1152436a..3078c0c9317f 100644
@@ -2640,19 +2640,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd)
 }
 EXPORT_SYMBOL(bpf_link_get_from_fd);
 
-struct bpf_tracing_link {
-	struct bpf_link link;
-	enum bpf_attach_type attach_type;
-	struct bpf_trampoline *trampoline;
-	struct bpf_prog *tgt_prog;
-};
-
 static void bpf_tracing_link_release(struct bpf_link *link)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
-	WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
+	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
 						tr_link->trampoline));
 
 	bpf_trampoline_put(tr_link->trampoline);
@@ -2665,7 +2658,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
 static void bpf_tracing_link_dealloc(struct bpf_link *link)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	kfree(tr_link);
 }
@@ -2674,7 +2667,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
 					 struct seq_file *seq)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	seq_printf(seq,
 		   "attach_type:\t%d\n",
@@ -2685,7 +2678,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
 					   struct bpf_link_info *info)
 {
 	struct bpf_tracing_link *tr_link =
-		container_of(link, struct bpf_tracing_link, link);
+		container_of(link, struct bpf_tracing_link, link.link);
 
 	info->tracing.attach_type = tr_link->attach_type;
 	bpf_trampoline_unpack_key(tr_link->trampoline->key,
@@ -2766,7 +2759,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 		err = -ENOMEM;
 		goto out_put_prog;
 	}
-	bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
+	bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
 		      &bpf_tracing_link_lops, prog);
 	link->attach_type = prog->expected_attach_type;
 
@@ -2836,11 +2829,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 		tgt_prog = prog->aux->dst_prog;
 	}
 
-	err = bpf_link_prime(&link->link, &link_primer);
+	err = bpf_link_prime(&link->link.link, &link_primer);
 	if (err)
 		goto out_unlock;
 
-	err = bpf_trampoline_link_prog(prog, tr);
+	err = bpf_trampoline_link_prog(&link->link, tr);
 	if (err) {
 		bpf_link_cleanup(&link_primer);
 		link = NULL;
kernel/bpf/trampoline.c +41 -32

171 lines not on this page · raw message

net/bpf/bpf_dummy_struct_ops.c +30 -5
index d0e54e30658a..b3717f3ab44b 100644
@@ -72,13 +72,28 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args)
 		    args->args[3], args->args[4]);
 }
 
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+	kfree(link);
+}
+
+static const struct bpf_link_ops bpf_struct_ops_link_lops = {
+	.release = bpf_struct_ops_link_release,
+	.dealloc = bpf_struct_ops_link_dealloc,
+};
+
 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 			    union bpf_attr __user *uattr)
 {
 	const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;
 	const struct btf_type *func_proto;
 	struct bpf_dummy_ops_test_args *args;
-	struct bpf_tramp_progs *tprogs;
+	struct bpf_tramp_links *tlinks;
+	struct bpf_tramp_link *link = NULL;
 	void *image = NULL;
 	unsigned int op_idx;
 	int prog_ret;
@@ -92,8 +107,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 	if (IS_ERR(args))
 		return PTR_ERR(args);
 
-	tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
-	if (!tprogs) {
+	tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+	if (!tlinks) {
 		err = -ENOMEM;
 		goto out;
 	}
@@ -105,10 +120,18 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 	}
 	set_vm_flush_reset_perms(image);
 
+	link = kzalloc(sizeof(*link), GFP_USER);
+	if (!link) {
+		err = -ENOMEM;
+		goto out;
+	}
+	bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
+
 	op_idx = prog->expected_attach_type;
-	err = bpf_struct_ops_prepare_trampoline(tprogs, prog,
+	err = bpf_struct_ops_prepare_trampoline(tlinks, link,
 						&st_ops->func_models[op_idx],
 						image, image + PAGE_SIZE);
+
 	if (err < 0)
 		goto out;
 
@@ -124,7 +147,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 out:
 	kfree(args);
 	bpf_jit_free_exec(image);
-	kfree(tprogs);
+	if (link)
+		bpf_link_put(&link->link);
+	kfree(tlinks);
 	return err;
 }
 
tools/bpf/bpftool/link.c +1 -0
index 8fb0116f9136..6353a789322b 100644
@@ -23,6 +23,7 @@ static const char * const link_type_name[] = {
 	[BPF_LINK_TYPE_XDP]			= "xdp",
 	[BPF_LINK_TYPE_PERF_EVENT]		= "perf_event",
 	[BPF_LINK_TYPE_KPROBE_MULTI]		= "kprobe_multi",
+	[BPF_LINK_TYPE_STRUCT_OPS]               = "struct_ops",
 };
 
 static struct hashmap *link_table;
tools/include/uapi/linux/bpf.h +1 -0
index d14b10b85e51..a4f557338af7 100644
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
 	BPF_LINK_TYPE_XDP = 6,
 	BPF_LINK_TYPE_PERF_EVENT = 7,
 	BPF_LINK_TYPE_KPROBE_MULTI = 8,
+	BPF_LINK_TYPE_STRUCT_OPS = 9,
 
 	MAX_BPF_LINK_TYPE,
 };
Signature · 1 line
2.30.2
Acked-by on v5 1/5
#2 lore raw

Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links

Quoted 5 lines · Kui-Feng Lee

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.

Quoted 8 lines

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---

Looks good, see two comments below.

Quoted 12 lines
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++--------
 include/linux/bpf.h            | 38 ++++++++++++------
 include/linux/bpf_types.h      |  1 +
 include/uapi/linux/bpf.h       |  1 +
 kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++----------
 kernel/bpf/syscall.c           | 23 ++++-------
 kernel/bpf/trampoline.c        | 73 +++++++++++++++++++---------------
 net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++---
 tools/bpf/bpftool/link.c       |  1 +
 tools/include/uapi/linux/bpf.h |  1 +
 10 files changed, 175 insertions(+), 103 deletions(-)

[...]

Quoted 11 lines

/* Different use cases for BPF trampoline:

include/linux/bpf.h +1 -1
@@ -704,7 +704,7 @@ struct bpf_tramp_progs {
 struct bpf_tramp_image;
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
-                               struct bpf_tramp_progs *tprogs,
+                               struct bpf_tramp_links *tlinks,
                                void *orig_call);
 /* these two functions are called from generated trampoline */
 u64 notrace __bpf_prog_enter(struct bpf_prog *prog);

include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808

Quoted 6 lines
include/linux/bpf.h +3 -0
@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
 {
        return bpf_func(ctx, insnsi);
 }
+
+struct bpf_link;
+

is this forward declaration still needed? was it supposed to be a struct bpf_tramp_link instead? and also probably higher above, before bpf_tramp_links?

Quoted 36 lines
include/linux/bpf.h +5 -5
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
                                          struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                           struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                                             struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;
@@ -960,7 +963,6 @@ struct bpf_prog_aux {
        bool tail_call_reachable;
        bool xdp_has_frags;
        bool use_bpf_prog_pack;
-       struct hlist_node tramp_hlist;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */
@@ -1047,6 +1049,18 @@ struct bpf_link_ops {
                              struct bpf_link_info *info);
 };
 
+struct bpf_tramp_link {

include/linux/bpf.h struct bpf_link_ops { · line 1062

Quoted 10 lines
include/linux/bpf.h +10 -0
+       struct bpf_link link;
+       struct hlist_node tramp_hlist;
+};
+
+struct bpf_tracing_link {
+       struct bpf_tramp_link link;
+       enum bpf_attach_type attach_type;
+       struct bpf_trampoline *trampoline;
+       struct bpf_prog *tgt_prog;
+};

struct bpf_tracing_link can stay in syscall.c, no? don't see anyone needing it outside of syscall.c

Quoted 14 lines
include/linux/bpf.h +3 -2
+
 struct bpf_link_primer {
        struct bpf_link *link;
        struct file *file;
@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);
 void bpf_struct_ops_put(const void *kdata);
 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
                                       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+                                     struct bpf_tramp_link *link,
                                      const struct btf_func_model *model,
                                      void *image, void *image_end);

[...]

#3 lore raw

Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links

Quoted 53 lines · Andrii Nakryiko
Quoted 6 lines · Kui-Feng Lee

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline.  struct bpf_tramp_link extends bpf_link to act as a linked list node.

Quoted 8 lines

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---

Looks good, see two comments below.

Quoted 14 lines

 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++-------- include/linux/bpf.h            | 38 ++++++++++++------ include/linux/bpf_types.h      |  1 + include/uapi/linux/bpf.h       |  1 + kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++-------- --
 kernel/bpf/syscall.c           | 23 ++++------- kernel/bpf/trampoline.c        | 73 +++++++++++++++++++-----------
----
 net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++--- tools/bpf/bpftool/link.c       |  1 + tools/include/uapi/linux/bpf.h |  1 +

 10 files changed, 175 insertions(+), 103 deletions(-)

[...]

Quoted 16 lines

 /* Different use cases for BPF trampoline:

include/linux/bpf.h +0 -0
@@ -704,7 +704,7 @@ struct bpf_tramp_progs {
  struct bpf_tramp_image;

 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
-                               struct bpf_tramp_progs *tprogs, +                               struct bpf_tramp_links *tlinks, void *orig_call);
 /* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(struct bpf_prog *prog);

(quoted diff) +0 -0
@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int

bpf_dispatcher_nop_func(
 {
        return bpf_func(ctx, insnsi);

Quoted 8 lines

include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808

Quoted 4 lines

 }

include/linux/bpf.h +3 -0
+
+struct bpf_link;
+

is this forward declaration still needed? was it supposed to be a struct bpf_tramp_link instead? and also probably higher above, before bpf_tramp_links?

You are right, I should remvoe it.

Quoted 46 lines
Quoted 46 lines

 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);

include/linux/bpf.h +1 -1
@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);
  void bpf_jit_uncharge_modmem(u32 size);
  bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
  #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link
 *link,

                                           struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                                             struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;

include/linux/bpf.h +0 -1
@@ -960,7 +963,6 @@ struct bpf_prog_aux {
         bool tail_call_reachable;
         bool xdp_has_frags;
         bool use_bpf_prog_pack;
-       struct hlist_node tramp_hlist;
         /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
         const struct btf_type *attach_func_proto;
         /* function name for valid attach_btf_id */
@@ -1047,6 +1049,18 @@ struct bpf_link_ops {
                               struct bpf_link_info *info);
  };
 
Quoted 15 lines

include/linux/bpf.h struct bpf_link_ops { · line 1062

Quoted 12 lines
include/linux/bpf.h +5 -0
+struct bpf_tramp_link {
+       struct bpf_link link;
+       struct hlist_node tramp_hlist;
+};
+

https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing

include/linux/bpf.h +6 -0
+struct bpf_tracing_link {
+       struct bpf_tramp_link link;
+       enum bpf_attach_type attach_type;
+       struct bpf_trampoline *trampoline;
+       struct bpf_prog *tgt_prog;
+};

struct bpf_tracing_link can stay in syscall.c, no? don't see anyone needing it outside of syscall.c

It will be used by invoke_bpf_prog() of bpf_jit_comp.c in the 3rd patch to get the cookie value.

Quoted 21 lines
Quoted 19 lines

+
 struct bpf_link_primer {
        struct bpf_link *link;
        struct file *file;

include/linux/bpf.h +0 -0
@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);
  void bpf_struct_ops_put(const void *kdata);

 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
                                       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog, +int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+                                     struct bpf_tramp_link *link, const struct btf_func_model *model,
                                      void *image, void *image_end);

[...]

#4 lore raw

Re: [PATCH bpf-next v5 1/5] bpf, x86: Generate trampolines from bpf_tramp_links

Quoted 128 lines · Kui-Feng Lee
Quoted 53 lines · Andrii Nakryiko
Quoted 6 lines · Kui-Feng Lee

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline. struct bpf_tramp_link extends bpf_link to act as a linked list node.

Quoted 8 lines

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links instead of bpf_tramp_progs.

---

Looks good, see two comments below.

Quoted 14 lines
 arch/x86/net/bpf_jit_comp.c    | 36 +++++++++--------
 include/linux/bpf.h            | 38 ++++++++++++------
 include/linux/bpf_types.h      |  1 +
 include/uapi/linux/bpf.h       |  1 +
 kernel/bpf/bpf_struct_ops.c    | 69 ++++++++++++++++++++++--------
--
 kernel/bpf/syscall.c           | 23 ++++-------
 kernel/bpf/trampoline.c        | 73 +++++++++++++++++++-----------
----
 net/bpf/bpf_dummy_struct_ops.c | 35 +++++++++++++---
 tools/bpf/bpftool/link.c       |  1 +
 tools/include/uapi/linux/bpf.h |  1 +
 10 files changed, 175 insertions(+), 103 deletions(-)

[...]

Quoted 16 lines

/* Different use cases for BPF trampoline:

include/linux/bpf.h +1 -1
@@ -704,7 +704,7 @@ struct bpf_tramp_progs {
 struct bpf_tramp_image;
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void
 *image, void *image_end,
                                const struct btf_func_model *m, u32
 flags,
-                               struct bpf_tramp_progs *tprogs,
+                               struct bpf_tramp_links *tlinks,
                                void *orig_call);

/* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(struct bpf_prog *prog);

(quoted diff) +0 -0
@@ -803,9 +803,12 @@ static __always_inline __nocfi unsigned int

bpf_dispatcher_nop_func(

 {
        return bpf_func(ctx, insnsi);
Quoted 8 lines

include/linux/bpf.h static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( · line 808

Quoted 4 lines

}

include/linux/bpf.h +3 -0
+
+struct bpf_link;
+

is this forward declaration still needed? was it supposed to be a struct bpf_tramp_link instead? and also probably higher above, before bpf_tramp_links?

You are right, I should remvoe it.

Quoted 46 lines
Quoted 46 lines

#ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
struct bpf_trampoline *bpf_trampoline_get(u64 key,
struct
bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);

include/linux/bpf.h +2 -2
@@ -856,12 +859,12 @@ int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link
 *link,
                                           struct bpf_trampoline
 *tr)
 {
        return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog
 *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link

*link,
struct bpf_trampoline
*tr)

 {
        return -ENOTSUPP;
include/linux/bpf.h +0 -1
@@ -960,7 +963,6 @@ struct bpf_prog_aux {
        bool tail_call_reachable;
        bool xdp_has_frags;
        bool use_bpf_prog_pack;
-       struct hlist_node tramp_hlist;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */
@@ -1047,6 +1049,18 @@ struct bpf_link_ops {
                              struct bpf_link_info *info);
 };
 
Quoted 15 lines

include/linux/bpf.h struct bpf_link_ops { · line 1062

Quoted 12 lines
include/linux/bpf.h +5 -0
+struct bpf_tramp_link {
+       struct bpf_link link;
+       struct hlist_node tramp_hlist;
+};
+

https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing

include/linux/bpf.h +6 -0
+struct bpf_tracing_link {
+       struct bpf_tramp_link link;
+       enum bpf_attach_type attach_type;
+       struct bpf_trampoline *trampoline;
+       struct bpf_prog *tgt_prog;
+};

struct bpf_tracing_link can stay in syscall.c, no? don't see anyone needing it outside of syscall.c

Quoted 3 lines

It will be used by invoke_bpf_prog() of bpf_jit_comp.c in the 3rd patch to get the cookie value.

Yep, saw it when I got to it, forgot to mention it here. Ignore this suggestion.

Quoted 22 lines
Quoted 21 lines
Quoted 19 lines
include/linux/bpf.h +3 -2
+
 struct bpf_link_primer {
        struct bpf_link *link;
        struct file *file;
@@ -1084,8 +1098,8 @@ bool bpf_struct_ops_get(const void *kdata);
 void bpf_struct_ops_put(const void *kdata);
 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void
 *key,
                                       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs
 *tprogs,
-                                     struct bpf_prog *prog,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links
 *tlinks,
+                                     struct bpf_tramp_link *link,

const struct btf_func_model
*model,

                                      void *image, void *image_end);

[...]

#5 lore raw

PATCH v5 2/5 bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

In reply to Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---
 arch/x86/net/bpf_jit_comp.c | 55 +++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++---
 kernel/bpf/syscall.c        |  7 +++--
 kernel/bpf/trampoline.c     | 20 +++++++++++---
 4 files changed, 89 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +55 -0
index 4dcc0b1ac770..0f521be68f7b 100644
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 {
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
+	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = l->link.prog;
 
+	/* mov rdi, 0 */
+	emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+
+	/* Prepare struct bpf_tramp_run_ctx.
+	 *
+	 * bpf_tramp_run_ctx is already preserved by
+	 * arch_prepare_bpf_trampoline().
+	 *
+	 * mov QWORD PTR [rsp + ctx_cookie_off], rdi
+	 */
+	EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off);
+
 	/* arg1: mov rdi, progs[i] */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
+	/* arg2: mov rsi, rsp (struct bpf_run_ctx *) */
+	EMIT3(0x48, 0x89, 0xE6);
+
 	if (emit_call(&prog,
 		      p->aux->sleepable ? __bpf_prog_enter_sleepable :
 		      __bpf_prog_enter, prog))
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
 	/* arg2: mov rsi, rbx <- start time in nsec */
 	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+	/* arg3: mov rdx, rsp (struct bpf_run_ctx *) */
+	EMIT3(0x48, 0x89, 0xE2);
 	if (emit_call(&prog,
 		      p->aux->sleepable ? __bpf_prog_exit_sleepable :
 		      __bpf_prog_exit, prog))
@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		}
 	}
 
+	if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+		EMIT1(0x52);	/* push rdx */
+
+	if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) {
+		/* Prepare struct bpf_tramp_run_ctx.
+		 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+	}
+
 	if (fentry->nr_links)
 		if (invoke_bpf(m, &prog, fentry, regs_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET))
@@ -2098,6 +2126,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		/* pop struct bpf_tramp_run_ctx
+		 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)
+			EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+		if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+			EMIT1(0x5A); /* pop rdx */
+
 		restore_regs(m, &prog, nr_args, regs_off);
 
 		/* call original function */
@@ -2110,6 +2147,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 		im->ip_after_call = prog;
 		memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
 		prog += X86_PATCH_SIZE;
+
+		if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+			EMIT1(0x52);	/* push rdx */
+
+		/* Prepare struct bpf_tramp_run_ctx.
+		 * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+		 */
+		if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)
+			EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
 	}
 
 	if (fmod_ret->nr_links) {
@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 			goto cleanup;
 		}
 
+	/* pop struct bpf_tramp_run_ctx
+	 * add rsp, sizeof(struct bpf_tramp_run_ctx)
+	 */
+	if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)
+		EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+	if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+		EMIT1(0x5A); /* pop rdx */
+
 	if (flags & BPF_TRAMP_F_RESTORE_REGS)
 		restore_regs(m, &prog, nr_args, regs_off);
 
include/linux/bpf.h +13 -4
index 2f7909eb51ec..d87df049e6b1 100644
@@ -681,6 +681,8 @@ struct bpf_tramp_links {
 	int nr_links;
 };
 
+struct bpf_tramp_run_ctx;
+
 /* Different use cases for BPF trampoline:
  * 1. replace nop at the function entry (kprobe equivalent)
  *    flags = BPF_TRAMP_F_RESTORE_REGS
@@ -707,10 +709,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *i
 				struct bpf_tramp_links *tlinks,
 				void *orig_call);
 /* these two functions are called from generated trampoline */
-u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
-u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
-void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
+u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);
+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
+				       struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
 
@@ -1304,6 +1307,12 @@ struct bpf_trace_run_ctx {
 	u64 bpf_cookie;
 };
 
+struct bpf_tramp_run_ctx {
+	struct bpf_run_ctx run_ctx;
+	u64 bpf_cookie;
+	struct bpf_run_ctx *saved_run_ctx;
+};
+
 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
 {
 	struct bpf_run_ctx *old_ctx = NULL;
kernel/bpf/syscall.c +5 -2
index 3078c0c9317f..56e69a582b21 100644
@@ -4775,6 +4775,7 @@ static bool syscall_prog_is_valid_access(int off, int size,
 BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 {
 	struct bpf_prog * __maybe_unused prog;
+	struct bpf_tramp_run_ctx __maybe_unused run_ctx;
 
 	switch (cmd) {
 	case BPF_MAP_CREATE:
@@ -4802,13 +4803,15 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 			return -EINVAL;
 		}
 
-		if (!__bpf_prog_enter_sleepable(prog)) {
+		run_ctx.bpf_cookie = 0;
+		run_ctx.saved_run_ctx = NULL;
+		if (!__bpf_prog_enter_sleepable(prog, &run_ctx)) {
 			/* recursion detected */
 			bpf_prog_put(prog);
 			return -EBUSY;
 		}
 		attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
-		__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */);
+		__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */, &run_ctx);
 		bpf_prog_put(prog);
 		return 0;
 #endif
kernel/bpf/trampoline.c +16 -4
index d5e6bc5517cb..baf1b65d523e 100644
@@ -568,11 +568,14 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)
  * [2..MAX_U64] - execute bpf prog and record execution time.
  *     This is start time.
  */
-u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
+u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 	__acquires(RCU)
 {
 	rcu_read_lock();
 	migrate_disable();
+
+	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+
 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
 		inc_misses_counter(prog);
 		return 0;
@@ -602,29 +605,38 @@ static void notrace update_prog_stats(struct bpf_prog *prog,
 	}
 }
 
-void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)
 	__releases(RCU)
 {
+	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+
 	update_prog_stats(prog, start);
 	__this_cpu_dec(*(prog->active));
 	migrate_enable();
 	rcu_read_unlock();
 }
 
-u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
 {
 	rcu_read_lock_trace();
 	migrate_disable();
 	might_fault();
+
 	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
 		inc_misses_counter(prog);
 		return 0;
 	}
+
+	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+
 	return bpf_prog_start_time();
 }
 
-void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
+				       struct bpf_tramp_run_ctx *run_ctx)
 {
+	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+
 	update_prog_stats(prog, start);
 	__this_cpu_dec(*(prog->active));
 	migrate_enable();
Signature · 1 line
2.30.2
#6 lore raw

Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 54 lines · Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---
 arch/x86/net/bpf_jit_comp.c | 55 +++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++---
 kernel/bpf/syscall.c        |  7 +++--
 kernel/bpf/trampoline.c     | 20 +++++++++++---
 4 files changed, 89 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +18 -0
index 4dcc0b1ac770..0f521be68f7b 100644
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 {
        u8 *prog = *pprog;
        u8 *jmp_insn;
+       int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;
 
+       /* mov rdi, 0 */
+       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+
+       /* Prepare struct bpf_tramp_run_ctx.
+        *
+        * bpf_tramp_run_ctx is already preserved by
+        * arch_prepare_bpf_trampoline().
+        *
+        * mov QWORD PTR [rsp + ctx_cookie_off], rdi
+        */
+       EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off);
+
        /* arg1: mov rdi, progs[i] */
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
+       /* arg2: mov rsi, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE6);
+
        if (emit_call(&prog,
                      p->aux->sleepable ? __bpf_prog_enter_sleepable :
                      __bpf_prog_enter, prog))
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
        /* arg2: mov rsi, rbx <- start time in nsec */
        emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+       /* arg3: mov rdx, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE2);
        if (emit_call(&prog,
                      p->aux->sleepable ? __bpf_prog_exit_sleepable :
                      __bpf_prog_exit, prog))

arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2101

Quoted 5 lines
arch/x86/net/bpf_jit_comp.c +2 -0
@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                }
        }
 
+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x52);    /* push rdx */

this nr_args < 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links || ... || ...) check, why not move if (nr_args < 3) inside the if right below?

arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2103

Quoted 2 lines
arch/x86/net/bpf_jit_comp.c +2 -0
+
+       if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links) {

if (nr_args > 3) here?

Quoted 9 lines
arch/x86/net/bpf_jit_comp.c +6 -0
+               /* Prepare struct bpf_tramp_run_ctx.
+                * sub rsp, sizeof(struct bpf_tramp_run_ctx)
+                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+       }
+
        if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off,
                               flags & BPF_TRAMP_F_RET_FENTRY_RET))

[...]

arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2185

Quoted 9 lines
        if (fmod_ret->nr_links) {
arch/x86/net/bpf_jit_comp.c +4 -0
@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                        goto cleanup;
                }
 
+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx)
+        */
+       if (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links)

well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking bpf_trampoline_update() and bpf_struct_ops_prepare_trampoline() doesn't seem like we ever do this. So seems like all these checks can be dropped?

arch/x86/net/bpf_jit_comp.c int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i · line 2136

Quoted 4 lines
arch/x86/net/bpf_jit_comp.c +4 -0
+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x5A); /* pop rdx */

same, move it inside if above?

Quoted 4 lines
arch/x86/net/bpf_jit_comp.c +1 -0
+
        if (flags & BPF_TRAMP_F_RESTORE_REGS)
                restore_regs(m, &prog, nr_args, regs_off);
 

[...]

#7 lore raw

Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

Quoted 106 lines · Andrii Nakryiko
Quoted 67 lines · Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---
 arch/x86/net/bpf_jit_comp.c | 55
+++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++---
 kernel/bpf/syscall.c        |  7 +++--
 kernel/bpf/trampoline.c     | 20 +++++++++++---

 4 files changed, 89 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +16 -0
index 4dcc0b1ac770..0f521be68f7b 100644
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,
  {
         u8 *prog = *pprog;
         u8 *jmp_insn;
+       int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx,
 bpf_cookie);
         struct bpf_prog *p = l->link.prog;
 
+       /* mov rdi, 0 */
+       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+
+       /* Prepare struct bpf_tramp_run_ctx.
+        *
+        * bpf_tramp_run_ctx is already preserved by
+        * arch_prepare_bpf_trampoline().
+        *
+        * mov QWORD PTR [rsp + ctx_cookie_off], rdi
+        */
+       EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off);
+
         /* arg1: mov rdi, progs[i] */
         emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32)
 (long) p);
+       /* arg2: mov rsi, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE6);
+

        if (emit_call(&prog,
                      p->aux->sleepable ?
__bpf_prog_enter_sleepable :
                      __bpf_prog_enter, prog))

arch/x86/net/bpf_jit_comp.c +2 -0
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,
         emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32)
 (long) p);
         /* arg2: mov rsi, rbx <- start time in nsec */
         emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+       /* arg3: mov rdx, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE2);
         if (emit_call(&prog,

                      p->aux->sleepable ? __bpf_prog_exit_sleepable :
                      __bpf_prog_exit, prog))

arch/x86/net/bpf_jit_comp.c +0 -0
@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
                }
        }

Quoted 3 lines

+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x52);    /* push rdx */

this nr_args < 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links || ... || ...) check, why not move if (nr_args < 3) inside the if right below?

Quoted 3 lines

+
+       if (fentry->nr_links || fexit->nr_links || fmod_ret-

Quoted 1 line

nr_links) {

if (nr_args > 3) here?

Quoted 10 lines

+               /* Prepare struct bpf_tramp_run_ctx.
+                * sub rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+       }
+
        if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))

[...]

Quoted 10 lines

        if (fmod_ret->nr_links) {

arch/x86/net/bpf_jit_comp.c +0 -0
@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
                        goto cleanup;
                }

+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx) +        */
+       if (fentry->nr_links || fexit->nr_links || fmod_ret-

Quoted 8 lines
Quoted 1 line
Quoted 1 line

nr_links)

well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking bpf_trampoline_update() and bpf_struct_ops_prepare_trampoline() doesn't seem like we ever do this.
So seems like all these checks can be dropped?

You are right. I had added this check for doing the following line only for some cases, and didn't aware the check is no more useful after changing the way of doing it.

Quoted 15 lines
Quoted 6 lines

+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x5A); /* pop rdx */

same, move it inside if above?

Quoted 4 lines

+
        if (flags & BPF_TRAMP_F_RESTORE_REGS) restore_regs(m, &prog, nr_args, regs_off);

[...]

#8 lore raw

Re: [PATCH bpf-next v5 2/5] bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack

In reply to Andrii Nakryiko
Quoted 68 lines · Andrii Nakryiko
Quoted 67 lines · Kui-Feng Lee

BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog.

---
 arch/x86/net/bpf_jit_comp.c | 55
+++++++++++++++++++++++++++++++++++++
 include/linux/bpf.h         | 17 +++++++++---
 kernel/bpf/syscall.c        |  7 +++--
 kernel/bpf/trampoline.c     | 20 +++++++++++---

 4 files changed, 89 insertions(+), 10 deletions(-)
arch/x86/net/bpf_jit_comp.c +16 -0
index 4dcc0b1ac770..0f521be68f7b 100644
@@ -1766,10 +1766,26 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,
  {
         u8 *prog = *pprog;
         u8 *jmp_insn;
+       int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx,
 bpf_cookie);
         struct bpf_prog *p = l->link.prog;
 
+       /* mov rdi, 0 */
+       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+
+       /* Prepare struct bpf_tramp_run_ctx.
+        *
+        * bpf_tramp_run_ctx is already preserved by
+        * arch_prepare_bpf_trampoline().
+        *
+        * mov QWORD PTR [rsp + ctx_cookie_off], rdi
+        */
+       EMIT4(0x48, 0x89, 0x7C, 0x24); EMIT1(ctx_cookie_off);
+
         /* arg1: mov rdi, progs[i] */
         emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32)
 (long) p);
+       /* arg2: mov rsi, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE6);
+

        if (emit_call(&prog,
                      p->aux->sleepable ?
__bpf_prog_enter_sleepable :
                      __bpf_prog_enter, prog))

arch/x86/net/bpf_jit_comp.c +2 -0
@@ -1815,6 +1831,8 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,
         emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32)
 (long) p);
         /* arg2: mov rsi, rbx <- start time in nsec */
         emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+       /* arg3: mov rdx, rsp (struct bpf_run_ctx *) */
+       EMIT3(0x48, 0x89, 0xE2);
         if (emit_call(&prog,

                      p->aux->sleepable ? __bpf_prog_exit_sleepable :
                      __bpf_prog_exit, prog))

arch/x86/net/bpf_jit_comp.c +0 -0
@@ -2079,6 +2097,16 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
                }
        }

Quoted 8 lines
Quoted 3 lines

+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x52);    /* push rdx */

this nr_args < 3 condition is new, maybe leave a comment on why we need this? Also instead of repeating this whole (fentry->nr_links || ... || ...) check, why not move if (nr_args < 3) inside the if right below?

I thought rdx is a nonvolatile (callee-saved) register. Checking ABI again, I was wrong. I am removing this part.

Quoted 54 lines
Quoted 4 lines

+
+       if (fentry->nr_links || fexit->nr_links || fmod_ret-

Quoted 1 line

nr_links) {

if (nr_args > 3) here?

Quoted 10 lines

+               /* Prepare struct bpf_tramp_run_ctx.
+                * sub rsp, sizeof(struct bpf_tramp_run_ctx) +                */
+               EMIT4(0x48, 0x83, 0xEC, sizeof(struct bpf_tramp_run_ctx));
+       }
+
        if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))

[...]

Quoted 11 lines

        if (fmod_ret->nr_links) {

arch/x86/net/bpf_jit_comp.c +0 -0
@@ -2133,6 +2179,15 @@ int arch_prepare_bpf_trampoline(struct

bpf_tramp_image *im, void *image, void *i
                        goto cleanup;
                }

+       /* pop struct bpf_tramp_run_ctx
+        * add rsp, sizeof(struct bpf_tramp_run_ctx) +        */
+       if (fentry->nr_links || fexit->nr_links || fmod_ret-

Quoted 1 line

nr_links)

well, actually, can it ever be that this condition doesn't hold? That would mean we are generating empty trampoline for some reason, no? Do we do that? Checking bpf_trampoline_update() and bpf_struct_ops_prepare_trampoline() doesn't seem like we ever do this.
So seems like all these checks can be dropped?

Quoted 6 lines

+               EMIT4(0x48, 0x83, 0xC4, sizeof(struct bpf_tramp_run_ctx));
+
+       if (nr_args < 3 && (fentry->nr_links || fexit->nr_links || fmod_ret->nr_links))
+               EMIT1(0x5A); /* pop rdx */

same, move it inside if above?

Quoted 4 lines

+
        if (flags & BPF_TRAMP_F_RESTORE_REGS) restore_regs(m, &prog, nr_args, regs_off);

[...]

#9 lore raw

PATCH v5 3/5 bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.

In reply to Kui-Feng Lee

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie to a link of a trace hook. A cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

---
 arch/x86/net/bpf_jit_comp.c    | 11 +++++++++--
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       |  8 ++++++++
 kernel/bpf/syscall.c           | 27 +++++++++++++++++++++++----
 kernel/trace/bpf_trace.c       | 17 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  7 +++++++
 6 files changed, 65 insertions(+), 6 deletions(-)
arch/x86/net/bpf_jit_comp.c +9 -2
index 0f521be68f7b..18d02dcd810a 100644
@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 			   struct bpf_tramp_link *l, int stack_size,
 			   bool save_ret)
 {
+	u64 cookie = 0;
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
 	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = l->link.prog;
 
-	/* mov rdi, 0 */
-	emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+	if (l->link.type == BPF_LINK_TYPE_TRACING) {
+		struct bpf_tracing_link *tr_link =
+			container_of(l, struct bpf_tracing_link, link);
+		cookie = tr_link->cookie;
+	}
+
+	/* mov rdi, cookie */
+	emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);
 
 	/* Prepare struct bpf_tramp_run_ctx.
 	 *
include/linux/bpf.h +1 -0
index d87df049e6b1..49db9c065701 100644
@@ -1062,6 +1062,7 @@ struct bpf_tracing_link {
 	enum bpf_attach_type attach_type;
 	struct bpf_trampoline *trampoline;
 	struct bpf_prog *tgt_prog;
+	u64 cookie;
 };
 
 struct bpf_link_primer {
include/uapi/linux/bpf.h +8 -0
index a4f557338af7..5e901e6d17ea 100644
@@ -1436,6 +1436,7 @@ union bpf_attr {
 	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
 		__u64 name;
 		__u32 prog_fd;
+		__u64 bpf_cookie;
 	} raw_tracepoint;
 
 	struct { /* anonymous struct for BPF_BTF_LOAD */
@@ -1490,6 +1491,13 @@ union bpf_attr {
 				__aligned_u64	addrs;
 				__aligned_u64	cookies;
 			} kprobe_multi;
+			struct {
+				/* black box user-provided value passed through
+				 * to BPF program at the execution time and
+				 * accessible through bpf_get_attach_cookie() BPF helper
+				 */
+				__u64		bpf_cookie;
+			} tracing;
 		};
 	} link_create;
 
kernel/bpf/syscall.c +23 -4
index 56e69a582b21..53d4da5c76b5 100644
@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
 	.fill_link_info = bpf_tracing_link_fill_link_info,
 };
 
-static int bpf_tracing_prog_attach(struct bpf_prog *prog,
-				   int tgt_prog_fd,
-				   u32 btf_id)
+static int bpf_tracing_prog_attach_cookie(struct bpf_prog *prog,
+					  int tgt_prog_fd,
+					  u32 btf_id,
+					  u64 bpf_cookie)
 {
 	struct bpf_link_primer link_primer;
 	struct bpf_prog *tgt_prog = NULL;
@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 	bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
 		      &bpf_tracing_link_lops, prog);
 	link->attach_type = prog->expected_attach_type;
+	link->cookie = bpf_cookie;
 
 	mutex_lock(&prog->aux->dst_mutex);
 
@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 	return err;
 }
 
+static int bpf_tracing_prog_attach(struct bpf_prog *prog,
+				   int tgt_prog_fd,
+				   u32 btf_id)
+{
+	return bpf_tracing_prog_attach_cookie(prog, tgt_prog_fd, btf_id, 0);
+}
+
 struct bpf_raw_tp_link {
 	struct bpf_link link;
 	struct bpf_raw_event_map *btp;
@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
 }
 #endif /* CONFIG_PERF_EVENTS */
 
-#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
+#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.bpf_cookie
 
 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
 {
@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
 		return BPF_PROG_TYPE_SK_LOOKUP;
 	case BPF_XDP:
 		return BPF_PROG_TYPE_XDP;
+	case BPF_TRACE_FENTRY:
+	case BPF_TRACE_FEXIT:
+	case BPF_MODIFY_RETURN:
+		return BPF_PROG_TYPE_TRACING;
 	default:
 		return BPF_PROG_TYPE_UNSPEC;
 	}
@@ -4251,6 +4264,12 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
 		return bpf_tracing_prog_attach(prog,
 					       attr->link_create.target_fd,
 					       attr->link_create.target_btf_id);
+	else if (prog->type == BPF_PROG_TYPE_TRACING)
+		return bpf_tracing_prog_attach_cookie(prog,
+						      0,
+						      0,
+						      attr->link_create.tracing.bpf_cookie);
+
 	return -EINVAL;
 }
 
kernel/trace/bpf_trace.c +17 -0
index b26f3da943de..c5713d5fba45 100644
@@ -1088,6 +1088,21 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = {
 	.arg1_type	= ARG_PTR_TO_CTX,
 };
 
+BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx)
+{
+	struct bpf_trace_run_ctx *run_ctx;
+
+	run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx);
+	return run_ctx->bpf_cookie;
+}
+
+static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = {
+	.func		= bpf_get_attach_cookie_tracing,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
+
 BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags)
 {
 #ifndef CONFIG_X86
@@ -1716,6 +1731,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL;
 	case BPF_FUNC_get_func_arg_cnt:
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
+	case BPF_FUNC_get_attach_cookie:
+		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
 	default:
 		fn = raw_tp_prog_func_proto(func_id, prog);
 		if (!fn && prog->expected_attach_type == BPF_TRACE_ITER)
tools/include/uapi/linux/bpf.h +7 -0
index a4f557338af7..a5ee57f09e04 100644
@@ -1490,6 +1490,13 @@ union bpf_attr {
 				__aligned_u64	addrs;
 				__aligned_u64	cookies;
 			} kprobe_multi;
+			struct {
+				/* black box user-provided value passed through
+				 * to BPF program at the execution time and
+				 * accessible through bpf_get_attach_cookie() BPF helper
+				 */
+				__u64		bpf_cookie;
+			} tracing;
 		};
 	} link_create;
 
Signature · 1 line
2.30.2
#10 lore raw

Re: [PATCH bpf-next v5 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.

Quoted 28 lines · Kui-Feng Lee

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie to a link of a trace hook. A cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

---
 arch/x86/net/bpf_jit_comp.c    | 11 +++++++++--
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       |  8 ++++++++
 kernel/bpf/syscall.c           | 27 +++++++++++++++++++++++----
 kernel/trace/bpf_trace.c       | 17 +++++++++++++++++
 tools/include/uapi/linux/bpf.h |  7 +++++++
 6 files changed, 65 insertions(+), 6 deletions(-)
arch/x86/net/bpf_jit_comp.c +1 -0
index 0f521be68f7b..18d02dcd810a 100644
@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
                           struct bpf_tramp_link *l, int stack_size,
                           bool save_ret)
 {
+       u64 cookie = 0;

arch/x86/net/bpf_jit_comp.c static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · line 1775

Quoted 10 lines
arch/x86/net/bpf_jit_comp.c +3 -2
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;
 
-       /* mov rdi, 0 */
-       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+       if (l->link.type == BPF_LINK_TYPE_TRACING) {
+               struct bpf_tracing_link *tr_link =
+                       container_of(l, struct bpf_tracing_link, link);

empty line between variable declaration block and statement

Quoted 25 lines
arch/x86/net/bpf_jit_comp.c +5 -0
+               cookie = tr_link->cookie;
+       }
+
+       /* mov rdi, cookie */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);
 
        /* Prepare struct bpf_tramp_run_ctx.
         *
include/linux/bpf.h +1 -0
index d87df049e6b1..49db9c065701 100644
@@ -1062,6 +1062,7 @@ struct bpf_tracing_link {
        enum bpf_attach_type attach_type;
        struct bpf_trampoline *trampoline;
        struct bpf_prog *tgt_prog;
+       u64 cookie;
 };
 
 struct bpf_link_primer {
include/uapi/linux/bpf.h +0 -0
index a4f557338af7..5e901e6d17ea 100644

include/uapi/linux/bpf.h union bpf_attr { · line 1439

Quoted 4 lines
include/uapi/linux/bpf.h +1 -0
@@ -1436,6 +1436,7 @@ union bpf_attr {
        struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
                __u64 name;
                __u32 prog_fd;
+               __u64 bpf_cookie;

I thought we are not adding bpf_cookie through RAW_TRACEPOINT_OPEN?

Quoted 22 lines
include/uapi/linux/bpf.h +7 -0
        } raw_tracepoint;
 
        struct { /* anonymous struct for BPF_BTF_LOAD */
@@ -1490,6 +1491,13 @@ union bpf_attr {
                                __aligned_u64   addrs;
                                __aligned_u64   cookies;
                        } kprobe_multi;
+                       struct {
+                               /* black box user-provided value passed through
+                                * to BPF program at the execution time and
+                                * accessible through bpf_get_attach_cookie() BPF helper
+                                */
+                               __u64           bpf_cookie;
+                       } tracing;
                };
        } link_create;
 
kernel/bpf/syscall.c +0 -0
index 56e69a582b21..53d4da5c76b5 100644

kernel/bpf/syscall.c static const struct bpf_link_ops bpf_tracing_link_lops = { · line 2701

Quoted 10 lines
kernel/bpf/syscall.c +4 -3
@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
        .fill_link_info = bpf_tracing_link_fill_link_info,
 };
 
-static int bpf_tracing_prog_attach(struct bpf_prog *prog,
-                                  int tgt_prog_fd,
-                                  u32 btf_id)
+static int bpf_tracing_prog_attach_cookie(struct bpf_prog *prog,
+                                         int tgt_prog_fd,
+                                         u32 btf_id,
+                                         u64 bpf_cookie)

let's keep it as bpf_tracing_prog_attach and just update all the call sites

Quoted 12 lines
kernel/bpf/syscall.c +1 -0
 {
        struct bpf_link_primer link_primer;
        struct bpf_prog *tgt_prog = NULL;
@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
        bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
                      &bpf_tracing_link_lops, prog);
        link->attach_type = prog->expected_attach_type;
+       link->cookie = bpf_cookie;
 
        mutex_lock(&prog->aux->dst_mutex);
 

kernel/bpf/syscall.c static int bpf_tracing_prog_attach(struct bpf_prog *prog, · line 2882

Quoted 10 lines
kernel/bpf/syscall.c +7 -0
@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
        return err;
 }
 
+static int bpf_tracing_prog_attach(struct bpf_prog *prog,
+                                  int tgt_prog_fd,
+                                  u32 btf_id)
+{
+       return bpf_tracing_prog_attach_cookie(prog, tgt_prog_fd, btf_id, 0);
+}
+

why this wrapper, just add that 0 for all existing bpf_tracing_prog_attach() invocations

Quoted 13 lines
kernel/bpf/syscall.c +1 -1
 struct bpf_raw_tp_link {
        struct bpf_link link;
        struct bpf_raw_event_map *btp;
@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
 }
 #endif /* CONFIG_PERF_EVENTS */
 
-#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
+#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.bpf_cookie
 
 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
 {

kernel/bpf/syscall.c attach_type_to_prog_type(enum bpf_attach_type attach_type) · line 3202

Quoted 7 lines
kernel/bpf/syscall.c +4 -0
@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
                return BPF_PROG_TYPE_SK_LOOKUP;
        case BPF_XDP:
                return BPF_PROG_TYPE_XDP;
+       case BPF_TRACE_FENTRY:
+       case BPF_TRACE_FEXIT:
+       case BPF_MODIFY_RETURN:
+               return BPF_PROG_TYPE_TRACING;

can you please split the part that adds ability to create fentry/fexit/etc links through LINK_CREATE into a separate commit, separate from wiring the bpf_cookie? It's two logically distinct changes.

Quoted 3 lines
kernel/bpf/syscall.c +0 -0
        default:
                return BPF_PROG_TYPE_UNSPEC;
        }

[...]

Quoted 5 lines
tools/include/uapi/linux/bpf.h +0 -0
index a4f557338af7..a5ee57f09e04 100644

tools/include/uapi/linux/bpf.h union bpf_attr { · line 1498

Quoted 9 lines
tools/include/uapi/linux/bpf.h +6 -0
@@ -1490,6 +1490,13 @@ union bpf_attr {
                                __aligned_u64   addrs;
                                __aligned_u64   cookies;
                        } kprobe_multi;
+                       struct {
+                               /* black box user-provided value passed through
+                                * to BPF program at the execution time and
+                                * accessible through bpf_get_attach_cookie() BPF helper
+                                */
+                               __u64           bpf_cookie;

for kprobe_multi we used "cookies", so let's do "cookie" here? We'll have to keep perf_event.bpf_cookie, of course.

Quoted 7 lines
tools/include/uapi/linux/bpf.h +1 -1
+                       } tracing;
                };
        } link_create;
 
--

2.30.2

#11 lore raw

Re: [PATCH bpf-next v5 3/5] bpf, x86: Attach a cookie to fentry/fexit/fmod_ret.

Quoted 39 lines · Andrii Nakryiko
Quoted 38 lines · Kui-Feng Lee

Add a bpf_cookie field to struct bpf_tracing_link to attach a cookie
to a link of a trace hook.  A cookie of a bpf_tracing_link is available by calling bpf_get_attach_cookie when running the BPF program of the attached link.

The value of a cookie will be set at bpf_tramp_run_ctx by the trampoline of the link.

---
 arch/x86/net/bpf_jit_comp.c    | 11 +++++++++-- include/linux/bpf.h            |  1 + include/uapi/linux/bpf.h       |  8 ++++++++ kernel/bpf/syscall.c           | 27 +++++++++++++++++++++++---- kernel/trace/bpf_trace.c       | 17 +++++++++++++++++ tools/include/uapi/linux/bpf.h |  7 +++++++

 6 files changed, 65 insertions(+), 6 deletions(-)
arch/x86/net/bpf_jit_comp.c +0 -0
index 0f521be68f7b..18d02dcd810a 100644
@@ -1764,13 +1764,20 @@ static int invoke_bpf_prog(const struct
 btf_func_model *m, u8 **pprog,

                           struct bpf_tramp_link *l, int stack_size,
                           bool save_ret)
 {
+       u64 cookie = 0;
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;

Quoted 8 lines
Quoted 6 lines

-       /* mov rdi, 0 */
-       emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
+       if (l->link.type == BPF_LINK_TYPE_TRACING) {
+               struct bpf_tracing_link *tr_link =
+                       container_of(l, struct bpf_tracing_link, link);

empty line between variable declaration block and statement

Got it!

Quoted 21 lines
Quoted 21 lines

+               cookie = tr_link->cookie;
+       }
+
+       /* mov rdi, cookie */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);

        /* Prepare struct bpf_tramp_run_ctx.
         *

include/linux/bpf.h +1 -0
index d87df049e6b1..49db9c065701 100644
@@ -1062,6 +1062,7 @@ struct bpf_tracing_link {
         enum bpf_attach_type attach_type;
         struct bpf_trampoline *trampoline;
         struct bpf_prog *tgt_prog;
+       u64 cookie;
  };
 
Quoted 13 lines
Quoted 11 lines
include/linux/bpf.h +0 -0
  struct bpf_link_primer {
include/uapi/linux/bpf.h +0 -0
index a4f557338af7..5e901e6d17ea 100644
@@ -1436,6 +1436,7 @@ union bpf_attr {

        struct { /* anonymous struct used by
BPF_RAW_TRACEPOINT_OPEN command */
                __u64 name;
                __u32 prog_fd;
+               __u64 bpf_cookie;

I thought we are not adding bpf_cookie through RAW_TRACEPOINT_OPEN?

Yes, this line should be removed.

Quoted 30 lines
Quoted 30 lines

        } raw_tracepoint;

        struct { /* anonymous struct for BPF_BTF_LOAD */

include/uapi/linux/bpf.h +2 -0
@@ -1490,6 +1491,13 @@ union bpf_attr {
                                 __aligned_u64   addrs;
                                 __aligned_u64   cookies;
                         } kprobe_multi;
+                       struct {
+                               /* black box user-provided value

passed through
+                                * to BPF program at the execution time and
+                                * accessible through bpf_get_attach_cookie() BPF helper
+                                */
+                               __u64           bpf_cookie; +                       } tracing;
                };
        } link_create;

kernel/bpf/syscall.c +0 -0
index 56e69a582b21..53d4da5c76b5 100644
@@ -2695,9 +2695,10 @@ static const struct bpf_link_ops
 bpf_tracing_link_lops = {
         .fill_link_info = bpf_tracing_link_fill_link_info,
  };
 
Quoted 10 lines

kernel/bpf/syscall.c static const struct bpf_link_ops bpf_tracing_link_lops = { · line 2701

Quoted 7 lines
kernel/bpf/syscall.c +4 -3
-static int bpf_tracing_prog_attach(struct bpf_prog *prog,
-                                  int tgt_prog_fd,
-                                  u32 btf_id)
+static int bpf_tracing_prog_attach_cookie(struct bpf_prog *prog,
+                                         int tgt_prog_fd,
+                                         u32 btf_id,
+                                         u64 bpf_cookie)

let's keep it as bpf_tracing_prog_attach and just update all the call sites

Roger

Quoted 18 lines
Quoted 18 lines
kernel/bpf/syscall.c +0 -0
  {
         struct bpf_link_primer link_primer;

        struct bpf_prog *tgt_prog = NULL;

kernel/bpf/syscall.c +1 -0
@@ -2762,6 +2763,7 @@ static int bpf_tracing_prog_attach(struct
 bpf_prog *prog,
         bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
                       &bpf_tracing_link_lops, prog);
         link->attach_type = prog->expected_attach_type;
+       link->cookie = bpf_cookie;
 
         mutex_lock(&prog->aux->dst_mutex);
@@ -2871,6 +2873,13 @@ static int bpf_tracing_prog_attach(struct
 bpf_prog *prog,
         return err;
  }
 
Quoted 12 lines
Quoted 9 lines
kernel/bpf/syscall.c +6 -0
+static int bpf_tracing_prog_attach(struct bpf_prog *prog,
+                                  int tgt_prog_fd,
+                                  u32 btf_id)
+{
+       return bpf_tracing_prog_attach_cookie(prog, tgt_prog_fd,
 btf_id, 0);
+}

https://docs.google.com/document/d/1FYYdi1Hw4pjulbkvI1VmG-kGqTJRCg7bh1vAF4bwjMc/edit?usp=sharing
+

why this wrapper, just add that 0 for all existing bpf_tracing_prog_attach() invocations

Got it!

Quoted 14 lines
Quoted 14 lines

 struct bpf_raw_tp_link {
        struct bpf_link link;
        struct bpf_raw_event_map *btp;

kernel/bpf/syscall.c +0 -0
@@ -3023,7 +3032,7 @@ static int bpf_perf_link_attach(const union

bpf_attr *attr, struct bpf_prog *pro
 }
 #endif /* CONFIG_PERF_EVENTS */

kernel/bpf/syscall.c +1 -1
-#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
+#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD

raw_tracepoint.bpf_cookie

Quoted 16 lines
Quoted 11 lines

 static int bpf_raw_tracepoint_open(const union bpf_attr *attr) {

kernel/bpf/syscall.c +0 -0
@@ -3187,6 +3196,10 @@ attach_type_to_prog_type(enum
 bpf_attach_type attach_type)
                 return BPF_PROG_TYPE_SK_LOOKUP;

        case BPF_XDP:
                return BPF_PROG_TYPE_XDP;
+       case BPF_TRACE_FENTRY:
+       case BPF_TRACE_FEXIT:
+       case BPF_MODIFY_RETURN:
+               return BPF_PROG_TYPE_TRACING;

can you please split the part that adds ability to create fentry/fexit/etc links through LINK_CREATE into a separate commit, separate from wiring the bpf_cookie? It's two logically distinct changes.

No problem.

Quoted 20 lines
Quoted 4 lines

        default:
                return BPF_PROG_TYPE_UNSPEC;
        }

[...]

Quoted 13 lines
tools/include/uapi/linux/bpf.h +2 -0
index a4f557338af7..a5ee57f09e04 100644
@@ -1490,6 +1490,13 @@ union bpf_attr {
                                 __aligned_u64   addrs;
                                 __aligned_u64   cookies;
                         } kprobe_multi;
+                       struct {
+                               /* black box user-provided value

passed through
+                                * to BPF program at the execution

Quoted 8 lines
Quoted 5 lines

time and
+                                * accessible through bpf_get_attach_cookie() BPF helper
+                                */
+                               __u64           bpf_cookie;

for kprobe_multi we used "cookies", so let's do "cookie" here? We'll have to keep perf_event.bpf_cookie, of course.

Got it!

Quoted 8 lines
Quoted 8 lines

+                       } tracing;
                };
        } link_create;

--
2.30.2

#12 lore raw

PATCH v5 4/5 lib/bpf: Assign cookies to links in libbpf.

In reply to Kui-Feng Lee

Add a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.

---
 tools/lib/bpf/bpf.c      |  7 +++++++
 tools/lib/bpf/bpf.h      |  3 +++
 tools/lib/bpf/libbpf.c   | 33 +++++++++++++++++++++++++++++++++
 tools/lib/bpf/libbpf.h   | 12 ++++++++++++
 tools/lib/bpf/libbpf.map |  1 +
 5 files changed, 56 insertions(+)
tools/lib/bpf/bpf.c +7 -0
index cf27251adb92..c2454979c3c4 100644
@@ -863,6 +863,13 @@ int bpf_link_create(int prog_fd, int target_fd,
 		if (!OPTS_ZEROED(opts, kprobe_multi))
 			return libbpf_err(-EINVAL);
 		break;
+	case BPF_TRACE_FENTRY:
+	case BPF_TRACE_FEXIT:
+	case BPF_MODIFY_RETURN:
+		attr.link_create.tracing.bpf_cookie = OPTS_GET(opts, tracing.bpf_cookie, 0);
+		if (!OPTS_ZEROED(opts, tracing))
+			return libbpf_err(-EINVAL);
+		break;
 	default:
 		if (!OPTS_ZEROED(opts, flags))
 			return libbpf_err(-EINVAL);
tools/lib/bpf/bpf.h +3 -0
index f4b4afb6d4ba..4cdbabcccefc 100644
@@ -410,6 +410,9 @@ struct bpf_link_create_opts {
 	__u32 iter_info_len;
 	__u32 target_btf_id;
 	union {
+		struct {
+			__u64 bpf_cookie;
+		} tracing;
 		struct {
 			__u64 bpf_cookie;
 		} perf_event;
tools/lib/bpf/libbpf.c +33 -0
index bf4f7ac54ebf..8586e1efd780 100644
@@ -11262,6 +11262,39 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
 	return bpf_program__attach_btf_id(prog);
 }
 
+struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
+						const struct bpf_trace_opts *opts)
+{
+	char errmsg[STRERR_BUFSIZE];
+	struct bpf_link *link;
+	int prog_fd, pfd;
+
+	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+	prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
+
+	link = calloc(1, sizeof(*link));
+	if (!link)
+		return libbpf_err_ptr(-ENOMEM);
+	link->detach = &bpf_link__detach_fd;
+
+	link_opts.tracing.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
+	pfd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &link_opts);
+	if (pfd < 0) {
+		pfd = -errno;
+		free(link);
+		pr_warn("prog '%s': failed to attach: %s\n",
+			prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
+		return libbpf_err_ptr(pfd);
+	}
+	link->fd = pfd;
+	return (struct bpf_link *)link;
+}
+
 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
 {
 	return bpf_program__attach_btf_id(prog);
tools/lib/bpf/libbpf.h +12 -0
index 63d66f1adf1a..e0dd3f9a5aca 100644
@@ -563,8 +563,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
 LIBBPF_API struct bpf_link *
 bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
 				   const char *tp_name);
+
+struct bpf_trace_opts {
+	/* size of this struct, for forward/backward compatibility */
+	size_t sz;
+	/* custom user-provided value fetchable through bpf_get_attach_cookie() */
+	__u64 bpf_cookie;
+};
+#define bpf_trace_opts__last_field bpf_cookie
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_trace(const struct bpf_program *prog);
+LIBBPF_API struct bpf_link *
+bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts);
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_lsm(const struct bpf_program *prog);
 LIBBPF_API struct bpf_link *
tools/lib/bpf/libbpf.map +1 -0
index 82f6d62176dd..9235da802e31 100644
@@ -424,6 +424,7 @@ LIBBPF_0.6.0 {
 LIBBPF_0.7.0 {
 	global:
 		bpf_btf_load;
+		bpf_program__attach_trace_opts;
 		bpf_program__expected_attach_type;
 		bpf_program__log_buf;
 		bpf_program__log_level;
Signature · 1 line
2.30.2
#13 lore raw

Re: [PATCH bpf-next v5 4/5] lib/bpf: Assign cookies to links in libbpf.

Quoted 6 lines · Kui-Feng Lee

Add a cookie field to the attributes of bpf_link_create(). Add bpf_program__attach_trace_opts() to attach a cookie to a link.

---

Please stick to "libbpf: " prefix for patch subject, we don't use "lib/bpf: "

Quoted 44 lines
 tools/lib/bpf/bpf.c      |  7 +++++++
 tools/lib/bpf/bpf.h      |  3 +++
 tools/lib/bpf/libbpf.c   | 33 +++++++++++++++++++++++++++++++++
 tools/lib/bpf/libbpf.h   | 12 ++++++++++++
 tools/lib/bpf/libbpf.map |  1 +
 5 files changed, 56 insertions(+)
tools/lib/bpf/bpf.c +7 -0
index cf27251adb92..c2454979c3c4 100644
@@ -863,6 +863,13 @@ int bpf_link_create(int prog_fd, int target_fd,
                if (!OPTS_ZEROED(opts, kprobe_multi))
                        return libbpf_err(-EINVAL);
                break;
+       case BPF_TRACE_FENTRY:
+       case BPF_TRACE_FEXIT:
+       case BPF_MODIFY_RETURN:
+               attr.link_create.tracing.bpf_cookie = OPTS_GET(opts, tracing.bpf_cookie, 0);
+               if (!OPTS_ZEROED(opts, tracing))
+                       return libbpf_err(-EINVAL);
+               break;
        default:
                if (!OPTS_ZEROED(opts, flags))
                        return libbpf_err(-EINVAL);
tools/lib/bpf/bpf.h +3 -0
index f4b4afb6d4ba..4cdbabcccefc 100644
@@ -410,6 +410,9 @@ struct bpf_link_create_opts {
        __u32 iter_info_len;
        __u32 target_btf_id;
        union {
+               struct {
+                       __u64 bpf_cookie;
+               } tracing;
                struct {
                        __u64 bpf_cookie;
                } perf_event;
tools/lib/bpf/libbpf.c +0 -0
index bf4f7ac54ebf..8586e1efd780 100644

tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11271

Quoted 10 lines
tools/lib/bpf/libbpf.c +7 -0
@@ -11262,6 +11262,39 @@ struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
        return bpf_program__attach_btf_id(prog);
 }
 
+struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
+                                               const struct bpf_trace_opts *opts)
+{
+       char errmsg[STRERR_BUFSIZE];
+       struct bpf_link *link;
+       int prog_fd, pfd;
+

unnecessary empty line, LIBBPF_OPTS is a variable declaration, so keep it together with other variables

Quoted 5 lines
tools/lib/bpf/libbpf.c +5 -0
+       LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+       prog_fd = bpf_program__fd(prog);
+       if (prog_fd < 0) {
+               pr_warn("prog '%s': can't attach before loaded\n", prog->name);

tools/lib/bpf/libbpf.c struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) · line 11286

Quoted 10 lines
tools/lib/bpf/libbpf.c +10 -0
+               return libbpf_err_ptr(-EINVAL);
+       }
+
+       link = calloc(1, sizeof(*link));
+       if (!link)
+               return libbpf_err_ptr(-ENOMEM);
+       link->detach = &bpf_link__detach_fd;
+
+       link_opts.tracing.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
+       pfd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &link_opts);

so this won't work on old kernels that don't support creating fentry/fexit links through LINK_CREATE. Let's for now check if bpf_cookie is zero, then redirect to bpf_program__attach_trace(). Ideally we should perform feature detection of ability to use LINK_CREATE to attach fentry and have a unified single bpf_program__attach_btf_id() implementation that will handle all that internally

Quoted 44 lines
tools/lib/bpf/libbpf.c +11 -0
+       if (pfd < 0) {
+               pfd = -errno;
+               free(link);
+               pr_warn("prog '%s': failed to attach: %s\n",
+                       prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
+               return libbpf_err_ptr(pfd);
+       }
+       link->fd = pfd;
+       return (struct bpf_link *)link;
+}
+
 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
 {
        return bpf_program__attach_btf_id(prog);
tools/lib/bpf/libbpf.h +12 -0
index 63d66f1adf1a..e0dd3f9a5aca 100644
@@ -563,8 +563,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
 LIBBPF_API struct bpf_link *
 bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
                                   const char *tp_name);
+
+struct bpf_trace_opts {
+       /* size of this struct, for forward/backward compatibility */
+       size_t sz;
+       /* custom user-provided value fetchable through bpf_get_attach_cookie() */
+       __u64 bpf_cookie;
+};
+#define bpf_trace_opts__last_field bpf_cookie
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_trace(const struct bpf_program *prog);
+LIBBPF_API struct bpf_link *
+bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts);
+
 LIBBPF_API struct bpf_link *
 bpf_program__attach_lsm(const struct bpf_program *prog);
 LIBBPF_API struct bpf_link *
tools/lib/bpf/libbpf.map +0 -0
index 82f6d62176dd..9235da802e31 100644

tools/lib/bpf/libbpf.map LIBBPF_0.6.0 { · line 427

Quoted 4 lines
tools/lib/bpf/libbpf.map +1 -0
@@ -424,6 +424,7 @@ LIBBPF_0.6.0 {
 LIBBPF_0.7.0 {
        global:
                bpf_btf_load;
+               bpf_program__attach_trace_opts;

this should go into LIBBPF_0.8.0

Quoted 6 lines
tools/lib/bpf/libbpf.map +0 -1
                bpf_program__expected_attach_type;
                bpf_program__log_buf;
                bpf_program__log_level;
--

2.30.2

#14 lore raw

PATCH v5 5/5 selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.

In reply to Kui-Feng Lee

Make sure BPF cookies are correct for fentry/fexit/fmod_ret.

---
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 52 +++++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 24 +++++++++
 2 files changed, 76 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +52 -0
index 923a6139b2d8..7f05056c66d4 100644
@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie *skel)
 	bpf_link__destroy(link);
 }
 
+static void tracing_subtest(struct test_bpf_cookie *skel)
+{
+	__u64 cookie;
+	int prog_fd;
+	int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1;
+
+	LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1);
+	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+	skel->bss->fentry_res = 0;
+	skel->bss->fexit_res = 0;
+
+	cookie = 0x100000;
+	prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+	link_opts.tracing.bpf_cookie = cookie;
+	fentry_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY, &link_opts);
+
+	cookie = 0x200000;
+	prog_fd = bpf_program__fd(skel->progs.fexit_test1);
+	link_opts.tracing.bpf_cookie = cookie;
+	fexit_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FEXIT, &link_opts);
+	if (!ASSERT_GE(fexit_fd, 0, "fexit.open"))
+		goto cleanup;
+
+	cookie = 0x300000;
+	prog_fd = bpf_program__fd(skel->progs.fmod_ret_test);
+	link_opts.tracing.bpf_cookie = cookie;
+	fmod_ret_fd = bpf_link_create(prog_fd, 0, BPF_MODIFY_RETURN, &link_opts);
+	if (!ASSERT_GE(fmod_ret_fd, 0, "fmod_ret.opoen"))
+		goto cleanup;
+
+	prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+	bpf_prog_test_run_opts(prog_fd, &opts);
+
+	prog_fd = bpf_program__fd(skel->progs.fmod_ret_test);
+	bpf_prog_test_run_opts(prog_fd, &opts);
+
+	ASSERT_EQ(skel->bss->fentry_res, 0x100000, "fentry_res");
+	ASSERT_EQ(skel->bss->fexit_res, 0x200000, "fexit_res");
+	ASSERT_EQ(skel->bss->fmod_ret_res, 0x300000, "fmod_ret_res");
+
+cleanup:
+	if (fentry_fd >= 0)
+		close(fentry_fd);
+	if (fexit_fd >= 0)
+		close(fexit_fd);
+	if (fmod_ret_fd >= 0)
+		close(fmod_ret_fd);
+}
+
 void test_bpf_cookie(void)
 {
 	struct test_bpf_cookie *skel;
@@ -432,6 +482,8 @@ void test_bpf_cookie(void)
 		tp_subtest(skel);
 	if (test__start_subtest("perf_event"))
 		pe_subtest(skel);
+	if (test__start_subtest("trampoline"))
+		tracing_subtest(skel);
 
 	test_bpf_cookie__destroy(skel);
 }
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +24 -0
index 0e2222968918..c37bc6ad0b70 100644
@@ -14,6 +14,9 @@ int uprobe_res;
 int uretprobe_res;
 int tp_res;
 int pe_res;
+int fentry_res;
+int fexit_res;
+int fmod_ret_res;
 
 static void update(void *ctx, int *res)
 {
@@ -82,4 +85,25 @@ int handle_pe(struct pt_regs *ctx)
 	return 0;
 }
 
+SEC("fentry/bpf_fentry_test1")
+int BPF_PROG(fentry_test1, int a)
+{
+	update(ctx, &fentry_res);
+	return 0;
+}
+
+SEC("fexit/bpf_fentry_test1")
+int BPF_PROG(fexit_test1, int a, int ret)
+{
+	update(ctx, &fexit_res);
+	return 0;
+}
+
+SEC("fmod_ret/bpf_modify_return_test")
+int BPF_PROG(fmod_ret_test, int _a, int *_b, int _ret)
+{
+	update(ctx, &fmod_ret_res);
+	return 1234;
+}
+
 char _license[] SEC("license") = "GPL";
Signature · 1 line
2.30.2
#15 lore raw

Re: [PATCH bpf-next v5 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.

Quoted 14 lines · Kui-Feng Lee

Make sure BPF cookies are correct for fentry/fexit/fmod_ret.

---
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 52 +++++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 24 +++++++++
 2 files changed, 76 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +0 -0
index 923a6139b2d8..7f05056c66d4 100644

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 418

Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6 -0
@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie *skel)
        bpf_link__destroy(link);
 }
 
+static void tracing_subtest(struct test_bpf_cookie *skel)
+{
+       __u64 cookie;
+       int prog_fd;
+       int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1;
+

unnecessary empty line

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 419

Quoted 1 line
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1 -0
+       LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1);

.repeat = 1 is not necessary, I think, .repeat = 0 is equivalent to that

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 425

Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6 -0
+       LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+       skel->bss->fentry_res = 0;
+       skel->bss->fexit_res = 0;
+
+       cookie = 0x100000;

nit: make this value bigger to make sure higher 32 bits of u64 are preserved properly. Maybe 0x1000000010000000 (and similarly with 2 and
3)

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 429

Quoted 4 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +4 -0
+       prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+       link_opts.tracing.bpf_cookie = cookie;
+       fentry_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY, &link_opts);
+

ASSERT_GE?

Quoted 7 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +7 -0
+       cookie = 0x200000;
+       prog_fd = bpf_program__fd(skel->progs.fexit_test1);
+       link_opts.tracing.bpf_cookie = cookie;
+       fexit_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FEXIT, &link_opts);
+       if (!ASSERT_GE(fexit_fd, 0, "fexit.open"))
+               goto cleanup;
+

[...]

#16 lore raw

Re: [PATCH bpf-next v5 5/5] selftest/bpf: The test cses of BPF cookie for fentry/fexit/fmod_ret.

Quoted 21 lines · Andrii Nakryiko
Quoted 20 lines · Kui-Feng Lee

Make sure BPF cookies are correct for fentry/fexit/fmod_ret.

---
 .../selftests/bpf/prog_tests/bpf_cookie.c     | 52
+++++++++++++++++++
 .../selftests/bpf/progs/test_bpf_cookie.c     | 24 +++++++++

 2 files changed, 76 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +0 -0
index 923a6139b2d8..7f05056c66d4 100644
@@ -410,6 +410,56 @@ static void pe_subtest(struct test_bpf_cookie
 *skel)
         bpf_link__destroy(link);
  }
 
Quoted 8 lines

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 418

Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6 -0
+static void tracing_subtest(struct test_bpf_cookie *skel)
+{
+       __u64 cookie;
+       int prog_fd;
+       int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1;
+

unnecessary empty line

Got it!

Quoted 5 lines

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 419

Quoted 2 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1 -0
+       LIBBPF_OPTS(bpf_test_run_opts, opts, .repeat = 1);

.repeat = 1 is not necessary, I think, .repeat = 0 is equivalent to that

I will test it.

Quoted 12 lines

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 425

Quoted 7 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +6 -0
+       LIBBPF_OPTS(bpf_link_create_opts, link_opts);
+
+       skel->bss->fentry_res = 0;
+       skel->bss->fexit_res = 0;
+
+       cookie = 0x100000;

nit: make this value bigger to make sure higher 32 bits of u64 are preserved properly. Maybe 0x1000000010000000 (and similarly with 2 and
3)

Ok!

Quoted 8 lines

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c static void pe_subtest(struct test_bpf_cookie *skel) · line 429

Quoted 6 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +4 -0
+       prog_fd = bpf_program__fd(skel->progs.fentry_test1);
+       link_opts.tracing.bpf_cookie = cookie;
+       fentry_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FENTRY,
 &link_opts);
+

ASSERT_GE?

sure!

Quoted 11 lines
Quoted 9 lines
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +7 -0
+       cookie = 0x200000;
+       prog_fd = bpf_program__fd(skel->progs.fexit_test1);
+       link_opts.tracing.bpf_cookie = cookie;
+       fexit_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_FEXIT,
 &link_opts);
+       if (!ASSERT_GE(fexit_fd, 0, "fexit.open"))
+               goto cleanup;
+

[...]