Skip to contentSkip to conversation

Add bpf_access_process_vm helper and sleepable bpf iterator programs

PATCH Superseded Kenny Yu 6 participants 62 messages Jan 13, 2022 – Jan 25, 2022

Subsystems (9)
Kselftest BPF Tracing BPF selftests +5 more subsystems
lore
Share & tools
lore

Kenny Yu

author
#0 lore raw

cover Add bpf_access_process_vm helper and sleepable bpf iterator programs

This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 10 +++++
 kernel/bpf/helpers.c                          | 19 ++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 10 +++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 43 +++++++++++++++++++
 8 files changed, 102 insertions(+)
Signature · 1 line
2.30.2
author
#2 lore raw

PATCH 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fdb3536afa7d..9ec40835bce8 100644
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#1 lore raw

PATCH 1/3 bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 10 ++++++++++
 kernel/bpf/helpers.c           | 19 +++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 10 ++++++++++
 5 files changed, 42 insertions(+)
include/linux/bpf.h +1 -0
index 6e947cd91152..d3d0ef8df148 100644
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
index b0383d371b9a..4b47ec8ae569 100644
@@ -5018,6 +5018,15 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags)
+ *	Description
+ *		Read *len* bytes from user space address *addr* in *tsk*'s
+ *		address space, and stores the data in *buf*. This is a wrapper
+ *		of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5215,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +19 -0
index 01cfdf40c838..dd588912e197 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,24 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, struct task_struct *, tsk,
+	   unsigned long, addr, void *, buf, int, len, unsigned int, gup_flags)
+{
+	return access_process_vm(tsk, addr, buf, len, gup_flags);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+	.func		= bpf_access_process_vm,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_BTF_ID,
+	.arg1_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg2_type	= ARG_ANYTHING,
+	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg5_type	= ARG_ANYTHING,
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_find_vma_proto;
 	case BPF_FUNC_trace_vprintk:
 		return bpf_get_trace_vprintk_proto();
+	case BPF_FUNC_access_process_vm:
+		return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
tools/include/uapi/linux/bpf.h +10 -0
index b0383d371b9a..4b47ec8ae569 100644
@@ -5018,6 +5018,15 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags)
+ *	Description
+ *		Read *len* bytes from user space address *addr* in *tsk*'s
+ *		address space, and stores the data in *buf*. This is a wrapper
+ *		of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5215,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
author
#3 lore raw

PATCH 3/3 selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 ++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 70 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
index b84f859b1267..fcda0ecd8746 100644
@@ -138,6 +138,20 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (CHECK(!skel, "bpf_iter_task__open_and_load",
+		  "skeleton open_and_load failed\n"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0

63 lines not on this page · raw message

Signature · 1 line
2.30.2
author
#4 lore raw

v2 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (4):
bpf: Add bpf_access_process_vm() helper
bpf: Add support for sleepable programs in bpf_iter_run_prog
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 10 ++++
 kernel/bpf/bpf_iter.c                         | 16 ++++--
 kernel/bpf/helpers.c                          | 19 +++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 10 ++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 ++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 9 files changed, 126 insertions(+), 3 deletions(-)
Signature · 1 line
2.30.2
author
#10 lore raw

PATCH v2 4/4 selftests/bpf: Add test for sleepable bpf iterator programs

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 ++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 70 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
index b84f859b1267..fcda0ecd8746 100644
@@ -138,6 +138,20 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (CHECK(!skel, "bpf_iter_task__open_and_load",
+		  "skeleton open_and_load failed\n"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0

63 lines not on this page · raw message

Signature · 1 line
2.30.2
author
#6 lore raw

PATCH v2 2/4 bpf: Add support for sleepable programs in bpf_iter_run_prog

In reply to Kenny Yu

The next patch adds the ability to create sleepable bpf iterator programs. This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

---
 kernel/bpf/bpf_iter.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
kernel/bpf/bpf_iter.c +13 -3
index b7aef5b3416d..d814ca6454af 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+	}
 	ret = bpf_prog_run(prog, ctx);
 	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable)
+		rcu_read_unlock_trace();
+	else
+		rcu_read_unlock();
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
Signature · 1 line
2.30.2
author
#9 lore raw

PATCH v2 3/4 libbpf: Add "iter.s" section for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fdb3536afa7d..9ec40835bce8 100644
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#5 lore raw

PATCH v2 1/4 bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 10 ++++++++++
 kernel/bpf/helpers.c           | 19 +++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 10 ++++++++++
 5 files changed, 42 insertions(+)
include/linux/bpf.h +1 -0
index 6e947cd91152..d3d0ef8df148 100644
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
index b0383d371b9a..4b47ec8ae569 100644
@@ -5018,6 +5018,15 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags)
+ *	Description
+ *		Read *len* bytes from user space address *addr* in *tsk*'s
+ *		address space, and stores the data in *buf*. This is a wrapper
+ *		of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5215,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +19 -0
index 01cfdf40c838..dd588912e197 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,24 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, struct task_struct *, tsk,
+	   unsigned long, addr, void *, buf, int, len, unsigned int, gup_flags)
+{
+	return access_process_vm(tsk, addr, buf, len, gup_flags);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+	.func		= bpf_access_process_vm,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_BTF_ID,
+	.arg1_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg2_type	= ARG_ANYTHING,
+	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg5_type	= ARG_ANYTHING,
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_find_vma_proto;
 	case BPF_FUNC_trace_vprintk:
 		return bpf_get_trace_vprintk_proto();
+	case BPF_FUNC_access_process_vm:
+		return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
tools/include/uapi/linux/bpf.h +10 -0
index b0383d371b9a..4b47ec8ae569 100644
@@ -5018,6 +5018,15 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags)
+ *	Description
+ *		Read *len* bytes from user space address *addr* in *tsk*'s
+ *		address space, and stores the data in *buf*. This is a wrapper
+ *		of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5215,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
#11 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu
Quoted 58 lines · Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 ++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 70 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
index b84f859b1267..fcda0ecd8746 100644
@@ -138,6 +138,20 @@ static void test_task(void)
        bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+       struct bpf_iter_task *skel;
+
+       skel = bpf_iter_task__open_and_load();
+       if (CHECK(!skel, "bpf_iter_task__open_and_load",
+                 "skeleton open_and_load failed\n"))
+               return;
+
+       do_dummy_read(skel->progs.dump_task_sleepable);
+
+       bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
        struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)
                test_bpf_map();
        if (test__start_subtest("task"))
                test_task();
+       if (test__start_subtest("task_sleepable"))
+               test_task_sleepable();
        if (test__start_subtest("task_stack"))
                test_task_stack();
        if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +1 -0
index c86b93f33b32..bb4b63043533 100644
@@ -2,6 +2,7 @@
 /* Copyright (c) 2020 Facebook */
 #include "bpf_iter.h"
 #include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
 
 char _license[] SEC("license") = "GPL";
 
@@ -23,3 +24,56 @@ int dump_task(struct bpf_iter__task *ctx)
        BPF_SEQ_PRINTF(seq, "%8d %8d\n", task->tgid, task->pid);

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34

Quoted 10 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +8 -0
        return 0;
 }
+
+// New helper added
+static long (*bpf_access_process_vm)(
+       struct task_struct *tsk,
+       unsigned long addr,
+       void *buf,
+       int len,
+       unsigned int gup_flags) = (void *)186;

This shouldn't be needed.
Since patch 1 updates tools/include/uapi/linux/bpf.h it will be in bpf_helper_defs.h automatically.

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37

Quoted 3 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3 -0
+
+// Copied from include/linux/mm.h
+#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */

Please use C style comments only.

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68

Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5 -0
+       numread = bpf_access_process_vm(task,
+                                       (unsigned long)ptr,
+                                       (void *)&user_data,
+                                       sizeof(uint32_t),
+                                       FOLL_REMOTE);

We probably would need to hide flags like FOLL_REMOTE inside the helper otherwise prog might confuse the kernel. In this case I'm not even sure that FOLL_REMOTE is needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything. fast_gup is not necessary either.

#7 lore raw

Re: [PATCH v2 bpf-next 2/4] bpf: Add support for sleepable programs in bpf_iter_run_prog

In reply to Kenny Yu
Quoted 25 lines · Kenny Yu

The next patch adds the ability to create sleepable bpf iterator programs. This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

---
 kernel/bpf/bpf_iter.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
kernel/bpf/bpf_iter.c +1 -0
index b7aef5b3416d..d814ca6454af 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
        struct list_head list;

kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 689

Quoted 7 lines
kernel/bpf/bpf_iter.c +2 -2
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
        int ret;
 
-       rcu_read_lock();
-       migrate_disable();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();

Pretty cool that a single 'if' is all that is needed to enable sleepable iterators.

Maybe combine under one 'if' ?
if (prog->aux->sleepable) {
lock_trace
migr_dis
might_fault
bpf_prog_run
migr_en
unlock_trace
} else {
lock
migr_dis
bpf_prog_run
migr_end
unlock
}

Would it be easier to read?

author
#12 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Alexei Starovoitov

Hi Alexei,

Quoted 11 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34

Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7 -0
+// New helper added
+static long (*bpf_access_process_vm)(
+       struct task_struct *tsk,
+       unsigned long addr,
+       void *buf,
+       int len,
+       unsigned int gup_flags) = (void *)186;

This shouldn't be needed.
Since patch 1 updates tools/include/uapi/linux/bpf.h it will be in bpf_helper_defs.h automatically.

I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?

Quoted 5 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37

Quoted 3 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3 -0
+
+// Copied from include/linux/mm.h
+#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */

Please use C style comments only.

I will fix.

Quoted 6 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68

Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5 -0
+       numread = bpf_access_process_vm(task,
+                                       (unsigned long)ptr,
+                                       (void *)&user_data,
+                                       sizeof(uint32_t),
+                                       FOLL_REMOTE);
Quoted 6 lines

We probably would need to hide flags like FOLL_REMOTE inside the helper otherwise prog might confuse the kernel. In this case I'm not even sure that FOLL_REMOTE is needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything. fast_gup is not necessary either.

Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:

bpf_access_process_vm(struct task_struct *tsk,
unsigned long addr,
void *buf,
int len);

Thanks for the feedback!

Kenny

author
#8 lore raw

Re: [PATCH v2 bpf-next 2/4] bpf: Add support for sleepable programs in bpf_iter_run_prog

In reply to Alexei Starovoitov

Hi Alexei,

Quoted 19 lines

Pretty cool that a single 'if' is all that is needed to enable sleepable iterators.

Maybe combine under one 'if' ?
if (prog->aux->sleepable) {
lock_trace
migr_dis
might_fault
bpf_prog_run
migr_en
unlock_trace
} else {
lock
migr_dis
bpf_prog_run
migr_end
unlock
}

Quoted 1 line

Would it be easier to read?

Yes, I agree, that is more readable. I'll make the change. I'll also follow Yonghong Song's suggestion (offline) of merging this patch into the first patch to keep bisectability together.

Thanks for the feedback!

Kenny

#13 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu
Quoted 15 lines · Kenny Yu

Hi Alexei,

Quoted 11 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34

Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7 -0
+// New helper added
+static long (*bpf_access_process_vm)(
+       struct task_struct *tsk,
+       unsigned long addr,
+       void *buf,
+       int len,
+       unsigned int gup_flags) = (void *)186;

This shouldn't be needed.
Since patch 1 updates tools/include/uapi/linux/bpf.h it will be in bpf_helper_defs.h automatically.

Quoted 3 lines

I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?

Look at vmtest.sh under tools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.

Quoted 22 lines
Quoted 6 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37

Quoted 4 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3 -0
+
+// Copied from include/linux/mm.h
+#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */

Please use C style comments only.

I will fix.

Quoted 6 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68

Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5 -0
+       numread = bpf_access_process_vm(task,
+                                       (unsigned long)ptr,
+                                       (void *)&user_data,
+                                       sizeof(uint32_t),
+                                       FOLL_REMOTE);
Quoted 6 lines

We probably would need to hide flags like FOLL_REMOTE inside the helper otherwise prog might confuse the kernel. In this case I'm not even sure that FOLL_REMOTE is needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything. fast_gup is not necessary either.

Quoted 8 lines

Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:

bpf_access_process_vm(struct task_struct *tsk,
unsigned long addr,
void *buf,
int len);

keeping generic u64 flags makes sense for the future, so I'd keep it.

But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (in probe_read it's just an address, here it's tsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:

buf, len, tsk, addr, flags

?

Quoted 4 lines

Thanks for the feedback!

Kenny

author
#14 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

Hi Andrii,

Quoted 3 lines

Look at vmtest.sh under tools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.

Thanks, this helps!

Quoted 1 line

keeping generic u64 flags makes sense for the future, so I'd keep it.

That makes sense, I'll keep the flags in that case.

Quoted 11 lines

But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (in probe_read it's just an address, here it's tsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:

buf, len, tsk, addr, flags

?

Yeah, that looks better for consistency. Should I still keep the name as bpf_access_process_vm, or call it something else to be more consistent with the naming of the other bpf helpers? The benefit of the bpf_access_process_vm name is that it makes it obvious it is wrapping an existing function access_process_vm.

Thanks for the feedback!

Kenny

#15 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Andrii Nakryiko
Quoted 66 lines · Andrii Nakryiko
Quoted 15 lines · Kenny Yu

Hi Alexei,

Quoted 11 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34

Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7 -0
+// New helper added
+static long (*bpf_access_process_vm)(
+       struct task_struct *tsk,
+       unsigned long addr,
+       void *buf,
+       int len,
+       unsigned int gup_flags) = (void *)186;

This shouldn't be needed.
Since patch 1 updates tools/include/uapi/linux/bpf.h it will be in bpf_helper_defs.h automatically.

Quoted 3 lines

I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?

Look at vmtest.sh under tools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.

Quoted 22 lines
Quoted 6 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37

Quoted 4 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3 -0
+
+// Copied from include/linux/mm.h
+#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */

Please use C style comments only.

I will fix.

Quoted 6 lines

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68

Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5 -0
+       numread = bpf_access_process_vm(task,
+                                       (unsigned long)ptr,
+                                       (void *)&user_data,
+                                       sizeof(uint32_t),
+                                       FOLL_REMOTE);
Quoted 6 lines

We probably would need to hide flags like FOLL_REMOTE inside the helper otherwise prog might confuse the kernel. In this case I'm not even sure that FOLL_REMOTE is needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything. fast_gup is not necessary either.

Quoted 8 lines

Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:

bpf_access_process_vm(struct task_struct *tsk,
unsigned long addr,
void *buf,
int len);

keeping generic u64 flags makes sense for the future, so I'd keep it.

But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (in probe_read it's just an address, here it's tsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:

Quoted 4 lines

buf, len, tsk, addr, flags

?

I would personally find it more intuitive to have process information passed as either the first argument (like process_vm_readv does), or as "last", just before the flags (as extra information required w.r.t. to local versions, e.g. bpf_copy_from_user).

author
#16 lore raw

Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs

Quoted 10 lines
Quoted 10 lines

But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (in probe_read it's just an address, here it's tsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:

buf, len, tsk, addr, flags

Quoted 7 lines
Quoted 2 lines

?

I would personally find it more intuitive to have process information passed as either the first argument (like process_vm_readv does), or as "last", just before the flags (as extra information required w.r.t. to local versions, e.g. bpf_copy_from_user).

I think that makes sense. I'll combine both Andrii's and Gabriele's suggestions and keep the signature as close to the existing helpers
(e.g., bpf_probe_read_user) and add the additional arguments at the end. I'll proceed with this signature:

bpf_access_process_vm(void *dst,
u32 size,
const void *unsafe_ptr,
struct task_struct *tsk,
unsigned int flags);

Thanks for all the suggestions!

Kenny

author
#17 lore raw

v3 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 10 +++++
 kernel/bpf/bpf_iter.c                         | 20 ++++++---
 kernel/bpf/helpers.c                          | 19 ++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 10 +++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 43 +++++++++++++++++++
 9 files changed, 117 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
author
#18 lore raw

PATCH v3 1/3 bpf: Add bpf_access_process_vm() helper

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/bpf_iter.c          | 20 +++++++++++++++-----
 kernel/bpf/helpers.c           | 21 +++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 6 files changed, 61 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 6e947cd91152..d3d0ef8df148 100644
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
index b0383d371b9a..27c2a18d5941 100644
@@ -5018,6 +5018,16 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)
+ *	Description
+ *		Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5216,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
-	ret = bpf_prog_run(prog, ctx);
-	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock_trace();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock();
+	}
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
kernel/bpf/helpers.c +21 -0
index 01cfdf40c838..30f1067ca8dc 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,26 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+	   const void __user *, user_ptr, struct task_struct *, tsk,
+	   u32, flags)
+{
+	/* flags is not used yet */
+	return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+	.func		= bpf_access_process_vm,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_BTF_ID,
+	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg5_type	= ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_find_vma_proto;
 	case BPF_FUNC_trace_vprintk:
 		return bpf_get_trace_vprintk_proto();
+	case BPF_FUNC_access_process_vm:
+		return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
tools/include/uapi/linux/bpf.h +11 -0
index b0383d371b9a..27c2a18d5941 100644
@@ -5018,6 +5018,16 @@ union bpf_attr {
  *
  *	Return
  *		The number of arguments of the traced function.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)
+ *	Description
+ *		Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5206,6 +5216,7 @@ union bpf_attr {
 	FN(get_func_arg),		\
 	FN(get_func_ret),		\
 	FN(get_func_arg_cnt),		\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
author
#21 lore raw

PATCH v3 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fdb3536afa7d..9ec40835bce8 100644
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#22 lore raw

PATCH v3 3/3 selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 44 +++++++++++++++++++
 2 files changed, 60 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
index b84f859b1267..fcda0ecd8746 100644
@@ -138,6 +138,20 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (CHECK(!skel, "bpf_iter_task__open_and_load",
+		  "skeleton open_and_load failed\n"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +44 -0

53 lines not on this page · raw message

Signature · 1 line
2.30.2
#19 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu

kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 681

Quoted 8 lines · Kenny Yu
kernel/bpf/helpers.c +7 -0
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+          const void __user *, user_ptr, struct task_struct *, tsk,
+          u32, flags)
+{
+       /* flags is not used yet */
+       return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}

Though the 'flags' argument is unused the helper has to check that it's zero and return -EINVAL otherwise.
If we don't do this we won't be able to change the behavior later due to backward compatibility.

author
#20 lore raw

Re: [PATCH v3 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 4 lines
Quoted 4 lines
kernel/bpf/helpers.c +4 -0
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+          const void __user *, user_ptr, struct task_struct *, tsk,
+          u32, flags)
+{
Quoted 8 lines

kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 681

Quoted 3 lines
kernel/bpf/helpers.c +3 -0
+       /* flags is not used yet */
+       return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}

Though the 'flags' argument is unused the helper has to check that it's zero and return -EINVAL otherwise.
If we don't do this we won't be able to change the behavior later due to backward compatibility.

Thanks for the feedback! Will fix.

Kenny

author
#24 lore raw

PATCH v4 1/3 bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/bpf_iter.c          | 20 +++++++++++++++-----
 kernel/bpf/helpers.c           | 23 +++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index dce54eb0aae8..29f174c08126 100644
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
index fe2272defcd9..38a85e6756b2 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)
+ *	Description
+ *		Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5249,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
-	ret = bpf_prog_run(prog, ctx);
-	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock_trace();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock();
+	}
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
kernel/bpf/helpers.c +23 -0
index 01cfdf40c838..6cd76280fe66 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+	   const void __user *, user_ptr, struct task_struct *, tsk,
+	   u32, flags)
+{
+	/* flags is not used yet */
+	if (flags != 0)
+		return -EINVAL;
+	return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+	.func		= bpf_access_process_vm,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_BTF_ID,
+	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg5_type	= ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_find_vma_proto;
 	case BPF_FUNC_trace_vprintk:
 		return bpf_get_trace_vprintk_proto();
+	case BPF_FUNC_access_process_vm:
+		return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
tools/include/uapi/linux/bpf.h +11 -0
index fe2272defcd9..38a85e6756b2 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)
+ *	Description
+ *		Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5249,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
author
#23 lore raw

v4 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.

Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 11 +++++
 kernel/bpf/bpf_iter.c                         | 20 ++++++---
 kernel/bpf/helpers.c                          | 23 ++++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 11 +++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 44 +++++++++++++++++++
 9 files changed, 124 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
author
#27 lore raw

PATCH v4 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fdb3536afa7d..9ec40835bce8 100644
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#28 lore raw

PATCH v4 3/3 selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 44 +++++++++++++++++++
 2 files changed, 60 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
index b84f859b1267..fcda0ecd8746 100644
@@ -138,6 +138,20 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (CHECK(!skel, "bpf_iter_task__open_and_load",
+		  "skeleton open_and_load failed\n"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +44 -0

53 lines not on this page · raw message

Signature · 1 line
2.30.2
#25 lore raw

Re: [PATCH v4 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu
Quoted 40 lines · Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
  include/linux/bpf.h            |  1 +
  include/uapi/linux/bpf.h       | 11 +++++++++++
  kernel/bpf/bpf_iter.c          | 20 +++++++++++++++-----
  kernel/bpf/helpers.c           | 23 +++++++++++++++++++++++
  kernel/trace/bpf_trace.c       |  2 ++
  tools/include/uapi/linux/bpf.h | 11 +++++++++++
  6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index dce54eb0aae8..29f174c08126 100644
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
  extern const struct bpf_func_proto bpf_find_vma_proto;
  extern const struct bpf_func_proto bpf_loop_proto;
  extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
  const struct bpf_func_proto *tracing_prog_func_proto(
    enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..38a85e6756b2 100644

include/uapi/linux/bpf.h union bpf_attr { · line 5053

Quoted 5 lines
include/uapi/linux/bpf.h +2 -0
@@ -5049,6 +5049,16 @@ union bpf_attr {
   *		This helper is currently supported by cgroup programs only.
   *	Return
   *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)

Maybe we can change 'flags' type to u64? This will leave more room for future potential extensions. In all recent helpers with added 'flags', most of them are u64.

Quoted 18 lines
include/uapi/linux/bpf.h +9 -0
+ *	Description
+ *		Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
   */
  #define __BPF_FUNC_MAPPER(FN)		\
  	FN(unspec),			\
@@ -5239,6 +5249,7 @@ union bpf_attr {
  	FN(get_func_arg_cnt),		\
  	FN(get_retval),			\
  	FN(set_retval),			\
+	FN(access_process_vm),		\
  	/* */
 

[...]

author
#26 lore raw

Re: [PATCH v4 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 5 lines
Quoted 1 line

+ * long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)

Maybe we can change 'flags' type to u64? This will leave more room for future potential extensions. In all recent helpers with added 'flags', most of them are u64.

I'll change it to u64. Thanks for the suggestion!

Kenny

author
#29 lore raw

v5 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.

Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.

Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 11 +++++
 kernel/bpf/bpf_iter.c                         | 20 ++++++---
 kernel/bpf/helpers.c                          | 23 ++++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 11 +++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 16 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 44 +++++++++++++++++++
 9 files changed, 124 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
author
#38 lore raw

PATCH v5 3/3 selftests/bpf: Add test for sleepable bpf iterator programs

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 19 ++++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 47 +++++++++++++++++++
 2 files changed, 66 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +19 -0
index b84f859b1267..f6fb4f95513d 100644
@@ -138,6 +138,23 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (CHECK(!skel, "bpf_iter_task__open_and_load",
+		  "skeleton open_and_load failed\n"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	ASSERT_GT(skel->bss->num_success_access_process_vm, 0,
+		  "num_success_access_process_vm");
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1269,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +47 -0

56 lines not on this page · raw message

Signature · 1 line
2.30.2
author
#37 lore raw

PATCH v5 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fdb3536afa7d..9ec40835bce8 100644
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#30 lore raw

PATCH v5 1/3 bpf: Add bpf_access_process_vm() helper

In reply to Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/bpf_iter.c          | 20 +++++++++++++++-----
 kernel/bpf/helpers.c           | 23 +++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index dce54eb0aae8..29f174c08126 100644
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
index fe2272defcd9..2ac56e2512df 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5249,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
-	ret = bpf_prog_run(prog, ctx);
-	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock_trace();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock();
+	}
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
kernel/bpf/helpers.c +23 -0
index 01cfdf40c838..9d7e86edc48e 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+	   const void __user *, user_ptr, struct task_struct *, tsk,
+	   u64, flags)
+{
+	/* flags is not used yet */
+	if (flags)
+		return -EINVAL;
+	return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+	.func		= bpf_access_process_vm,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_BTF_ID,
+	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg5_type	= ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_find_vma_proto;
 	case BPF_FUNC_trace_vprintk:
 		return bpf_get_trace_vprintk_proto();
+	case BPF_FUNC_access_process_vm:
+		return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
tools/include/uapi/linux/bpf.h +11 -0
index fe2272defcd9..2ac56e2512df 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This is a
+ *		wrapper of **access_process_vm**\ ().
+ *	Return
+ *		The number of bytes written to the buffer, or a negative error
+ *		in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5249,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(access_process_vm),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
#31 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 165 lines · Kenny Yu

This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/bpf_iter.c          | 20 +++++++++++++++-----
 kernel/bpf/helpers.c           | 23 +++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index dce54eb0aae8..29f174c08126 100644
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_access_process_vm_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
index fe2272defcd9..2ac56e2512df 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *             This helper is currently supported by cgroup programs only.
  *     Return
  *             0 on success, or a negative error in case of failure.
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This is a
+ *             wrapper of **access_process_vm**\ ().
+ *     Return
+ *             The number of bytes written to the buffer, or a negative error
+ *             in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)          \
        FN(unspec),                     \
@@ -5239,6 +5249,7 @@ union bpf_attr {
        FN(get_func_arg_cnt),           \
        FN(get_retval),                 \
        FN(set_retval),                 \
+       FN(access_process_vm),          \
        /* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
        struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
        int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 
        /* bpf program can only return 0 or 1:
         *  0 : okay
kernel/bpf/helpers.c +23 -0
index 01cfdf40c838..9d7e86edc48e 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
        .arg3_type      = ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size,
+          const void __user *, user_ptr, struct task_struct *, tsk,
+          u64, flags)
+{
+       /* flags is not used yet */
+       if (flags)
+               return -EINVAL;
+       return access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+}
+
+const struct bpf_func_proto bpf_access_process_vm_proto = {
+       .func           = bpf_access_process_vm,
+       .gpl_only       = false,
+       .ret_type       = RET_INTEGER,
+       .arg1_type      = ARG_PTR_TO_UNINIT_MEM,
+       .arg2_type      = ARG_CONST_SIZE_OR_ZERO,
+       .arg3_type      = ARG_ANYTHING,
+       .arg4_type      = ARG_PTR_TO_BTF_ID,
+       .arg4_btf_id    = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+       .arg5_type      = ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
        if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..1a6a81ce2e36 100644
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                return &bpf_find_vma_proto;
        case BPF_FUNC_trace_vprintk:
                return bpf_get_trace_vprintk_proto();
+       case BPF_FUNC_access_process_vm:
+               return prog->aux->sleepable ? &bpf_access_process_vm_proto : NULL;
        default:
                return bpf_base_func_proto(func_id);
        }
tools/include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..2ac56e2512df 100644
@@ -5049,6 +5049,16 @@ union bpf_attr {
  *             This helper is currently supported by cgroup programs only.
  *     Return
  *             0 on success, or a negative error in case of failure.

tools/include/uapi/linux/bpf.h union bpf_attr { · line 5061

Quoted 10 lines
tools/include/uapi/linux/bpf.h +10 -0
+ *
+ * long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This is a
+ *             wrapper of **access_process_vm**\ ().
+ *     Return
+ *             The number of bytes written to the buffer, or a negative error
+ *             in case of failure.

wait, can it read less than *size* and return success?

bpf_probe_read_kernel() returns:

0 on success, or a negative error in case of failure.

Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).

BTW, should we also add a C string reading helper as well, just like there is bpf_probe_read_user_str() and bpf_probe_read_user()?

Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.

And not to start the bikeshedding session, but we have bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.

Quoted 14 lines
tools/include/uapi/linux/bpf.h +1 -1
  */
 #define __BPF_FUNC_MAPPER(FN)          \
        FN(unspec),                     \
@@ -5239,6 +5249,7 @@ union bpf_attr {
        FN(get_func_arg_cnt),           \
        FN(get_retval),                 \
        FN(set_retval),                 \
+       FN(access_process_vm),          \
        /* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
--

2.30.2

#39 lore raw

Re: [PATCH v5 bpf-next 3/3] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu
Quoted 15 lines · Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 19 ++++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 47 +++++++++++++++++++
 2 files changed, 66 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +0 -0
index b84f859b1267..f6fb4f95513d 100644

tools/testing/selftests/bpf/prog_tests/bpf_iter.c static void test_task(void) · line 147

Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +7 -0
@@ -138,6 +138,23 @@ static void test_task(void)
        bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+       struct bpf_iter_task *skel;
+
+       skel = bpf_iter_task__open_and_load();
+       if (CHECK(!skel, "bpf_iter_task__open_and_load",
+                 "skeleton open_and_load failed\n"))

Please use ASSERT_OK_PTR() instead.

Quoted 62 lines
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +12 -0
+               return;
+
+       do_dummy_read(skel->progs.dump_task_sleepable);
+
+       ASSERT_GT(skel->bss->num_success_access_process_vm, 0,
+                 "num_success_access_process_vm");
+
+       bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
        struct bpf_iter_task_stack *skel;
@@ -1252,6 +1269,8 @@ void test_bpf_iter(void)
                test_bpf_map();
        if (test__start_subtest("task"))
                test_task();
+       if (test__start_subtest("task_sleepable"))
+               test_task_sleepable();
        if (test__start_subtest("task_stack"))
                test_task_stack();
        if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +25 -0
index c86b93f33b32..3fa735af96f7 100644
@@ -2,6 +2,7 @@
 /* Copyright (c) 2020 Facebook */
 #include "bpf_iter.h"
 #include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
 
 char _license[] SEC("license") = "GPL";
 
@@ -23,3 +24,49 @@ int dump_task(struct bpf_iter__task *ctx)
        BPF_SEQ_PRINTF(seq, "%8d %8d\n", task->tgid, task->pid);
        return 0;
 }
+
+int num_success_access_process_vm = 0;
+
+SEC("iter.s/task")
+int dump_task_sleepable(struct bpf_iter__task *ctx)
+{
+       struct seq_file *seq = ctx->meta->seq;
+       struct task_struct *task = ctx->task;
+       static const char info[] = "    === END ===";
+       struct pt_regs *regs;
+       void *ptr;
+       uint32_t user_data = 0;
+       int numread;
+
+       if (task == (void *)0) {
+               BPF_SEQ_PRINTF(seq, "%s\n", info);
+               return 0;
+       }
+
+       regs = (struct pt_regs *)bpf_task_pt_regs(task);
+       if (regs == (void *)0) {
+               BPF_SEQ_PRINTF(seq, "%s\n", info);
+               return 0;
+       }

tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 60

Quoted 10 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +10 -0
+       ptr = (void *)PT_REGS_IP(regs);
+
+       /* Try to read the contents of the task's instruction pointer from the
+        * remote task's address space.
+        */
+       numread = bpf_access_process_vm(&user_data,
+                                       sizeof(uint32_t),
+                                       ptr,
+                                       task,
+                                       0);

nit: keep it on one line (up to 100 characters is ok)

Quoted 15 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +12 -1
+       if (numread != sizeof(uint32_t)) {
+               BPF_SEQ_PRINTF(seq, "%s\n", info);
+               return 0;
+       }
+       ++num_success_access_process_vm;
+
+       if (ctx->meta->seq_num == 0)
+               BPF_SEQ_PRINTF(seq, "    tgid      gid     data\n");
+
+       BPF_SEQ_PRINTF(seq, "%8d %8d %8d\n", task->tgid, task->pid, user_data);
+       return 0;
+}
--

2.30.2

#32 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

In reply to Andrii Nakryiko

On Thu, Jan 20, 2022 at 2:46 PM Andrii Nakryiko
<[email protected]> wrote:
a

Quoted 9 lines
Quoted 4 lines

+ * wrapper of **access_process_vm**\ ().
+ * Return
+ * The number of bytes written to the buffer, or a negative error
+ * in case of failure.

wait, can it read less than *size* and return success?

bpf_probe_read_kernel() returns:

Quoted 8 lines

0 on success, or a negative error in case of failure.

Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).

BTW, should we also add a C string reading helper as well, just like there is bpf_probe_read_user_str() and bpf_probe_read_user()?

That would be difficult. There is no suitable kernel api for that.

Quoted 9 lines

Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.

And not to start the bikeshedding session, but we have bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.

How about bpf_copy_from_user_task() ?
The task is the second to last argument, so the name fits ? Especially if we call it this way it would be best to align return codes with bpf_copy_from_user.
Adding memset() in case of failure is mandatory too. I've missed this bit earlier.

The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we should memset() the whole buffer and return -EFAULT or memset only the leftover bytes and return 0? I think the former is best to align with bpf_copy_from_user.

#33 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 15 lines · Alexei Starovoitov

On Thu, Jan 20, 2022 at 2:46 PM Andrii Nakryiko
<[email protected]> wrote:
a

Quoted 11 lines
Quoted 4 lines

+ * wrapper of **access_process_vm**\ ().
+ * Return
+ * The number of bytes written to the buffer, or a negative error
+ * in case of failure.

wait, can it read less than *size* and return success?

bpf_probe_read_kernel() returns:

0 on success, or a negative error in case of failure.

Quoted 8 lines
Quoted 6 lines

Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).

BTW, should we also add a C string reading helper as well, just like there is bpf_probe_read_user_str() and bpf_probe_read_user()?

That would be difficult. There is no suitable kernel api for that.

Ok, but maybe we can add it later. Otherwise it will be hard to profiler Python processes and such, because you most certainly will need to read zero-terminated strings there.

Quoted 11 lines
Quoted 10 lines

Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.

And not to start the bikeshedding session, but we have bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.

Quoted 2 lines

How about bpf_copy_from_user_task() ?
The task is the second to last argument, so the name fits ?

yeah, I like the name

Quoted 4 lines

Especially if we call it this way it would be best to align return codes with bpf_copy_from_user.
Adding memset() in case of failure is mandatory too. I've missed this bit earlier.

Yep, good catch! Seems like copy_from_user() currently returns amount of bytes *not* read and memsets those unread bytes to zero. So for efficiency we could probably memset only those that were read.

Quoted 6 lines

The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we should memset() the whole buffer and return -EFAULT or memset only the leftover bytes and return 0? I think the former is best to align with bpf_copy_from_user.

Yeah, I think all or nothing approach (either complete success and zero return, or memset and error return) is best and most in line with other similar helpers.

author
#34 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 4 lines
Quoted 2 lines

How about bpf_copy_from_user_task() ?
The task is the second to last argument, so the name fits ?

yeah, I like the name

I'll change the name to bpf_copy_from_user_task.

Quoted 15 lines
Quoted 4 lines

Especially if we call it this way it would be best to align return codes with bpf_copy_from_user.
Adding memset() in case of failure is mandatory too. I've missed this bit earlier.

Yep, good catch! Seems like copy_from_user() currently returns amount of bytes *not* read and memsets those unread bytes to zero. So for efficiency we could probably memset only those that were read.

Quoted 5 lines

The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we should memset() the whole buffer and return -EFAULT or memset only the leftover bytes and return 0? I think the former is best to align with bpf_copy_from_user.

Quoted 3 lines

Yeah, I think all or nothing approach (either complete success and zero return, or memset and error return) is best and most in line with other similar helpers.

Thanks for the suggestions! I'll go with the all-or-nothing approach to be consistent with bpf_copy_from_user and will make the following changes:

* Return value: returns 0 on success, or negative error on failure.
* If we had a partial read, we will memset the read bytes to 0 and return -EFAULT

Quoted 2 lines

Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.

Will add that to the docs.

Quoted 5 lines
Quoted 1 line

That would be difficult. There is no suitable kernel api for that.

Ok, but maybe we can add it later. Otherwise it will be hard to profiler Python processes and such, because you most certainly will need to read zero-terminated strings there.

I will NOT add a C string helper in this patch series, and I'll explore how to add this in the future once this patch series is merged.

Quoted 5 lines

Review

Quoted 3 lines
(quoted diff) +3 -0
+       skel = bpf_iter_task__open_and_load();
+       if (CHECK(!skel, "bpf_iter_task__open_and_load",
+                 "skeleton open_and_load failed\n"))

Please use ASSERT_OK_PTR() instead.

Will fix.

Quoted 7 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+       numread = bpf_access_process_vm(&user_data,
+                                       sizeof(uint32_t),
+                                       ptr,
+                                       task,
+                                       0);

nit: keep it on one line (up to 100 characters is ok)

Will fix.

Thanks for the suggestions everyone!

Kenny

#35 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 7 lines · Kenny Yu

Thanks for the suggestions! I'll go with the all-or-nothing approach to be consistent with bpf_copy_from_user and will make the following changes:

* Return value: returns 0 on success, or negative error on failure.
* If we had a partial read, we will memset the read bytes to 0 and return -EFAULT

Not read bytes, but all bytes.
copy_from_user zeros leftover for us, but access_process_vm doesn't do this on partial read.

author
#36 lore raw

Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper

Quoted 6 lines
Quoted 2 lines

* If we had a partial read, we will memset the read bytes to 0 and return -EFAULT

Not read bytes, but all bytes.
copy_from_user zeros leftover for us, but access_process_vm doesn't do this on partial read.

I'll memset all bytes on error. Thanks for the suggestion!

Kenny

author
#40 lore raw

v6 cover Add bpf_copy_from_user_task helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_copy_from_user_task to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v5:
* Rename bpf_access_process_vm to bpf_copy_from_user_task.
* Change return value to be all-or-nothing. If we get a partial read, memset all bytes to 0 and return -EFAULT.
* Add to docs that the helper can only be used by sleepable BPF programs.
* Fix nits in selftests.

Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.

Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.

Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (3):
bpf: Add bpf_copy_from_user_task() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 10 +++++
 kernel/bpf/bpf_iter.c                         | 20 ++++++---
 kernel/bpf/helpers.c                          | 31 +++++++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 10 +++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 18 ++++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 43 +++++++++++++++++++
 9 files changed, 131 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
author
#49 lore raw

PATCH v6 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index fc6d530e20db..ea7149606e67 100644
@@ -8607,6 +8607,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
 	SEC_DEF("xdp_cpumap/",		XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#41 lore raw

PATCH v6 1/3 bpf: Add bpf_copy_from_user_task() helper

In reply to Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 10 ++++++++++
 kernel/bpf/bpf_iter.c          | 20 ++++++++++++++-----
 kernel/bpf/helpers.c           | 35 ++++++++++++++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 10 ++++++++++
 6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 80e3387ea3af..5917883e528b 100644
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
index fe2272defcd9..d82d9423874d 100644
@@ -5049,6 +5049,15 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This helper
+ *		can only be used by sleepable programs.
+ *	Return
+ *		0 on success, or a negative error in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5248,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(copy_from_user_task),	\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
-	ret = bpf_prog_run(prog, ctx);
-	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock_trace();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock();
+	}
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
kernel/bpf/helpers.c +35 -0
index 01cfdf40c838..3bc37016fad0 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,40 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
+	   const void __user *, user_ptr, struct task_struct *, tsk, u64, flags)
+{
+	int ret;
+
+	/* flags is not used yet */
+	if (flags)
+		return -EINVAL;
+
+	ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+	if (ret >= 0) {
+		if (ret == size)
+			ret = 0;
+		else {
+			/* If partial read, clear all bytes and return error */
+			memset(dst, 0, size);
+			ret = -EFAULT;
+		}
+	}
+	return ret;
+}
+
+const struct bpf_func_proto bpf_copy_from_user_task_proto = {
+	.func		= bpf_copy_from_user_task,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_BTF_ID,
+	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg5_type	= ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 21aa30644219..d07f568646cb 100644
@@ -1235,6 +1235,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_get_task_stack_proto;
 	case BPF_FUNC_copy_from_user:
 		return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
+	case BPF_FUNC_copy_from_user_task:
+		return prog->aux->sleepable ? &bpf_copy_from_user_task_proto : NULL;
 	case BPF_FUNC_snprintf_btf:
 		return &bpf_snprintf_btf_proto;
 	case BPF_FUNC_per_cpu_ptr:
tools/include/uapi/linux/bpf.h +10 -0
index fe2272defcd9..d82d9423874d 100644
@@ -5049,6 +5049,15 @@ union bpf_attr {
  *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This helper
+ *		can only be used by sleepable programs.
+ *	Return
+ *		0 on success, or a negative error in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5239,6 +5248,7 @@ union bpf_attr {
 	FN(get_func_arg_cnt),		\
 	FN(get_retval),			\
 	FN(set_retval),			\
+	FN(copy_from_user_task),	\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
author
#51 lore raw

PATCH v6 3/3 selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 20 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 74 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +20 -0
index b84f859b1267..5142a7d130b2 100644
@@ -138,6 +138,24 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
+		  "num_expected_failure_copy_from_user_task");
+	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
+		  "num_success_copy_from_user_task");
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1270,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0

63 lines not on this page · raw message

Signature · 1 line
2.30.2
#42 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

In reply to Kenny Yu
Quoted 41 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 10 ++++++++++
 kernel/bpf/bpf_iter.c          | 20 ++++++++++++++-----
 kernel/bpf/helpers.c           | 35 ++++++++++++++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 10 ++++++++++
 6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 80e3387ea3af..5917883e528b 100644
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..d82d9423874d 100644

include/uapi/linux/bpf.h union bpf_attr { · line 5058

Quoted 10 lines
include/uapi/linux/bpf.h +7 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {
  *             This helper is currently supported by cgroup programs only.
  *     Return
  *             0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This helper
+ *             can only be used by sleepable programs.

"On error dst buffer is zeroed out."? This is an explicit guarantee.

Quoted 39 lines
include/uapi/linux/bpf.h +3 -0
+ *     Return
+ *             0 on success, or a negative error in case of failure.
  */
 #define __BPF_FUNC_MAPPER(FN)          \
        FN(unspec),                     \
@@ -5239,6 +5248,7 @@ union bpf_attr {
        FN(get_func_arg_cnt),           \
        FN(get_retval),                 \
        FN(set_retval),                 \
+       FN(copy_from_user_task),        \
        /* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +6 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
        struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
        int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);

kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702

Quoted 10 lines
kernel/bpf/bpf_iter.c +9 -0
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 

I think this sleepable bpf_iter change deserves its own patch. It has nothing to do with bpf_copy_from_user_task() helper.

Quoted 21 lines
kernel/bpf/bpf_iter.c +0 -0
        /* bpf program can only return 0 or 1:
         *  0 : okay
kernel/bpf/helpers.c +4 -0
index 01cfdf40c838..3bc37016fad0 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,40 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
        .arg3_type      = ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
+          const void __user *, user_ptr, struct task_struct *, tsk, u64, flags)
+{

kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 687

Quoted 10 lines
kernel/bpf/helpers.c +10 -0
+       int ret;
+
+       /* flags is not used yet */
+       if (flags)
+               return -EINVAL;
+
+       ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+       if (ret >= 0) {
+               if (ret == size)
+                       ret = 0;

is there a point in calling access_process_vm() with size == 0? It would validate that get_task_mm() succeeds, but that's pretty much it? So maybe instead just exit early if size is zero? It will be also less convoluted logic:

if (size == 0)

    return 0;
if (access_process_vm(...)) {
    memset(0);
    return -EFAULT;
}

return 0;

Quoted 9 lines
kernel/bpf/helpers.c +9 -0
+               else {
+                       /* If partial read, clear all bytes and return error */
+                       memset(dst, 0, size);
+                       ret = -EFAULT;
+               }
+       }
+       return ret;
+}
+

[...]

Acked-by on v6 2/3
#50 lore raw

Re: [PATCH v6 bpf-next 2/3] libbpf: Add "iter.s" section for sleepable bpf iterator programs

In reply to Kenny Yu
Quoted 6 lines · Kenny Yu

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---

Quoted 18 lines
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -1
index fc6d530e20db..ea7149606e67 100644
@@ -8607,6 +8607,7 @@ static const struct bpf_sec_def section_defs[] = {
        SEC_DEF("lsm/",                 LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
        SEC_DEF("lsm.s/",               LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
        SEC_DEF("iter/",                TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+       SEC_DEF("iter.s/",              TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
        SEC_DEF("syscall",              SYSCALL, 0, SEC_SLEEPABLE),
        SEC_DEF("xdp_devmap/",          XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
        SEC_DEF("xdp_cpumap/",          XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
--

2.30.2

Acked-by on v6 3/3
#52 lore raw

Re: [PATCH v6 bpf-next 3/3] selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu
Quoted 6 lines · Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---

Quoted 4 lines
 .../selftests/bpf/prog_tests/bpf_iter.c       | 20 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 74 insertions(+)

[...]

#43 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

In reply to Andrii Nakryiko
Quoted 99 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
  include/linux/bpf.h            |  1 +
  include/uapi/linux/bpf.h       | 10 ++++++++++
  kernel/bpf/bpf_iter.c          | 20 ++++++++++++++-----
  kernel/bpf/helpers.c           | 35 ++++++++++++++++++++++++++++++++++
  kernel/trace/bpf_trace.c       |  2 ++
  tools/include/uapi/linux/bpf.h | 10 ++++++++++
  6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 80e3387ea3af..5917883e528b 100644
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
  extern const struct bpf_func_proto bpf_find_vma_proto;
  extern const struct bpf_func_proto bpf_loop_proto;
  extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
  const struct bpf_func_proto *tracing_prog_func_proto(
    enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..d82d9423874d 100644

include/uapi/linux/bpf.h union bpf_attr { · line 5058

Quoted 10 lines
include/uapi/linux/bpf.h +7 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {
   *             This helper is currently supported by cgroup programs only.
   *     Return
   *             0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This helper
+ *             can only be used by sleepable programs.

"On error dst buffer is zeroed out."? This is an explicit guarantee.

Quoted 44 lines
include/uapi/linux/bpf.h +3 -0
+ *     Return
+ *             0 on success, or a negative error in case of failure.
   */
  #define __BPF_FUNC_MAPPER(FN)          \
         FN(unspec),                     \
@@ -5239,6 +5248,7 @@ union bpf_attr {
         FN(get_func_arg_cnt),           \
         FN(get_retval),                 \
         FN(set_retval),                 \
+       FN(copy_from_user_task),        \
         /* */
 
  /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +11 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
  #include <linux/anon_inodes.h>
  #include <linux/filter.h>
  #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
  struct bpf_iter_target_info {
         struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
  {
         int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
Quoted 8 lines

kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702

Quoted 5 lines
kernel/bpf/bpf_iter.c +4 -0
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 

I think this sleepable bpf_iter change deserves its own patch. It has nothing to do with bpf_copy_from_user_task() helper.

Without the above change, using bpf_copy_from_user_task() will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducing bpf_copy_from_user_task() so we won't have bisecting issues.

Quoted 3 lines
Quoted 3 lines
kernel/bpf/bpf_iter.c +0 -0
         /* bpf program can only return 0 or 1:
          *  0 : okay

[...]

#44 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

Quoted 112 lines · Yonghong Song
Quoted 99 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
  include/linux/bpf.h            |  1 +
  include/uapi/linux/bpf.h       | 10 ++++++++++
  kernel/bpf/bpf_iter.c          | 20 ++++++++++++++-----
  kernel/bpf/helpers.c           | 35 ++++++++++++++++++++++++++++++++++
  kernel/trace/bpf_trace.c       |  2 ++
  tools/include/uapi/linux/bpf.h | 10 ++++++++++
  6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 80e3387ea3af..5917883e528b 100644
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
  extern const struct bpf_func_proto bpf_find_vma_proto;
  extern const struct bpf_func_proto bpf_loop_proto;
  extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
  const struct bpf_func_proto *tracing_prog_func_proto(
    enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..d82d9423874d 100644

include/uapi/linux/bpf.h union bpf_attr { · line 5058

Quoted 10 lines
include/uapi/linux/bpf.h +7 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {
   *             This helper is currently supported by cgroup programs only.
   *     Return
   *             0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This helper
+ *             can only be used by sleepable programs.

"On error dst buffer is zeroed out."? This is an explicit guarantee.

Quoted 44 lines
include/uapi/linux/bpf.h +3 -0
+ *     Return
+ *             0 on success, or a negative error in case of failure.
   */
  #define __BPF_FUNC_MAPPER(FN)          \
         FN(unspec),                     \
@@ -5239,6 +5248,7 @@ union bpf_attr {
         FN(get_func_arg_cnt),           \
         FN(get_retval),                 \
         FN(set_retval),                 \
+       FN(copy_from_user_task),        \
         /* */
 
  /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +11 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
  #include <linux/anon_inodes.h>
  #include <linux/filter.h>
  #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
  struct bpf_iter_target_info {
         struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
  {
         int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
Quoted 8 lines

kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702

Quoted 5 lines
kernel/bpf/bpf_iter.c +4 -0
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 

I think this sleepable bpf_iter change deserves its own patch. It has nothing to do with bpf_copy_from_user_task() helper.

Quoted 4 lines

Without the above change, using bpf_copy_from_user_task() will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducing bpf_copy_from_user_task() so we won't have bisecting issues.

Sure, patch #1 for sleepable bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.

Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
kernel/bpf/bpf_iter.c +0 -0
         /* bpf program can only return 0 or 1:
          *  0 : okay

[...]

#45 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

Quoted 113 lines · Andrii Nakryiko
Quoted 112 lines · Yonghong Song
Quoted 107 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.

This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
   include/linux/bpf.h            |  1 +
   include/uapi/linux/bpf.h       | 10 ++++++++++
   kernel/bpf/bpf_iter.c          | 20 ++++++++++++++-----
   kernel/bpf/helpers.c           | 35 ++++++++++++++++++++++++++++++++++
   kernel/trace/bpf_trace.c       |  2 ++
   tools/include/uapi/linux/bpf.h | 10 ++++++++++
   6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
index 80e3387ea3af..5917883e528b 100644
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
   extern const struct bpf_func_proto bpf_find_vma_proto;
   extern const struct bpf_func_proto bpf_loop_proto;
   extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
   const struct bpf_func_proto *tracing_prog_func_proto(
     enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +0 -0
index fe2272defcd9..d82d9423874d 100644

include/uapi/linux/bpf.h union bpf_attr { · line 5058

Quoted 10 lines
include/uapi/linux/bpf.h +7 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {
    *             This helper is currently supported by cgroup programs only.
    *     Return
    *             0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *     Description
+ *             Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *             address space, and stores the data in *dst*. *flags* is not
+ *             used yet and is provided for future extensibility. This helper
+ *             can only be used by sleepable programs.

"On error dst buffer is zeroed out."? This is an explicit guarantee.

Quoted 39 lines
include/uapi/linux/bpf.h +3 -0
+ *     Return
+ *             0 on success, or a negative error in case of failure.
    */
   #define __BPF_FUNC_MAPPER(FN)          \
          FN(unspec),                     \
@@ -5239,6 +5248,7 @@ union bpf_attr {
          FN(get_func_arg_cnt),           \
          FN(get_retval),                 \
          FN(set_retval),                 \
+       FN(copy_from_user_task),        \
          /* */
 
   /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +6 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
   #include <linux/anon_inodes.h>
   #include <linux/filter.h>
   #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
   struct bpf_iter_target_info {
          struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
   {
          int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);

kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702

Quoted 10 lines
kernel/bpf/bpf_iter.c +9 -0
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 

I think this sleepable bpf_iter change deserves its own patch. It has nothing to do with bpf_copy_from_user_task() helper.

Quoted 8 lines
Quoted 4 lines

Without the above change, using bpf_copy_from_user_task() will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducing bpf_copy_from_user_task() so we won't have bisecting issues.

Sure, patch #1 for sleepable bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.

okay with me.

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
kernel/bpf/bpf_iter.c +0 -0
          /* bpf program can only return 0 or 1:
           *  0 : okay

[...]

#47 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

In reply to Andrii Nakryiko
Quoted 3 lines · Andrii Nakryiko

"On error dst buffer is zeroed out."? This is an explicit guarantee.

I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure? Can't this step be skipped for performance and leave it up to the callee to perform if they need it?

#48 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

s/callee/caller

Quoted 10 lines · Gabriele
Quoted 3 lines · Andrii Nakryiko

"On error dst buffer is zeroed out."? This is an explicit guarantee.

I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure? Can't this step be skipped for performance and leave it up to the callee to perform if they need it?

Signature · 6 lines
"Egli è scritto in lingua matematica, e i caratteri son triangoli,
cerchi, ed altre figure
geometriche, senza i quali mezzi è impossibile a intenderne umanamente parola;
senza questi è un aggirarsi vanamente per un oscuro laberinto."

-- G. Galilei, Il saggiatore.
author
#46 lore raw

Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper

In reply to Yonghong Song
Quoted 1 line

"On error dst buffer is zeroed out."? This is an explicit guarantee.

Will add to the docs.

Quoted 5 lines

is there a point in calling access_process_vm() with size == 0? It would validate that get_task_mm() succeeds, but that's pretty much it? So maybe instead just exit early if size is zero? It will be also less convoluted logic:

Quoted 7 lines

if (size == 0)

    return 0;
if (access_process_vm(...)) {
    memset(0);
    return -EFAULT;
}

return 0;

Will do an explicit check for size == 0.
Note that we still need to check if the return value of access_process_vm == size to see if we have a partial read.

Quoted 8 lines
Quoted 4 lines

Without the above change, using bpf_copy_from_user_task() will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducing bpf_copy_from_user_task() so we won't have bisecting issues.

Sure, patch #1 for sleepable bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.

I'll split this into 2 patches and place the bpf_iter patch first.

Quoted 3 lines

I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure?

I believe the intent behind this is for security/privacy reasons. On error, we don't want to unintentionally leak partially read data.

Thanks everyone for the suggestions!

Kenny

author
#53 lore raw

v7 cover Add bpf_copy_from_user_task helper and sleepable bpf iterator programs

In reply to Kenny Yu

This patch series makes the following changes:
* Adds a new bpf helper bpf_copy_from_user_task to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

Changes since v6:
* Split first patch into two patches: first patch to add support for bpf iterators to use sleepable helpers, and the second to add the new bpf helper.
* Simplify implementation of bpf_copy_from_user_task based on feedback.
* Add to docs that the destination buffer will be zero-ed on error.

Changes since v5:
* Rename bpf_access_process_vm to bpf_copy_from_user_task.
* Change return value to be all-or-nothing. If we get a partial read, memset all bytes to 0 and return -EFAULT.
* Add to docs that the helper can only be used by sleepable BPF programs.
* Fix nits in selftests.

Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.

Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.

Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.

Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.

Kenny Yu (4):
bpf: Add support for bpf iterator programs to use sleepable helpers
bpf: Add bpf_copy_from_user_task() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs

 include/linux/bpf.h                           |  1 +
 include/uapi/linux/bpf.h                      | 11 ++++
 kernel/bpf/bpf_iter.c                         | 20 +++++--
 kernel/bpf/helpers.c                          | 34 ++++++++++++
 kernel/trace/bpf_trace.c                      |  2 +
 tools/include/uapi/linux/bpf.h                | 11 ++++
 tools/lib/bpf/libbpf.c                        |  1 +
 .../selftests/bpf/prog_tests/bpf_iter.c       | 20 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 9 files changed, 149 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
author
#59 lore raw

PATCH v7 3/4 libbpf: Add "iter.s" section for sleepable bpf iterator programs

This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.

---
 tools/lib/bpf/libbpf.c | 1 +
 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
index a8c750373ad5..57894f125df3 100644
@@ -8612,6 +8612,7 @@ static const struct bpf_sec_def section_defs[] = {
 	SEC_DEF("lsm/",			LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
 	SEC_DEF("lsm.s/",		LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
 	SEC_DEF("iter/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
+	SEC_DEF("iter.s/",		TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
 	SEC_DEF("syscall",		SYSCALL, 0, SEC_SLEEPABLE),
 	SEC_DEF("xdp.frags/devmap",	XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
 	SEC_DEF("xdp_devmap/",		XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
author
#60 lore raw

PATCH v7 4/4 selftests/bpf: Add test for sleepable bpf iterator programs

In reply to Kenny Yu

This adds a test for bpf iterator programs to make use of sleepable bpf helpers.

---
 .../selftests/bpf/prog_tests/bpf_iter.c       | 20 +++++++
 .../selftests/bpf/progs/bpf_iter_task.c       | 54 +++++++++++++++++++
 2 files changed, 74 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +20 -0
index b84f859b1267..5142a7d130b2 100644
@@ -138,6 +138,24 @@ static void test_task(void)
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_sleepable(void)
+{
+	struct bpf_iter_task *skel;
+
+	skel = bpf_iter_task__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load"))
+		return;
+
+	do_dummy_read(skel->progs.dump_task_sleepable);
+
+	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
+		  "num_expected_failure_copy_from_user_task");
+	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
+		  "num_success_copy_from_user_task");
+
+	bpf_iter_task__destroy(skel);
+}
+
 static void test_task_stack(void)
 {
 	struct bpf_iter_task_stack *skel;
@@ -1252,6 +1270,8 @@ void test_bpf_iter(void)
 		test_bpf_map();
 	if (test__start_subtest("task"))
 		test_task();
+	if (test__start_subtest("task_sleepable"))
+		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
 		test_task_stack();
 	if (test__start_subtest("task_file"))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0

63 lines not on this page · raw message

Signature · 1 line
2.30.2
author
#56 lore raw

PATCH v7 2/4 bpf: Add bpf_copy_from_user_task() helper

In reply to Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/helpers.c           | 34 ++++++++++++++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 5 files changed, 59 insertions(+)
include/linux/bpf.h +1 -0
index e8ec8d2f2fe3..4f13642c6a47 100644
@@ -2238,6 +2238,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_strncmp_proto;
+extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
index 16a7574292a5..4a2f7041ebae 100644
@@ -5076,6 +5076,16 @@ union bpf_attr {
  *		associated to *xdp_md*, at *offset*.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This helper
+ *		can only be used by sleepable programs.
+ *	Return
+ *		0 on success, or a negative error in case of failure. On error
+ *		*dst* buffer is zeroed out.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5269,6 +5279,7 @@ union bpf_attr {
 	FN(xdp_get_buff_len),		\
 	FN(xdp_load_bytes),		\
 	FN(xdp_store_bytes),		\
+	FN(copy_from_user_task),	\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +34 -0
index 01cfdf40c838..2688b0064289 100644
@@ -16,6 +16,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
+#include <linux/btf_ids.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -671,6 +672,39 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
+	   const void __user *, user_ptr, struct task_struct *, tsk, u64, flags)
+{
+	int ret;
+
+	/* flags is not used yet */
+	if (flags)
+		return -EINVAL;
+
+	if (size == 0)
+		return 0;
+
+	ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+	if (ret == size)
+		return 0;
+
+	memset(dst, 0, size);
+	/* Return -EFAULT for partial read */
+	return (ret < 0) ? ret : -EFAULT;
+}
+
+const struct bpf_func_proto bpf_copy_from_user_task_proto = {
+	.func		= bpf_copy_from_user_task,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
+	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_BTF_ID,
+	.arg4_btf_id	= &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+	.arg5_type	= ARG_ANYTHING
+};
+
 BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
 {
 	if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
index 06a9e220069e..a2024ba32a20 100644
@@ -1235,6 +1235,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_get_task_stack_proto;
 	case BPF_FUNC_copy_from_user:
 		return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
+	case BPF_FUNC_copy_from_user_task:
+		return prog->aux->sleepable ? &bpf_copy_from_user_task_proto : NULL;
 	case BPF_FUNC_snprintf_btf:
 		return &bpf_snprintf_btf_proto;
 	case BPF_FUNC_per_cpu_ptr:
tools/include/uapi/linux/bpf.h +11 -0
index 16a7574292a5..4a2f7041ebae 100644
@@ -5076,6 +5076,16 @@ union bpf_attr {
  *		associated to *xdp_md*, at *offset*.
  *	Return
  *		0 on success, or a negative error in case of failure.
+ *
+ * long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags)
+ *	Description
+ *		Read *size* bytes from user space address *user_ptr* in *tsk*'s
+ *		address space, and stores the data in *dst*. *flags* is not
+ *		used yet and is provided for future extensibility. This helper
+ *		can only be used by sleepable programs.
+ *	Return
+ *		0 on success, or a negative error in case of failure. On error
+ *		*dst* buffer is zeroed out.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5269,6 +5279,7 @@ union bpf_attr {
 	FN(xdp_get_buff_len),		\
 	FN(xdp_load_bytes),		\
 	FN(xdp_store_bytes),		\
+	FN(copy_from_user_task),	\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
author
#54 lore raw

PATCH v7 1/4 bpf: Add support for bpf iterator programs to use sleepable helpers

In reply to Kenny Yu

This patch allows bpf iterator programs to use sleepable helpers by changing bpf_iter_run_prog to use the appropriate synchronization. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

---
 kernel/bpf/bpf_iter.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)
kernel/bpf/bpf_iter.c +15 -5
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
 	struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
 	int ret;
 
-	rcu_read_lock();
-	migrate_disable();
-	ret = bpf_prog_run(prog, ctx);
-	migrate_enable();
-	rcu_read_unlock();
+	if (prog->aux->sleepable) {
+		rcu_read_lock_trace();
+		migrate_disable();
+		might_fault();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock_trace();
+	} else {
+		rcu_read_lock();
+		migrate_disable();
+		ret = bpf_prog_run(prog, ctx);
+		migrate_enable();
+		rcu_read_unlock();
+	}
 
 	/* bpf program can only return 0 or 1:
 	 *  0 : okay
Signature · 1 line
2.30.2
Acked-by on v7 1/4
#55 lore raw

Re: [PATCH v7 bpf-next 1/4] bpf: Add support for bpf iterator programs to use sleepable helpers

Quoted 9 lines · Kenny Yu

This patch allows bpf iterator programs to use sleepable helpers by changing bpf_iter_run_prog to use the appropriate synchronization. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.

---

Quoted 44 lines
 kernel/bpf/bpf_iter.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)
kernel/bpf/bpf_iter.c +15 -6
index b7aef5b3416d..110029ede71e 100644
@@ -5,6 +5,7 @@
 #include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
 
 struct bpf_iter_target_info {
        struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
        int ret;
 
-       rcu_read_lock();
-       migrate_disable();
-       ret = bpf_prog_run(prog, ctx);
-       migrate_enable();
-       rcu_read_unlock();
+       if (prog->aux->sleepable) {
+               rcu_read_lock_trace();
+               migrate_disable();
+               might_fault();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock_trace();
+       } else {
+               rcu_read_lock();
+               migrate_disable();
+               ret = bpf_prog_run(prog, ctx);
+               migrate_enable();
+               rcu_read_unlock();
+       }
 
        /* bpf program can only return 0 or 1:
         *  0 : okay
--

2.30.2

Acked-by on v7 2/4
#57 lore raw

Re: [PATCH v7 bpf-next 2/4] bpf: Add bpf_copy_from_user_task() helper

In reply to Kenny Yu
Quoted 10 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---

LGTM.

Quoted 7 lines
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/helpers.c           | 34 ++++++++++++++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 5 files changed, 59 insertions(+)

[...]

kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 693

Quoted 7 lines
kernel/bpf/helpers.c +7 -0
+       ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+       if (ret == size)
+               return 0;
+
+       memset(dst, 0, size);
+       /* Return -EFAULT for partial read */
+       return (ret < 0) ? ret : -EFAULT;

nit: unnecessary ()

Quoted 14 lines
kernel/bpf/helpers.c +14 -0
+}
+
+const struct bpf_func_proto bpf_copy_from_user_task_proto = {
+       .func           = bpf_copy_from_user_task,
+       .gpl_only       = false,
+       .ret_type       = RET_INTEGER,
+       .arg1_type      = ARG_PTR_TO_UNINIT_MEM,
+       .arg2_type      = ARG_CONST_SIZE_OR_ZERO,
+       .arg3_type      = ARG_ANYTHING,
+       .arg4_type      = ARG_PTR_TO_BTF_ID,
+       .arg4_btf_id    = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
+       .arg5_type      = ARG_ANYTHING
+};
+

[...]

#58 lore raw

Re: [PATCH v7 bpf-next 2/4] bpf: Add bpf_copy_from_user_task() helper

Quoted 35 lines · Andrii Nakryiko
Quoted 10 lines · Kenny Yu

This adds a helper for bpf programs to read the memory of other tasks.

As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.

---

LGTM.

Quoted 7 lines
 include/linux/bpf.h            |  1 +
 include/uapi/linux/bpf.h       | 11 +++++++++++
 kernel/bpf/helpers.c           | 34 ++++++++++++++++++++++++++++++++++
 kernel/trace/bpf_trace.c       |  2 ++
 tools/include/uapi/linux/bpf.h | 11 +++++++++++
 5 files changed, 59 insertions(+)

[...]

kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 693

Quoted 7 lines
kernel/bpf/helpers.c +7 -0
+       ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
+       if (ret == size)
+               return 0;
+
+       memset(dst, 0, size);
+       /* Return -EFAULT for partial read */
+       return (ret < 0) ? ret : -EFAULT;
Quoted 1 line

nit: unnecessary ()

I fixed it while applying.
Also added a few unlikely().

Thanks everyone!

#61 lore raw

Re: [PATCH v7 bpf-next 0/4] Add bpf_copy_from_user_task helper and sleepable bpf iterator programs

In reply to Kenny Yu
This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:

Quoted 10 lines · On Mon, 24 Jan 2022 10:53:59 -0800 you wrote:

This patch series makes the following changes:
* Adds a new bpf helper bpf_copy_from_user_task to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.

As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.

[...]

Here is the summary with links:
- [v7,bpf-next,1/4] bpf: Add support for bpf iterator programs to use sleepable helpers

    https://git.kernel.org/bpf/bpf-next/c/b77fb25dcb34
  - [v7,bpf-next,2/4] bpf: Add bpf_copy_from_user_task() helper
    https://git.kernel.org/bpf/bpf-next/c/376040e47334
  - [v7,bpf-next,3/4] libbpf: Add "iter.s" section for sleepable bpf iterator programs
    https://git.kernel.org/bpf/bpf-next/c/a8b77f7463a5
  - [v7,bpf-next,4/4] selftests/bpf: Add test for sleepable bpf iterator programs
    https://git.kernel.org/bpf/bpf-next/c/45105c2eb751

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html