Add bpf_access_process_vm helper and sleepable bpf iterator programs
Subsystems (9)
cover Add bpf_access_process_vm helper and sleepable bpf iterator programs
This patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 +++++ kernel/bpf/helpers.c | 19 ++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 10 +++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 43 +++++++++++++++++++ 8 files changed, 102 insertions(+)
Signature · 1 line
2.30.2
PATCH 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs
This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH 1/3 bpf: Add bpf_access_process_vm() helper
In reply to Kenny YuThis adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/helpers.c | 19 +++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 5 files changed, 42 insertions(+)
include/linux/bpf.h +1 -0
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
@@ -5018,6 +5018,15 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) +* Description +* Read *len* bytes from user space address *addr* in *tsk*'s +* address space, and stores the data in *buf*. This is a wrapper +* of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5215,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +19 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,24 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, struct task_struct *, tsk, + unsigned long, addr, void *, buf, int, len, unsigned int, gup_flags) +{ +return access_process_vm (tsk, addr, buf, len, gup_flags); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_BTF_ID , + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg2_type =ARG_ANYTHING , + .arg3_type =ARG_PTR_TO_UNINIT_MEM , + .arg4_type =ARG_CONST_SIZE_OR_ZERO , + .arg5_type =ARG_ANYTHING , +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }
tools/include/uapi/linux/bpf.h +10 -0
@@ -5018,6 +5018,15 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) +* Description +* Read *len* bytes from user space address *addr* in *tsk*'s +* address space, and stores the data in *buf*. This is a wrapper +* of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5215,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
PATCH 3/3 selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuThis adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 16 ++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 70 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
@@ -138,6 +138,20 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0
63 lines not on this page · raw message
Signature · 1 line
2.30.2
v2 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (4):
bpf: Add bpf_access_process_vm() helper
bpf: Add support for sleepable programs in bpf_iter_run_prog
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++ kernel/bpf/bpf_iter.c | 16 ++++-- kernel/bpf/helpers.c | 19 +++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 10 ++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 16 ++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 9 files changed, 126 insertions(+), 3 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 4/4 selftests/bpf: Add test for sleepable bpf iterator programs
This adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 16 ++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 70 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
@@ -138,6 +138,20 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0
63 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v2 2/4 bpf: Add support for sleepable programs in bpf_iter_run_prog
In reply to Kenny YuThe next patch adds the ability to create sleepable bpf iterator programs. This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
--- kernel/bpf/bpf_iter.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)
kernel/bpf/bpf_iter.c +13 -3
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + }else { +rcu_read_lock (); +migrate_disable (); + } ret =bpf_prog_run (prog, ctx);migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) +rcu_read_unlock_trace (); +else +rcu_read_unlock ();/* bpf program can only return 0 or 1: * 0 : okay
Signature · 1 line
2.30.2
PATCH v2 3/4 libbpf: Add "iter.s" section for sleepable bpf iterator programs
In reply to Kenny YuThis adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v2 1/4 bpf: Add bpf_access_process_vm() helper
In reply to Kenny YuThis adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/helpers.c | 19 +++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 5 files changed, 42 insertions(+)
include/linux/bpf.h +1 -0
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
@@ -5018,6 +5018,15 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) +* Description +* Read *len* bytes from user space address *addr* in *tsk*'s +* address space, and stores the data in *buf*. This is a wrapper +* of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5215,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +19 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,24 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, struct task_struct *, tsk, + unsigned long, addr, void *, buf, int, len, unsigned int, gup_flags) +{ +return access_process_vm (tsk, addr, buf, len, gup_flags); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_BTF_ID , + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg2_type =ARG_ANYTHING , + .arg3_type =ARG_PTR_TO_UNINIT_MEM , + .arg4_type =ARG_CONST_SIZE_OR_ZERO , + .arg5_type =ARG_ANYTHING , +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }
tools/include/uapi/linux/bpf.h +10 -0
@@ -5018,6 +5018,15 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags) +* Description +* Read *len* bytes from user space address *addr* in *tsk*'s +* address space, and stores the data in *buf*. This is a wrapper +* of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5215,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuQuoted 58 lines · Kenny Yu
This adds a test for bpf iterator programs to make use of sleepable bpf helpers.
Signed-off-by Kenny Yu <[email protected]>--- .../selftests/bpf/prog_tests/bpf_iter.c | 16 ++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 70 insertions(+)tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16
-0@@ -138,6 +138,20 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))tools/testing/selftests/bpf/progs/bpf_iter_task.c +1
-0@@ -2,6 +2,7 @@/* Copyright (c) 2020 Facebook */ #include "bpf_iter.h" #include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> char _license[] SEC("license") = "GPL";@@ -23,3 +24,56 @@ int dump_task(struct bpf_iter__task *ctx)BPF_SEQ_PRINTF (seq,"%8d %8d\n" , task->tgid, task->pid);
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34
Quoted 10 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +8
-0return 0 ; } + +// New helper added +static long (*bpf_access_process_vm)( +struct task_struct *tsk, +unsigned long addr, +void *buf, +int len, +unsigned int gup_flags) = (void *)186 ;
This shouldn't be needed.
Since patch 1 updates tools/include/uapi/linux/bpf.h it will be in bpf_helper_defs.h automatically.
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37
Quoted 3 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3
-0+ +// Copied from include/linux/mm.h +#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */
Please use C style comments only.
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68
Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5
-0+ numread =bpf_access_process_vm (task, + (unsigned long )ptr, + (void *)&user_data, +sizeof (uint32_t ), +FOLL_REMOTE );
We probably would need to hide flags like FOLL_REMOTE inside the helper otherwise prog might confuse the kernel. In this case I'm not even sure that FOLL_REMOTE is needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything. fast_gup is not necessary either.
Re: [PATCH v2 bpf-next 2/4] bpf: Add support for sleepable programs in bpf_iter_run_prog
In reply to Kenny YuQuoted 25 lines · Kenny Yu
The next patch adds the ability to create sleepable bpf iterator programs. This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.Signed-off-by Kenny Yu <[email protected]>--- kernel/bpf/bpf_iter.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)kernel/bpf/bpf_iter.c +1
-0@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 689
Quoted 7 lines
kernel/bpf/bpf_iter.c +2
-2@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); +if (prog->aux->sleepable) { +rcu_read_lock_trace ();
Pretty cool that a single 'if' is all that is needed to enable sleepable iterators.
Maybe combine under one 'if' ?
if (prog->aux->sleepable) {
lock_trace
migr_dis
might_fault
bpf_prog_run
migr_en
unlock_trace
} else {
lock
migr_dis
bpf_prog_run
migr_end
unlock
}
Would it be easier to read?
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Alexei StarovoitovHi Alexei,
Quoted 11 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34
Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7
-0+// New helper added +static long (*bpf_access_process_vm)( +struct task_struct *tsk, +unsigned long addr, +void *buf, +int len, +unsigned int gup_flags) = (void *)186 ;This shouldn't be needed.
Since patch 1 updatestools/include/uapi/linux/bpf.hit will be inbpf_helper_defs.hautomatically.
I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?
Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37
Quoted 3 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3
-0+ +// Copied from include/linux/mm.h +#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ Please use C style comments only.
I will fix.
Quoted 6 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68
Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5
-0+ numread =bpf_access_process_vm (task, + (unsigned long )ptr, + (void *)&user_data, +sizeof (uint32_t ), +FOLL_REMOTE );
Quoted 6 lines
We probably would need to hide flags like
FOLL_REMOTEinside the helper otherwise prog might confuse the kernel. In this case I'm not even sure thatFOLL_REMOTEis needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything.fast_gupis not necessary either.
Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:
bpf_access_process_vm(struct task_struct *tsk,
unsigned long addr,
void *buf,
int len);
Thanks for the feedback!
Kenny
Re: [PATCH v2 bpf-next 2/4] bpf: Add support for sleepable programs in bpf_iter_run_prog
In reply to Alexei StarovoitovHi Alexei,
Quoted 19 lines
Pretty cool that a single 'if' is all that is needed to enable sleepable iterators.
Maybe combine under one 'if' ?
if (prog->aux->sleepable) {
lock_trace
migr_dis
might_fault
bpf_prog_run
migr_en
unlock_trace
} else {
lock
migr_dis
bpf_prog_run
migr_end
unlock
}
Quoted 1 line
Would it be easier to read?
Yes, I agree, that is more readable. I'll make the change. I'll also follow Yonghong Song's suggestion (offline) of merging this patch into the first patch to keep bisectability together.
Thanks for the feedback!
Kenny
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuQuoted 15 lines · Kenny Yu
Hi Alexei,
Quoted 11 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34
Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7
-0+// New helper added +static long (*bpf_access_process_vm)( +struct task_struct *tsk, +unsigned long addr, +void *buf, +int len, +unsigned int gup_flags) = (void *)186 ;This shouldn't be needed.
Since patch 1 updatestools/include/uapi/linux/bpf.hit will be inbpf_helper_defs.hautomatically.
Quoted 3 lines
I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?
Look at vmtest.sh under tools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.
Quoted 22 lines
Quoted 6 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37
Quoted 4 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3
-0+ +// Copied from include/linux/mm.h +#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ Please use C style comments only.
I will fix.
Quoted 6 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68
Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5
-0+ numread =bpf_access_process_vm (task, + (unsigned long )ptr, + (void *)&user_data, +sizeof (uint32_t ), +FOLL_REMOTE );Quoted 6 lines
We probably would need to hide flags like
FOLL_REMOTEinside the helper otherwise prog might confuse the kernel. In this case I'm not even sure thatFOLL_REMOTEis needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything.fast_gupis not necessary either.
Quoted 8 lines
Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:
bpf_access_process_vm(struct
task_struct*tsk,
unsigned long addr,
void *buf,
int len);
keeping generic u64 flags makes sense for the future, so I'd keep it.
But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (in probe_read it's just an address, here it's tsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:
buf, len, tsk, addr, flags
?
Quoted 4 lines
Thanks for the feedback!
Kenny
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
Hi Andrii,
Quoted 3 lines
Look at
vmtest.shundertools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.
Thanks, this helps!
Quoted 1 line
keeping generic u64 flags makes sense for the future, so I'd keep it.
That makes sense, I'll keep the flags in that case.
Quoted 11 lines
But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and
bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (inprobe_readit's just an address, here it'stsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:buf, len, tsk, addr, flags
?
Yeah, that looks better for consistency. Should I still keep the name as bpf_access_process_vm, or call it something else to be more consistent with the naming of the other bpf helpers? The benefit of the bpf_access_process_vm name is that it makes it obvious it is wrapping an existing function access_process_vm.
Thanks for the feedback!
Kenny
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Andrii NakryikoQuoted 66 lines · Andrii Nakryiko
Quoted 15 lines · Kenny Yu
Hi Alexei,
Quoted 11 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 34
Quoted 7 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +7
-0+// New helper added +static long (*bpf_access_process_vm)( +struct task_struct *tsk, +unsigned long addr, +void *buf, +int len, +unsigned int gup_flags) = (void *)186 ;This shouldn't be needed.
Since patch 1 updatestools/include/uapi/linux/bpf.hit will be inbpf_helper_defs.hautomatically.Quoted 3 lines
I will fix. This is my first time writing selftests, so I am not too familiar with how these are built and run. For my understanding, are these tests meant to be built and run after booting the new kernel?
Look at
vmtest.shundertools/testing/selftests/bpf, it handles building kernel, selftests and spinning up qemu instance for running selftests inside it.Quoted 22 lines
Quoted 6 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 37
Quoted 4 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +3
-0+ +// Copied from include/linux/mm.h +#define FOLL_REMOTE 0x2000/* we are working on non-current tsk/mm */ Please use C style comments only.
I will fix.
Quoted 6 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 68
Quoted 5 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +5
-0+ numread =bpf_access_process_vm (task, + (unsigned long )ptr, + (void *)&user_data, +sizeof (uint32_t ), +FOLL_REMOTE );Quoted 6 lines
We probably would need to hide flags like
FOLL_REMOTEinside the helper otherwise prog might confuse the kernel. In this case I'm not even sure thatFOLL_REMOTEis needed. I suspect gup_flags=0 in all cases will work fine. We're not doing write here and not pining anything.fast_gupis not necessary either.Quoted 8 lines
Thanks for the suggestion! I'll remove the flag argument from the helper to simplify the API for bpf programs. This means that the helper will have the following signature:
bpf_access_process_vm(struct
task_struct*tsk,
unsigned long addr,
void *buf,
int len);keeping generic u64 flags makes sense for the future, so I'd keep it.
But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and
bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (inprobe_readit's just an address, here it'stsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:
Quoted 4 lines
buf, len, tsk, addr, flags
?
I would personally find it more intuitive to have process information passed as either the first argument (like process_vm_readv does), or as "last", just before the flags (as extra information required w.r.t. to local versions, e.g. bpf_copy_from_user).
Re: [PATCH v2 bpf-next 4/4] selftests/bpf: Add test for sleepable bpf iterator programs
Quoted 10 lines
Quoted 10 lines
But I also wanted to point out that this helper is logically in the same family as bpf_probe_read_kernel/user and
bpf_copy_from_user, etc, where we have consistent pattern that first two arguments specify destination buffer (so buf + len) and the remaining ones specify source (inprobe_readit's just an address, here it'stsk_addr). So I wonder if it would be less surprising and more consistent to reorder and have:buf, len, tsk, addr, flags
Quoted 7 lines
Quoted 2 lines
?
I would personally find it more intuitive to have process information passed as either the first argument (like
process_vm_readvdoes), or as "last", just before the flags (as extra information required w.r.t. to local versions, e.g.bpf_copy_from_user).
I think that makes sense. I'll combine both Andrii's and Gabriele's suggestions and keep the signature as close to the existing helpers
(e.g., bpf_probe_read_user) and add the additional arguments at the end. I'll proceed with this signature:
bpf_access_process_vm(void *dst,
u32 size,
const void *unsafe_ptr,
struct task_struct *tsk,
unsigned int flags);
Thanks for all the suggestions!
Kenny
v3 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 +++++ kernel/bpf/bpf_iter.c | 20 ++++++--- kernel/bpf/helpers.c | 19 ++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 10 +++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 43 +++++++++++++++++++ 9 files changed, 117 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
PATCH v3 1/3 bpf: Add bpf_access_process_vm() helper
This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- kernel/bpf/helpers.c | 21 +++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 6 files changed, 61 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
@@ -2222,6 +2222,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
@@ -5018,6 +5018,16 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags) +* Description +* Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5216,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay
kernel/bpf/helpers.c +21 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,26 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u32, flags) +{ +/* flags is not used yet */ +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }
tools/include/uapi/linux/bpf.h +11 -0
@@ -5018,6 +5018,16 @@ union bpf_attr {* * Return * The number of arguments of the traced function. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags) +* Description +* Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5206,6 +5216,7 @@ union bpf_attr {FN (get_func_arg), \FN (get_func_ret), \FN (get_func_arg_cnt), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
PATCH v3 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs
In reply to Kenny YuThis adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v3 3/3 selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuThis adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
@@ -138,6 +138,20 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +44 -0
53 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCH v3 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
In reply to Kenny Yukernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 681
Quoted 8 lines · Kenny Yu
kernel/bpf/helpers.c +7
-0+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u32, flags) +{ +/* flags is not used yet */ +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +}
Though the 'flags' argument is unused the helper has to check that it's zero and return -EINVAL otherwise.
If we don't do this we won't be able to change the behavior later due to backward compatibility.
Re: [PATCH v3 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 4 lines
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u32, flags) +{
Quoted 8 lines
kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 681
Quoted 3 lines
kernel/bpf/helpers.c +3
-0+/* flags is not used yet */ +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +}Though the 'flags' argument is unused the helper has to check that it's zero and return
-EINVALotherwise.
If we don't do this we won't be able to change the behavior later due to backward compatibility.
Thanks for the feedback! Will fix.
Kenny
PATCH v4 1/3 bpf: Add bpf_access_process_vm() helper
In reply to Kenny YuThis adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- kernel/bpf/helpers.c | 23 +++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags) +* Description +* Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay
kernel/bpf/helpers.c +23 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u32, flags) +{ +/* flags is not used yet */ +if (flags !=0 ) +return -EINVAL ; +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }
tools/include/uapi/linux/bpf.h +11 -0
@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags) +* Description +* Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
v4 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.
Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++ kernel/bpf/bpf_iter.c | 20 ++++++--- kernel/bpf/helpers.c | 23 ++++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 11 +++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 44 +++++++++++++++++++ 9 files changed, 124 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
PATCH v4 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs
This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v4 3/3 selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuThis adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 44 +++++++++++++++++++ 2 files changed, 60 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +16 -0
@@ -138,6 +138,20 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1266,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +44 -0
53 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCH v4 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
In reply to Kenny YuQuoted 40 lines · Kenny Yu
This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- kernel/bpf/helpers.c | 23 +++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +0
-0
include/uapi/linux/bpf.h union bpf_attr { · line 5053
Quoted 5 lines
include/uapi/linux/bpf.h +2
-0@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)
Maybe we can change 'flags' type to u64? This will leave more room for future potential extensions. In all recent helpers with added 'flags', most of them are u64.
Quoted 18 lines
include/uapi/linux/bpf.h +9
-0+* Description +* Read *size* bytes from user space address *unsafe_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ + FN(access_process_vm), \/* */
[...]
Re: [PATCH v4 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 5 lines
Quoted 1 line
+ * long
bpf_access_process_vm(void *dst, u32 size, const void *unsafe_ptr, struct task_struct *tsk, u32 flags)Maybe we can change 'flags' type to u64? This will leave more room for future potential extensions. In all recent helpers with added 'flags', most of them are u64.
I'll change it to u64. Thanks for the suggestion!
Kenny
v5 cover Add bpf_access_process_vm helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_access_process_vm to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.
Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.
Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (3):
bpf: Add bpf_access_process_vm() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++ kernel/bpf/bpf_iter.c | 20 ++++++--- kernel/bpf/helpers.c | 23 ++++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 11 +++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 16 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 44 +++++++++++++++++++ 9 files changed, 124 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
PATCH v5 3/3 selftests/bpf: Add test for sleepable bpf iterator programs
This adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 19 ++++++++ .../selftests/bpf/progs/bpf_iter_task.c | 47 +++++++++++++++++++ 2 files changed, 66 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +19 -0
@@ -138,6 +138,23 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +ASSERT_GT (skel->bss->num_success_access_process_vm,0 , +"num_success_access_process_vm" ); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1269,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +47 -0
56 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v5 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs
In reply to Kenny YuThis adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8599,6 +8599,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v5 1/3 bpf: Add bpf_access_process_vm() helper
In reply to Kenny YuThis adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- kernel/bpf/helpers.c | 23 +++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay
kernel/bpf/helpers.c +23 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u64, flags) +{ +/* flags is not used yet */ +if (flags) +return -EINVAL ; +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }
tools/include/uapi/linux/bpf.h +11 -0
@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 165 lines · Kenny Yu
This adds a helper for bpf programs to access the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- kernel/bpf/helpers.c | 23 +++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 6 files changed, 63 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2220,6 +2220,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_access_process_vm_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +11
-0@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper kernel/bpf/bpf_iter.c +15
-5@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay kernel/bpf/helpers.c +23
-0@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h" @@ -671,6 +672,28 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_access_process_vm, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, + u64, flags) +{ +/* flags is not used yet */ +if (flags) +return -EINVAL ; +return access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +} + +const struct bpf_func_proto bpf_access_process_vm_proto = { + .func = bpf_access_process_vm, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)kernel/trace/bpf_trace.c +2
-0@@ -1257,6 +1257,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_find_vma_proto;case BPF_FUNC_trace_vprintk:return bpf_get_trace_vprintk_proto (); +case BPF_FUNC_access_process_vm: +return prog->aux->sleepable ? &bpf_access_process_vm_proto :NULL ;default :return bpf_base_func_proto (func_id); }tools/include/uapi/linux/bpf.h +0
-0@@ -5049,6 +5049,16 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure.
tools/include/uapi/linux/bpf.h union bpf_attr { · line 5061
Quoted 10 lines
tools/include/uapi/linux/bpf.h +10
-0+* +* long bpf_access_process_vm(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This is a +* wrapper of **access_process_vm**\ (). +* Return +* The number of bytes written to the buffer, or a negative error +* in case of failure.
wait, can it read less than *size* and return success?
bpf_probe_read_kernel() returns:
0 on success, or a negative error in case of failure.
Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).
BTW, should we also add a C string reading helper as well, just like there is bpf_probe_read_user_str() and bpf_probe_read_user()?
Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.
And not to start the bikeshedding session, but we have bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.
Quoted 14 lines
tools/include/uapi/linux/bpf.h +1
-1*/ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5249,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (access_process_vm), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -- 2.30.2
Re: [PATCH v5 bpf-next 3/3] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuQuoted 15 lines · Kenny Yu
This adds a test for bpf iterator programs to make use of sleepable bpf helpers.
Signed-off-by Kenny Yu <[email protected]>--- .../selftests/bpf/prog_tests/bpf_iter.c | 19 ++++++++ .../selftests/bpf/progs/bpf_iter_task.c | 47 +++++++++++++++++++ 2 files changed, 66 insertions(+)tools/testing/selftests/bpf/prog_tests/bpf_iter.c +0
-0
tools/testing/selftests/bpf/prog_tests/bpf_iter.c static void test_task(void) · line 147
Quoted 10 lines
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +7
-0@@ -138,6 +138,23 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (CHECK (!skel,"bpf_iter_task__open_and_load" , +"skeleton open_and_load failed\n" ))
Please use ASSERT_OK_PTR() instead.
Quoted 62 lines
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +12
-0+return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +ASSERT_GT (skel->bss->num_success_access_process_vm,0 , +"num_success_access_process_vm" ); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;@@ -1252,6 +1269,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))tools/testing/selftests/bpf/progs/bpf_iter_task.c +25
-0@@ -2,6 +2,7 @@/* Copyright (c) 2020 Facebook */ #include "bpf_iter.h" #include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> char _license[] SEC("license") = "GPL";@@ -23,3 +24,49 @@ int dump_task(struct bpf_iter__task *ctx)BPF_SEQ_PRINTF (seq,"%8d %8d\n" , task->tgid, task->pid);return 0 ; } + +int num_success_access_process_vm =0 ; + +SEC("iter.s/task") +int dump_task_sleepable (struct bpf_iter__task *ctx) +{ +struct seq_file *seq = ctx->meta->seq; +struct task_struct *task = ctx->task; +static const char info[] =" === END ===" ; +struct pt_regs *regs; +void *ptr; +uint32_t user_data =0 ; +int numread; + +if (task == (void *)0 ) { +BPF_SEQ_PRINTF (seq,"%s\n" , info); +return 0 ; + } + + regs = (struct pt_regs *)bpf_task_pt_regs (task); +if (regs == (void *)0 ) { +BPF_SEQ_PRINTF (seq,"%s\n" , info); +return 0 ; + }
tools/testing/selftests/bpf/progs/bpf_iter_task.c int dump_task(struct bpf_iter__task *ctx) · line 60
Quoted 10 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +10
-0+ ptr = (void *)PT_REGS_IP (regs); + +/* Try to read the contents of the task's instruction pointer from the +* remote task's address space. +*/ + numread =bpf_access_process_vm (&user_data, +sizeof (uint32_t ), + ptr, + task, +0 );
nit: keep it on one line (up to 100 characters is ok)
Quoted 15 lines
tools/testing/selftests/bpf/progs/bpf_iter_task.c +12
-1+if (numread !=sizeof (uint32_t )) { +BPF_SEQ_PRINTF (seq,"%s\n" , info); +return 0 ; + } + ++num_success_access_process_vm; + +if (ctx->meta->seq_num ==0 ) +BPF_SEQ_PRINTF (seq," tgid gid data\n" ); + +BPF_SEQ_PRINTF (seq,"%8d %8d %8d\n" , task->tgid, task->pid, user_data); +return 0 ; +} --2.30.2
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
In reply to Andrii NakryikoOn Thu, Jan 20, 2022 at 2:46 PM Andrii Nakryiko
<[email protected]> wrote:
a
Quoted 9 lines
Quoted 4 lines
+ * wrapper of **access_process_vm**\ ().
+ * Return
+ * The number of bytes written to the buffer, or a negative error
+ * in case of failure.wait, can it read less than *size* and return success?
bpf_probe_read_kernel()returns:
Quoted 8 lines
0 on success, or a negative error in case of failure.
Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).
BTW, should we also add a C string reading helper as well, just like there is
bpf_probe_read_user_str()andbpf_probe_read_user()?
That would be difficult. There is no suitable kernel api for that.
Quoted 9 lines
Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.
And not to start the bikeshedding session, but we have
bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.
How about bpf_copy_from_user_task() ?
The task is the second to last argument, so the name fits ? Especially if we call it this way it would be best to align return codes with bpf_copy_from_user.
Adding memset() in case of failure is mandatory too. I've missed this bit earlier.
The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we should memset() the whole buffer and return -EFAULT or memset only the leftover bytes and return 0? I think the former is best to align with bpf_copy_from_user.
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 15 lines · Alexei Starovoitov
On Thu, Jan 20, 2022 at 2:46 PM Andrii Nakryiko
<[email protected]> wrote:
aQuoted 11 lines
Quoted 4 lines
+ * wrapper of **access_process_vm**\ ().
+ * Return
+ * The number of bytes written to the buffer, or a negative error
+ * in case of failure.wait, can it read less than *size* and return success?
bpf_probe_read_kernel()returns:0 on success, or a negative error in case of failure.
Quoted 8 lines
Quoted 6 lines
Let's be consistent. Returning the number of read bytes makes more sense in cases when we don't know the amount of bytes to be actually read ahead of time (e.g., when reading zero-terminated strings).
BTW, should we also add a C string reading helper as well, just like there is
bpf_probe_read_user_str()andbpf_probe_read_user()?That would be difficult. There is no suitable kernel api for that.
Ok, but maybe we can add it later. Otherwise it will be hard to profiler Python processes and such, because you most certainly will need to read zero-terminated strings there.
Quoted 11 lines
Quoted 10 lines
Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.
And not to start the bikeshedding session, but we have
bpf_copy_from_user(), wouldn't something like bpf_copy_from_user_{vm,process,remote}() be more in line and less surprising for BPF users. BTW, "access" implies writing just as much as reading, so using "access" in the sense of "read" seems wrong and confusing.
Quoted 2 lines
How about
bpf_copy_from_user_task()?
The task is the second to last argument, so the name fits ?
yeah, I like the name
Quoted 4 lines
Especially if we call it this way it would be best to align return codes with
bpf_copy_from_user.
Addingmemset()in case of failure is mandatory too. I've missed this bit earlier.
Yep, good catch! Seems like copy_from_user() currently returns amount of bytes *not* read and memsets those unread bytes to zero. So for efficiency we could probably memset only those that were read.
Quoted 6 lines
The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we shouldmemset()the whole buffer and return-EFAULTor memset only the leftover bytes and return 0? I think the former is best to align withbpf_copy_from_user.
Yeah, I think all or nothing approach (either complete success and zero return, or memset and error return) is best and most in line with other similar helpers.
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 4 lines
Quoted 2 lines
How about
bpf_copy_from_user_task()?
The task is the second to last argument, so the name fits ?yeah, I like the name
I'll change the name to bpf_copy_from_user_task.
Quoted 15 lines
Quoted 4 lines
Especially if we call it this way it would be best to align return codes with
bpf_copy_from_user.
Addingmemset()in case of failure is mandatory too. I've missed this bit earlier.Yep, good catch! Seems like
copy_from_user()currently returns amount of bytes *not* read and memsets those unread bytes to zero. So for efficiency we could probably memset only those that were read.Quoted 5 lines
The question is to decide what to do with
ret > 0 && ret < size condition.
Is it a failure and we shouldmemset()the whole buffer and return-EFAULTor memset only the leftover bytes and return 0? I think the former is best to align withbpf_copy_from_user.
Quoted 3 lines
Yeah, I think all or nothing approach (either complete success and zero return, or memset and error return) is best and most in line with other similar helpers.
Thanks for the suggestions! I'll go with the all-or-nothing approach to be consistent with bpf_copy_from_user and will make the following changes:
* Return value: returns 0 on success, or negative error on failure.
* If we had a partial read, we will memset the read bytes to 0 and return -EFAULT
Quoted 2 lines
Another thing, I think it's important to mention that this helper can be used only from sleepable BPF programs.
Will add that to the docs.
Quoted 5 lines
Quoted 1 line
That would be difficult. There is no suitable kernel api for that.
Ok, but maybe we can add it later. Otherwise it will be hard to profiler Python processes and such, because you most certainly will need to read zero-terminated strings there.
I will NOT add a C string helper in this patch series, and I'll explore how to add this in the future once this patch series is merged.
Quoted 5 lines
Review
Quoted 3 lines
(quoted diff) +3
-0+ skel = bpf_iter_task__open_and_load(); + if (CHECK(!skel, "bpf_iter_task__open_and_load", + "skeleton open_and_load failed\n"))Please use
ASSERT_OK_PTR()instead.
Will fix.
Quoted 7 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+ numread = bpf_access_process_vm(&user_data, + sizeof(uint32_t), + ptr, + task, + 0);nit: keep it on one line (up to 100 characters is ok)
Will fix.
Thanks for the suggestions everyone!
Kenny
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 7 lines · Kenny Yu
Thanks for the suggestions! I'll go with the all-or-nothing approach to be consistent with
bpf_copy_from_userand will make the following changes:* Return value: returns 0 on success, or negative error on failure.
* If we had a partial read, we will memset the read bytes to 0 and return-EFAULT
Not read bytes, but all bytes.copy_from_user zeros leftover for us, but access_process_vm doesn't do this on partial read.
Re: [PATCH v5 bpf-next 1/3] bpf: Add bpf_access_process_vm() helper
Quoted 6 lines
Quoted 2 lines
* If we had a partial read, we will memset the read bytes to 0 and return
-EFAULTNot read bytes, but all bytes.
copy_from_userzeros leftover for us, butaccess_process_vmdoesn't do this on partial read.
I'll memset all bytes on error. Thanks for the suggestion!
Kenny
v6 cover Add bpf_copy_from_user_task helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_copy_from_user_task to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v5:
* Rename bpf_access_process_vm to bpf_copy_from_user_task.
* Change return value to be all-or-nothing. If we get a partial read, memset all bytes to 0 and return -EFAULT.
* Add to docs that the helper can only be used by sleepable BPF programs.
* Fix nits in selftests.
Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.
Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.
Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (3):
bpf: Add bpf_copy_from_user_task() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 +++++ kernel/bpf/bpf_iter.c | 20 ++++++--- kernel/bpf/helpers.c | 31 +++++++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 10 +++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 18 ++++++++ .../selftests/bpf/progs/bpf_iter_task.c | 43 +++++++++++++++++++ 9 files changed, 131 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
PATCH v6 2/3 libbpf: Add "iter.s" section for sleepable bpf iterator programs
This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8607,6 +8607,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v6 1/3 bpf: Add bpf_copy_from_user_task() helper
In reply to Kenny YuThis adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes bpf_iter_run_prog to use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_iter.c | 20 ++++++++++++++----- kernel/bpf/helpers.c | 35 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-)
include/linux/bpf.h +1 -0
@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +10 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. +* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/bpf_iter.c +15 -5
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay
kernel/bpf/helpers.c +35 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,40 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) +{ +int ret; + +/* flags is not used yet */ +if (flags) +return -EINVAL ; + + ret =access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +if (ret >=0 ) { +if (ret == size) + ret =0 ; +else { +/* If partial read, clear all bytes and return error */ +memset (dst,0 , size); + ret = -EFAULT ; + } + } +return ret; +} + +const struct bpf_func_proto bpf_copy_from_user_task_proto = { + .func = bpf_copy_from_user_task, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1235,6 +1235,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_get_task_stack_proto;case BPF_FUNC_copy_from_user:return prog->aux->sleepable ? &bpf_copy_from_user_proto :NULL ; +case BPF_FUNC_copy_from_user_task: +return prog->aux->sleepable ? &bpf_copy_from_user_task_proto :NULL ;case BPF_FUNC_snprintf_btf:return &bpf_snprintf_btf_proto;case BPF_FUNC_per_cpu_ptr:
tools/include/uapi/linux/bpf.h +10 -0
@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. +* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
PATCH v6 3/3 selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuThis adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 20 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 74 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +20 -0
@@ -138,6 +138,24 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (!ASSERT_OK_PTR (skel,"bpf_iter_task__open_and_load" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +ASSERT_GT (skel->bss->num_expected_failure_copy_from_user_task,0 , +"num_expected_failure_copy_from_user_task" ); +ASSERT_GT (skel->bss->num_success_copy_from_user_task,0 , +"num_success_copy_from_user_task" ); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1270,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0
63 lines not on this page · raw message
Signature · 1 line
2.30.2
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
In reply to Kenny YuQuoted 41 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_iter.c | 20 ++++++++++++++----- kernel/bpf/helpers.c | 35 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +0
-0
include/uapi/linux/bpf.h union bpf_attr { · line 5058
Quoted 10 lines
include/uapi/linux/bpf.h +7
-0@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs.
"On error dst buffer is zeroed out."? This is an explicit guarantee.
Quoted 39 lines
include/uapi/linux/bpf.h +3
-0+* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper kernel/bpf/bpf_iter.c +6
-5@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx);
kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702
Quoted 10 lines
kernel/bpf/bpf_iter.c +9
-0+migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }
I think this sleepable bpf_iter change deserves its own patch. It has nothing to do with bpf_copy_from_user_task() helper.
Quoted 21 lines
kernel/bpf/bpf_iter.c +0
-0/* bpf program can only return 0 or 1: * 0 : okay kernel/bpf/helpers.c +4
-0@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h" @@ -671,6 +672,40 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) +{
kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 687
Quoted 10 lines
kernel/bpf/helpers.c +10
-0+int ret; + +/* flags is not used yet */ +if (flags) +return -EINVAL ; + + ret =access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +if (ret >=0 ) { +if (ret == size) + ret =0 ;
is there a point in calling access_process_vm() with size == 0? It would validate that get_task_mm() succeeds, but that's pretty much it? So maybe instead just exit early if size is zero? It will be also less convoluted logic:
if (size == 0)
return 0;
if (access_process_vm(...)) {
memset(0);
return -EFAULT;
}return 0;
Quoted 9 lines
kernel/bpf/helpers.c +9
-0+else { +/* If partial read, clear all bytes and return error */ +memset (dst,0 , size); + ret = -EFAULT ; + } + } +return ret; +} +
[...]
Re: [PATCH v6 bpf-next 2/3] libbpf: Add "iter.s" section for sleepable bpf iterator programs
In reply to Kenny YuQuoted 6 lines · Kenny Yu
This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
Signed-off-by Kenny Yu <[email protected]>---
Quoted 18 lines
tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)tools/lib/bpf/libbpf.c +1
-1@@ -8607,6 +8607,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp_devmap/" ,XDP ,BPF_XDP_DEVMAP ,SEC_ATTACHABLE ), SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), --2.30.2
Re: [PATCH v6 bpf-next 3/3] selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuQuoted 6 lines · Kenny Yu
This adds a test for bpf iterator programs to make use of sleepable bpf helpers.
Signed-off-by Kenny Yu <[email protected]>---
Quoted 4 lines
.../selftests/bpf/prog_tests/bpf_iter.c | 20 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 74 insertions(+)
[...]
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
In reply to Andrii NakryikoQuoted 99 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_iter.c | 20 ++++++++++++++----- kernel/bpf/helpers.c | 35 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +0
-0include/uapi/linux/bpf.h union bpf_attr { · line 5058
Quoted 10 lines
include/uapi/linux/bpf.h +7
-0@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. "On error dst buffer is zeroed out."? This is an explicit guarantee.
Quoted 44 lines
include/uapi/linux/bpf.h +3
-0+* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper kernel/bpf/bpf_iter.c +11
-5@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable ();
Quoted 8 lines
kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702
Quoted 5 lines
kernel/bpf/bpf_iter.c +4
-0+ ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }I think this sleepable
bpf_iterchange deserves its own patch. It has nothing to do withbpf_copy_from_user_task()helper.
Without the above change, using bpf_copy_from_user_task() will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducing bpf_copy_from_user_task() so we won't have bisecting issues.
Quoted 3 lines
Quoted 3 lines
kernel/bpf/bpf_iter.c +0
-0/* bpf program can only return 0 or 1: * 0 : okay
[...]
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
Quoted 112 lines · Yonghong Song
Quoted 99 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_iter.c | 20 ++++++++++++++----- kernel/bpf/helpers.c | 35 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +0
-0include/uapi/linux/bpf.h union bpf_attr { · line 5058
Quoted 10 lines
include/uapi/linux/bpf.h +7
-0@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. "On error dst buffer is zeroed out."? This is an explicit guarantee.
Quoted 44 lines
include/uapi/linux/bpf.h +3
-0+* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper kernel/bpf/bpf_iter.c +11
-5@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable ();Quoted 8 lines
kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702
Quoted 5 lines
kernel/bpf/bpf_iter.c +4
-0+ ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }I think this sleepable
bpf_iterchange deserves its own patch. It has nothing to do withbpf_copy_from_user_task()helper.
Quoted 4 lines
Without the above change, using
bpf_copy_from_user_task()will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducingbpf_copy_from_user_task()so we won't have bisecting issues.
Sure, patch #1 for sleepable bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
kernel/bpf/bpf_iter.c +0
-0/* bpf program can only return 0 or 1: * 0 : okay [...]
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
Quoted 113 lines · Andrii Nakryiko
Quoted 112 lines · Yonghong Song
Quoted 107 lines · Andrii Nakryiko
Quoted 41 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks. This also adds the ability for bpf iterator programs to be sleepable.
This changes
bpf_iter_run_progto use the appropriate synchronization for sleepable bpf programs. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 10 ++++++++++ kernel/bpf/bpf_iter.c | 20 ++++++++++++++----- kernel/bpf/helpers.c | 35 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 10 ++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-)include/linux/bpf.h +1
-0@@ -2229,6 +2229,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);include/uapi/linux/bpf.h +0
-0include/uapi/linux/bpf.h union bpf_attr { · line 5058
Quoted 10 lines
include/uapi/linux/bpf.h +7
-0@@ -5049,6 +5049,15 @@ union bpf_attr {* This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. "On error dst buffer is zeroed out."? This is an explicit guarantee.
Quoted 39 lines
include/uapi/linux/bpf.h +3
-0+* Return +* 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \@@ -5239,6 +5248,7 @@ union bpf_attr {FN (get_func_arg_cnt), \FN (get_retval), \FN (set_retval), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper kernel/bpf/bpf_iter.c +6
-5@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx);kernel/bpf/bpf_iter.c int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) · line 702
Quoted 10 lines
kernel/bpf/bpf_iter.c +9
-0+migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }I think this sleepable
bpf_iterchange deserves its own patch. It has nothing to do withbpf_copy_from_user_task()helper.
Quoted 8 lines
Quoted 4 lines
Without the above change, using
bpf_copy_from_user_task()will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducingbpf_copy_from_user_task()so we won't have bisecting issues.Sure, patch #1 for sleepable
bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.
okay with me.
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
kernel/bpf/bpf_iter.c +0
-0/* bpf program can only return 0 or 1: * 0 : okay [...]
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
In reply to Andrii NakryikoQuoted 3 lines · Andrii Nakryiko
"On error dst buffer is zeroed out."? This is an explicit guarantee.
I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure? Can't this step be skipped for performance and leave it up to the callee to perform if they need it?
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
s/callee/caller
Quoted 10 lines · Gabriele
Quoted 3 lines · Andrii Nakryiko
"On error dst buffer is zeroed out."? This is an explicit guarantee.
I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure? Can't this step be skipped for performance and leave it up to the callee to perform if they need it?
Signature · 6 lines
"Egli è scritto in lingua matematica, e i caratteri son triangoli, cerchi, ed altre figure geometriche, senza i quali mezzi è impossibile a intenderne umanamente parola; senza questi è un aggirarsi vanamente per un oscuro laberinto." -- G. Galilei, Il saggiatore.
Re: [PATCH v6 bpf-next 1/3] bpf: Add bpf_copy_from_user_task() helper
In reply to Yonghong SongQuoted 1 line
"On error dst buffer is zeroed out."? This is an explicit guarantee.
Will add to the docs.
Quoted 5 lines
is there a point in calling
access_process_vm()with size == 0? It would validate thatget_task_mm()succeeds, but that's pretty much it? So maybe instead just exit early if size is zero? It will be also less convoluted logic:
Quoted 7 lines
if (size == 0)
return 0; if (access_process_vm(...)) { memset(0); return -EFAULT; }return 0;
Will do an explicit check for size == 0.
Note that we still need to check if the return value of access_process_vm == size to see if we have a partial read.
Quoted 8 lines
Quoted 4 lines
Without the above change, using
bpf_copy_from_user_task()will trigger rcu warning and may produce incorrect result. One option is to put the above in a preparation patch before introducingbpf_copy_from_user_task()so we won't have bisecting issues.Sure, patch #1 for sleepable
bpf_iter, patch #2 for the helper? I mean, it's not a big deal, but both seem to deserve their own focused patches.
I'll split this into 2 patches and place the bpf_iter patch first.
Quoted 3 lines
I appreciate that existing helpers already do this and it's good to follow suit for consistency, but what is the rationale behind zeroing memory on failure?
I believe the intent behind this is for security/privacy reasons. On error, we don't want to unintentionally leak partially read data.
Thanks everyone for the suggestions!
Kenny
v7 cover Add bpf_copy_from_user_task helper and sleepable bpf iterator programs
In reply to Kenny YuThis patch series makes the following changes:
* Adds a new bpf helper bpf_copy_from_user_task to read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.
As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
Changes since v6:
* Split first patch into two patches: first patch to add support for bpf iterators to use sleepable helpers, and the second to add the new bpf helper.
* Simplify implementation of bpf_copy_from_user_task based on feedback.
* Add to docs that the destination buffer will be zero-ed on error.
Changes since v5:
* Rename bpf_access_process_vm to bpf_copy_from_user_task.
* Change return value to be all-or-nothing. If we get a partial read, memset all bytes to 0 and return -EFAULT.
* Add to docs that the helper can only be used by sleepable BPF programs.
* Fix nits in selftests.
Changes since v4:
* Make flags into u64.
* Use user_ptr arg name to be consistent with bpf_copy_from_user.
* Add an extra check in selftests to verify access_process_vm calls succeeded.
Changes since v3:
* Check if flags is 0 and return -EINVAL if not.
* Rebase on latest bpf-next branch and fix merge conflicts.
Changes since v2:
* Reorder arguments in bpf_access_process_vm to match existing related bpf helpers (e.g. bpf_probe_read_kernel, bpf_probe_read_user, bpf_copy_from_user).
* flags argument is provided for future extensibility and is not currently used, and we always invoke access_process_vm with no flags.
* Merge bpf helper patch and bpf_iter_run_prog patch together for better bisectability in case of failures.
* Clean up formatting and comments in selftests.
Changes since v1:
* Fixed "Invalid wait context" issue in bpf_iter_run_prog by using rcu_read_lock_trace() for sleepable bpf iterator programs.
Kenny Yu (4):
bpf: Add support for bpf iterator programs to use sleepable helpers
bpf: Add bpf_copy_from_user_task() helper
libbpf: Add "iter.s" section for sleepable bpf iterator programs selftests/bpf: Add test for sleepable bpf iterator programs
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 ++++ kernel/bpf/bpf_iter.c | 20 +++++-- kernel/bpf/helpers.c | 34 ++++++++++++ kernel/trace/bpf_trace.c | 2 + tools/include/uapi/linux/bpf.h | 11 ++++ tools/lib/bpf/libbpf.c | 1 + .../selftests/bpf/prog_tests/bpf_iter.c | 20 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 9 files changed, 149 insertions(+), 5 deletions(-)
Signature · 1 line
2.30.2
PATCH v7 3/4 libbpf: Add "iter.s" section for sleepable bpf iterator programs
This adds a new bpf section "iter.s" to allow bpf iterator programs to be sleepable.
--- tools/lib/bpf/libbpf.c | 1 + 1 file changed, 1 insertion(+)
tools/lib/bpf/libbpf.c +1 -0
@@ -8612,6 +8612,7 @@ static const struct bpf_sec_def section_defs[] = {SEC_DEF ("lsm/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF , attach_lsm),SEC_DEF ("lsm.s/" ,LSM ,BPF_LSM_MAC ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_lsm),SEC_DEF ("iter/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF , attach_iter), +SEC_DEF ("iter.s/" ,TRACING ,BPF_TRACE_ITER ,SEC_ATTACH_BTF |SEC_SLEEPABLE , attach_iter),SEC_DEF ("syscall" ,SYSCALL ,0 ,SEC_SLEEPABLE ),SEC_DEF ("xdp.frags/devmap" ,XDP ,BPF_XDP_DEVMAP ,SEC_XDP_FRAGS ), SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
Signature · 1 line
2.30.2
PATCH v7 4/4 selftests/bpf: Add test for sleepable bpf iterator programs
In reply to Kenny YuThis adds a test for bpf iterator programs to make use of sleepable bpf helpers.
--- .../selftests/bpf/prog_tests/bpf_iter.c | 20 +++++++ .../selftests/bpf/progs/bpf_iter_task.c | 54 +++++++++++++++++++ 2 files changed, 74 insertions(+)
tools/testing/selftests/bpf/prog_tests/bpf_iter.c +20 -0
@@ -138,6 +138,24 @@ static void test_task(void)bpf_iter_task__destroy (skel); } +static void test_task_sleepable (void ) +{ +struct bpf_iter_task *skel; + + skel =bpf_iter_task__open_and_load (); +if (!ASSERT_OK_PTR (skel,"bpf_iter_task__open_and_load" )) +return ; + +do_dummy_read (skel->progs.dump_task_sleepable); + +ASSERT_GT (skel->bss->num_expected_failure_copy_from_user_task,0 , +"num_expected_failure_copy_from_user_task" ); +ASSERT_GT (skel->bss->num_success_copy_from_user_task,0 , +"num_success_copy_from_user_task" ); + +bpf_iter_task__destroy (skel); +} +static void test_task_stack (void ) {struct bpf_iter_task_stack *skel;
@@ -1252,6 +1270,8 @@ void test_bpf_iter(void)test_bpf_map ();if (test__start_subtest ("task" ))test_task (); +if (test__start_subtest ("task_sleepable" )) +test_task_sleepable ();if (test__start_subtest ("task_stack" ))test_task_stack ();if (test__start_subtest ("task_file" ))
tools/testing/selftests/bpf/progs/bpf_iter_task.c +54 -0
63 lines not on this page · raw message
Signature · 1 line
2.30.2
PATCH v7 2/4 bpf: Add bpf_copy_from_user_task() helper
In reply to Kenny YuThis adds a helper for bpf programs to read the memory of other tasks.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
--- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/helpers.c | 34 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 5 files changed, 59 insertions(+)
include/linux/bpf.h +1 -0
@@ -2238,6 +2238,7 @@ extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;extern const struct bpf_func_proto bpf_find_vma_proto;extern const struct bpf_func_proto bpf_loop_proto;extern const struct bpf_func_proto bpf_strncmp_proto; +extern const struct bpf_func_proto bpf_copy_from_user_task_proto;const struct bpf_func_proto *tracing_prog_func_proto (enum bpf_func_id func_id,const struct bpf_prog *prog);
include/uapi/linux/bpf.h +11 -0
@@ -5076,6 +5076,16 @@ union bpf_attr {* associated to *xdp_md*, at *offset*. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. +* Return +* 0 on success, or a negative error in case of failure. On error +* *dst* buffer is zeroed out. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5269,6 +5279,7 @@ union bpf_attr {FN (xdp_get_buff_len), \FN (xdp_load_bytes), \FN (xdp_store_bytes), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
kernel/bpf/helpers.c +34 -0
@@ -16,6 +16,7 @@#include <linux/pid_namespace.h> #include <linux/proc_ns.h> #include <linux/security.h> +#include <linux/btf_ids.h> #include "../../lib/kstrtox.h"
@@ -671,6 +672,39 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, + const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) +{ +int ret; + +/* flags is not used yet */ +if (flags) +return -EINVAL ; + +if (size ==0 ) +return 0 ; + + ret =access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +if (ret == size) +return 0 ; + +memset (dst,0 , size); +/* Return -EFAULT for partial read */ +return (ret <0 ) ? ret : -EFAULT ; +} + +const struct bpf_func_proto bpf_copy_from_user_task_proto = { + .func = bpf_copy_from_user_task, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; + BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu) {if (cpu >= nr_cpu_ids)
kernel/trace/bpf_trace.c +2 -0
@@ -1235,6 +1235,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)return &bpf_get_task_stack_proto;case BPF_FUNC_copy_from_user:return prog->aux->sleepable ? &bpf_copy_from_user_proto :NULL ; +case BPF_FUNC_copy_from_user_task: +return prog->aux->sleepable ? &bpf_copy_from_user_task_proto :NULL ;case BPF_FUNC_snprintf_btf:return &bpf_snprintf_btf_proto;case BPF_FUNC_per_cpu_ptr:
tools/include/uapi/linux/bpf.h +11 -0
@@ -5076,6 +5076,16 @@ union bpf_attr {* associated to *xdp_md*, at *offset*. * Return * 0 on success, or a negative error in case of failure. +* +* long bpf_copy_from_user_task(void *dst, u32 size, const void *user_ptr, struct task_struct *tsk, u64 flags) +* Description +* Read *size* bytes from user space address *user_ptr* in *tsk*'s +* address space, and stores the data in *dst*. *flags* is not +* used yet and is provided for future extensibility. This helper +* can only be used by sleepable programs. +* Return +* 0 on success, or a negative error in case of failure. On error +* *dst* buffer is zeroed out. */ #define __BPF_FUNC_MAPPER (FN ) \ FN(unspec), \
@@ -5269,6 +5279,7 @@ union bpf_attr {FN (xdp_get_buff_len), \FN (xdp_load_bytes), \FN (xdp_store_bytes), \ +FN (copy_from_user_task), \/* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper
Signature · 1 line
2.30.2
PATCH v7 1/4 bpf: Add support for bpf iterator programs to use sleepable helpers
In reply to Kenny YuThis patch allows bpf iterator programs to use sleepable helpers by changing bpf_iter_run_prog to use the appropriate synchronization. With sleepable bpf iterator programs, we can no longer use rcu_read_lock() and must use rcu_read_lock_trace() instead to protect the bpf program.
--- kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-)
kernel/bpf/bpf_iter.c +15 -5
@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay
Signature · 1 line
2.30.2
Re: [PATCH v7 bpf-next 1/4] bpf: Add support for bpf iterator programs to use sleepable helpers
Quoted 9 lines · Kenny Yu
This patch allows bpf iterator programs to use sleepable helpers by changing
bpf_iter_run_progto use the appropriate synchronization. With sleepable bpf iterator programs, we can no longer usercu_read_lock()and must usercu_read_lock_trace()instead to protect the bpf program.Signed-off-by Kenny Yu <[email protected]>---
Quoted 44 lines
kernel/bpf/bpf_iter.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-)kernel/bpf/bpf_iter.c +15
-6@@ -5,6 +5,7 @@#include <linux/anon_inodes.h> #include <linux/filter.h> #include <linux/bpf.h> +#include <linux/rcupdate_trace.h> struct bpf_iter_target_info {struct list_head list;@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) {int ret; -rcu_read_lock (); -migrate_disable (); - ret =bpf_prog_run (prog, ctx); -migrate_enable (); -rcu_read_unlock (); +if (prog->aux->sleepable) { +rcu_read_lock_trace (); +migrate_disable (); +might_fault (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock_trace (); + }else { +rcu_read_lock (); +migrate_disable (); + ret =bpf_prog_run (prog, ctx); +migrate_enable (); +rcu_read_unlock (); + }/* bpf program can only return 0 or 1: * 0 : okay -- 2.30.2
Re: [PATCH v7 bpf-next 2/4] bpf: Add bpf_copy_from_user_task() helper
In reply to Kenny YuQuoted 10 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>---
LGTM.
Quoted 7 lines
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/helpers.c | 34 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 5 files changed, 59 insertions(+)
[...]
kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 693
Quoted 7 lines
kernel/bpf/helpers.c +7
-0+ ret =access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +if (ret == size) +return 0 ; + +memset (dst,0 , size); +/* Return -EFAULT for partial read */ +return (ret <0 ) ? ret : -EFAULT ;
nit: unnecessary ()
Quoted 14 lines
kernel/bpf/helpers.c +14
-0+} + +const struct bpf_func_proto bpf_copy_from_user_task_proto = { + .func = bpf_copy_from_user_task, + .gpl_only = false, + .ret_type =RET_INTEGER , + .arg1_type =ARG_PTR_TO_UNINIT_MEM , + .arg2_type =ARG_CONST_SIZE_OR_ZERO , + .arg3_type =ARG_ANYTHING , + .arg4_type =ARG_PTR_TO_BTF_ID , + .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK ], + .arg5_type =ARG_ANYTHING +}; +
[...]
Re: [PATCH v7 bpf-next 2/4] bpf: Add bpf_copy_from_user_task() helper
Quoted 35 lines · Andrii Nakryiko
Quoted 10 lines · Kenny Yu
This adds a helper for bpf programs to read the memory of other tasks.
As an example use case at Meta, we are using a bpf task iterator program and this new helper to print C++ async stack traces for all threads of a given process.
Signed-off-by Kenny Yu <[email protected]>---
LGTM.
Acked-by Andrii Nakryiko <[email protected]>Quoted 7 lines
include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 11 +++++++++++ kernel/bpf/helpers.c | 34 ++++++++++++++++++++++++++++++++++ kernel/trace/bpf_trace.c | 2 ++ tools/include/uapi/linux/bpf.h | 11 +++++++++++ 5 files changed, 59 insertions(+)[...]
kernel/bpf/helpers.c const struct bpf_func_proto bpf_copy_from_user_proto = { · line 693
Quoted 7 lines
kernel/bpf/helpers.c +7
-0+ ret =access_process_vm (tsk, (unsigned long )user_ptr, dst, size,0 ); +if (ret == size) +return 0 ; + +memset (dst,0 , size); +/* Return -EFAULT for partial read */ +return (ret <0 ) ? ret : -EFAULT ;
Quoted 1 line
nit: unnecessary ()
I fixed it while applying.
Also added a few unlikely().
Thanks everyone!
Re: [PATCH v7 bpf-next 0/4] Add bpf_copy_from_user_task helper and sleepable bpf iterator programs
In reply to Kenny YuThis series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Quoted 10 lines · On Mon, 24 Jan 2022 10:53:59 -0800 you wrote:
This patch series makes the following changes:
* Adds a new bpf helperbpf_copy_from_user_taskto read user space memory from a different task.
* Adds the ability to create sleepable bpf iterator programs.As an example of how this will be used, at Meta we are using bpf task iterator programs and this new bpf helper to read C++ async stack traces of a running process for debugging C++ binaries in production.
[...]
Here is the summary with links:
- [v7,bpf-next,1/4] bpf: Add support for bpf iterator programs to use sleepable helpers
https://git.kernel.org/bpf/bpf-next/c/b77fb25dcb34 - [v7,bpf-next,2/4] bpf: Add bpf_copy_from_user_task() helper https://git.kernel.org/bpf/bpf-next/c/376040e47334 - [v7,bpf-next,3/4] libbpf: Add "iter.s" section for sleepable bpf iterator programs https://git.kernel.org/bpf/bpf-next/c/a8b77f7463a5 - [v7,bpf-next,4/4] selftests/bpf: Add test for sleepable bpf iterator programs https://git.kernel.org/bpf/bpf-next/c/45105c2eb751
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html