bpf: Add open coded version of kmem_cache iterator
Subsystems (5)
PATCH v2 1/2 bpf: Add open coded version of kmem_cache iterator
Add a new open coded iterator for kmem_cache which can be called from a BPF program like below. It doesn't take any argument and traverses all kmem_cache entries.
struct kmem_cache *pos;
bpf_for_each(kmem_cache, pos) {
...
}As it needs to grab slab_mutex, it should be called from sleepable BPF programs only.
Also update the existing iterator code to use the open coded version internally as suggested by Andrii.
---
v2)
* prevent restart after the last element (Martin)
* update existing code to use the open coded version (Andrii)
kernel/bpf/helpers.c | 3 + kernel/bpf/kmem_cache_iter.c | 151 +++++++++++++++++++++++++---------- 2 files changed, 110 insertions(+), 44 deletions(-)
kernel/bpf/helpers.c +3 -0
@@ -3112,6 +3112,9 @@ BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_get_kmem_cache) +BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)BTF_KFUNCS_END (common_btf_ids) static const struct btf_kfunc_id_set common_kfunc_set = {
kernel/bpf/kmem_cache_iter.c +107 -44
200 lines not on this page · raw message
Signature · 1 line
2.47.0.105.g07ac214952-goog
PATCH v2 2/2 selftests/bpf: Add a test for open coded kmem_cache iter
The new subtest is attached to sleepable fentry of syncfs() syscall. It iterates the kmem_cache using bpf_for_each loop and count the number of entries. Finally it checks it with the number of entries from the regular iterator.
$ ./vmtest.sh -- ./test_progs -t kmem_cache_iter ... #130/1 kmem_cache_iter/check_task_struct:OK #130/2 kmem_cache_iter/check_slabinfo:OK #130/3 kmem_cache_iter/open_coded_iter:OK #130 kmem_cache_iter:OK Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILED
Also simplify the code by using attach routine of the skeleton.
---
v2)
* remove unnecessary detach (Martin)
* check pid in syncfs to prevent surprise (Martin)
* remove unnecessary local variable (Andrii)
.../testing/selftests/bpf/bpf_experimental.h | 6 ++++ .../bpf/prog_tests/kmem_cache_iter.c | 28 +++++++++++-------- .../selftests/bpf/progs/kmem_cache_iter.c | 28 +++++++++++++++++++ 3 files changed, 50 insertions(+), 12 deletions(-)
tools/testing/selftests/bpf/bpf_experimental.h +6 -0
@@ -582,4 +582,10 @@ extern int bpf_wq_set_callback_impl(struct bpf_wq *wq, unsigned int flags__k, void *aux__ign) __ksym;#define bpf_wq_set_callback (timer, cb, flags) \ bpf_wq_set_callback_impl(timer, cb, flags, NULL) + +struct bpf_iter_kmem_cache ; +extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym; +extern struct kmem_cache *bpf_iter_kmem_cache_next (struct bpf_iter_kmem_cache *it)__weak __ksym; +extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym; +#endif
tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +16 -12
@@ -68,12 +68,20 @@ static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel)fclose (fp); } +static void subtest_kmem_cache_iter_open_coded (struct kmem_cache_iter *skel) +{ + skel->bss->tgid =getpid (); + +/* To trigger the open coded iterator attached to the syscall */ +syncfs (0 ); + +/* It should be same as we've seen from the explicit iterator */ +ASSERT_EQ (skel->bss->open_coded_seen, skel->bss->kmem_cache_seen,"open_code_seen_eq" ); +} +void test_kmem_cache_iter (void ) { -DECLARE_LIBBPF_OPTS (bpf_iter_attach_opts, opts);struct kmem_cache_iter *skel =NULL ; -union bpf_iter_link_info linfo = {}; -struct bpf_link *link;char buf[256 ];int iter_fd;
@@ -81,16 +89,12 @@ void test_kmem_cache_iter(void)if (!ASSERT_OK_PTR (skel,"kmem_cache_iter__open_and_load" ))return ; - opts.link_info = &linfo; - opts.link_info_len =sizeof (linfo); - - link =bpf_program__attach_iter (skel->progs.slab_info_collector, &opts); -if (!ASSERT_OK_PTR (link,"attach_iter" )) +if (!ASSERT_OK (kmem_cache_iter__attach (skel),"skel_attach" )) goto destroy; - iter_fd =bpf_iter_create (bpf_link__fd (link)); + iter_fd =bpf_iter_create (bpf_link__fd (skel->links.slab_info_collector));if (!ASSERT_GE (iter_fd,0 ,"iter_create" )) - goto free_link; + goto destroy;memset (buf,0 ,sizeof (buf));while (read (iter_fd, buf,sizeof (buf) >0 )) {
@@ -105,11 +109,11 @@ void test_kmem_cache_iter(void)subtest_kmem_cache_iter_check_task_struct (skel);if (test__start_subtest ("check_slabinfo" ))subtest_kmem_cache_iter_check_slabinfo (skel); +if (test__start_subtest ("open_coded_iter" )) +subtest_kmem_cache_iter_open_coded (skel);close (iter_fd); -free_link: -bpf_link__destroy (link); destroy:kmem_cache_iter__destroy (skel); }
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +28 -0
@@ -2,6 +2,8 @@/* Copyright (c) 2024 Google */ #include "bpf_iter.h" +#include "bpf_experimental.h" +#include "bpf_misc.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h>
@@ -30,9 +32,12 @@ struct {extern struct kmem_cache *bpf_get_kmem_cache (u64 addr) __ksym; +unsigned int tgid; +/* Result, will be checked by userspace */ int task_struct_found;int kmem_cache_seen; +int open_coded_seen; SEC("iter/kmem_cache")int slab_info_collector (struct bpf_iter__kmem_cache *ctx)
@@ -85,3 +90,26 @@ int BPF_PROG(check_task_struct) task_struct_found =-2 ;return 0 ; } + +SEC("fentry.s/" SYS_PREFIX "sys_syncfs") +int open_coded_iter (const void *ctx) +{ +struct kmem_cache *s; + +if (tgid !=bpf_get_current_pid_tgid () >>32 ) +return 0 ; + +bpf_for_each (kmem_cache, s) { +struct kmem_cache_result *r; + + r =bpf_map_lookup_elem (&slab_result, &open_coded_seen); +if (!r) +break ; + + open_coded_seen++; + +if (r->obj_size != s->size) +break ; + } +return 0 ; +}
Signature · 1 line
2.47.0.105.g07ac214952-goog
Re: [PATCH v2 bpf-next 2/2] selftests/bpf: Add a test for open coded kmem_cache iter
Quoted 133 lines · Namhyung Kim
The new subtest is attached to sleepable fentry of
syncfs()syscall. It iterates thekmem_cacheusingbpf_for_eachloop and count the number of entries. Finally it checks it with the number of entries from the regular iterator.$ ./vmtest.sh -- ./test_progs -t kmem_cache_iter ... #130/1 kmem_cache_iter/check_task_struct:OK #130/2 kmem_cache_iter/check_slabinfo:OK #130/3 kmem_cache_iter/open_coded_iter:OK #130 kmem_cache_iter:OK Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILEDAlso simplify the code by using attach routine of the skeleton.
Signed-off-by Namhyung Kim <[email protected]>---
v2)
* remove unnecessary detach (Martin)
* check pid in syncfs to prevent surprise (Martin)
* remove unnecessary local variable (Andrii).../testing/selftests/bpf/bpf_experimental.h | 6 ++++ .../bpf/prog_tests/kmem_cache_iter.c | 28 +++++++++++-------- .../selftests/bpf/progs/kmem_cache_iter.c | 28 +++++++++++++++++++ 3 files changed, 50 insertions(+), 12 deletions(-)tools/testing/selftests/bpf/bpf_experimental.h +6
-0@@ -582,4 +582,10 @@ extern int bpf_wq_set_callback_impl(struct bpf_wq *wq, unsigned int flags__k, void *aux__ign) __ksym;#define bpf_wq_set_callback (timer, cb, flags) \ bpf_wq_set_callback_impl(timer, cb, flags, NULL) + +struct bpf_iter_kmem_cache ; +extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym; +extern struct kmem_cache *bpf_iter_kmem_cache_next (struct bpf_iter_kmem_cache *it)__weak __ksym; +extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym; +#endif tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +16
-12@@ -68,12 +68,20 @@ static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel)fclose (fp); } +static void subtest_kmem_cache_iter_open_coded (struct kmem_cache_iter *skel) +{ + skel->bss->tgid =getpid (); + +/* To trigger the open coded iterator attached to the syscall */ +syncfs (0 ); + +/* It should be same as we've seen from the explicit iterator */ +ASSERT_EQ (skel->bss->open_coded_seen, skel->bss->kmem_cache_seen,"open_code_seen_eq" ); +} +void test_kmem_cache_iter (void ) { -DECLARE_LIBBPF_OPTS (bpf_iter_attach_opts, opts);struct kmem_cache_iter *skel =NULL ; -union bpf_iter_link_info linfo = {}; -struct bpf_link *link;char buf[256 ];int iter_fd;@@ -81,16 +89,12 @@ void test_kmem_cache_iter(void)if (!ASSERT_OK_PTR (skel,"kmem_cache_iter__open_and_load" ))return ; - opts.link_info = &linfo; - opts.link_info_len =sizeof (linfo); - - link =bpf_program__attach_iter (skel->progs.slab_info_collector, &opts); -if (!ASSERT_OK_PTR (link,"attach_iter" )) +if (!ASSERT_OK (kmem_cache_iter__attach (skel),"skel_attach" )) goto destroy; - iter_fd =bpf_iter_create (bpf_link__fd (link)); + iter_fd =bpf_iter_create (bpf_link__fd (skel->links.slab_info_collector));if (!ASSERT_GE (iter_fd,0 ,"iter_create" )) - goto free_link; + goto destroy;memset (buf,0 ,sizeof (buf));while (read (iter_fd, buf,sizeof (buf) >0 )) {@@ -105,11 +109,11 @@ void test_kmem_cache_iter(void)subtest_kmem_cache_iter_check_task_struct (skel);if (test__start_subtest ("check_slabinfo" ))subtest_kmem_cache_iter_check_slabinfo (skel); +if (test__start_subtest ("open_coded_iter" )) +subtest_kmem_cache_iter_open_coded (skel);close (iter_fd); -free_link: -bpf_link__destroy (link); destroy:kmem_cache_iter__destroy (skel); }tools/testing/selftests/bpf/progs/kmem_cache_iter.c +5
-0@@ -2,6 +2,8 @@/* Copyright (c) 2024 Google */ #include "bpf_iter.h" +#include "bpf_experimental.h" +#include "bpf_misc.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> @@ -30,9 +32,12 @@ struct {extern struct kmem_cache *bpf_get_kmem_cache (u64 addr) __ksym; +unsigned int tgid; +/* Result, will be checked by userspace */ int task_struct_found;int kmem_cache_seen; +int open_coded_seen; SEC("iter/kmem_cache")int slab_info_collector (struct bpf_iter__kmem_cache *ctx)@@ -85,3 +90,26 @@ int BPF_PROG(check_task_struct) task_struct_found =-2 ;
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 100
Quoted 10 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +8
-0return 0 ; } + +SEC("fentry.s/" SYS_PREFIX "sys_syncfs") +int open_coded_iter (const void *ctx) +{ +struct kmem_cache *s; + +if (tgid !=bpf_get_current_pid_tgid () >>32 ) +return 0 ;
Pls use syscall prog type and prog_run() it. No need to attach to exotic syscalls and filter by pid.
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 112
Quoted 12 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +12
-0+ +bpf_for_each (kmem_cache, s) { +struct kmem_cache_result *r; + + r =bpf_map_lookup_elem (&slab_result, &open_coded_seen); +if (!r) +break ; + + open_coded_seen++; + +if (r->obj_size != s->size) +break ;
The order of 'if' and ++ should probably be changed ? Otherwise the last object isn't sufficiently checked.
pw-bot: cr
Re: [PATCH v2 bpf-next 2/2] selftests/bpf: Add a test for open coded kmem_cache iter
Hello,
Quoted 139 lines · Alexei Starovoitov
Quoted 138 lines · Namhyung Kim
The new subtest is attached to sleepable fentry of
syncfs()syscall. It iterates thekmem_cacheusingbpf_for_eachloop and count the number of entries. Finally it checks it with the number of entries from the regular iterator.$ ./vmtest.sh -- ./test_progs -t kmem_cache_iter ... #130/1 kmem_cache_iter/check_task_struct:OK #130/2 kmem_cache_iter/check_slabinfo:OK #130/3 kmem_cache_iter/open_coded_iter:OK #130 kmem_cache_iter:OK Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILEDAlso simplify the code by using attach routine of the skeleton.
Signed-off-by Namhyung Kim <[email protected]>---
v2)
* remove unnecessary detach (Martin)
* check pid in syncfs to prevent surprise (Martin)
* remove unnecessary local variable (Andrii).../testing/selftests/bpf/bpf_experimental.h | 6 ++++ .../bpf/prog_tests/kmem_cache_iter.c | 28 +++++++++++-------- .../selftests/bpf/progs/kmem_cache_iter.c | 28 +++++++++++++++++++ 3 files changed, 50 insertions(+), 12 deletions(-)tools/testing/selftests/bpf/bpf_experimental.h +6
-0@@ -582,4 +582,10 @@ extern int bpf_wq_set_callback_impl(struct bpf_wq *wq, unsigned int flags__k, void *aux__ign) __ksym;#define bpf_wq_set_callback (timer, cb, flags) \ bpf_wq_set_callback_impl(timer, cb, flags, NULL) + +struct bpf_iter_kmem_cache ; +extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym; +extern struct kmem_cache *bpf_iter_kmem_cache_next (struct bpf_iter_kmem_cache *it)__weak __ksym; +extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym; +#endif tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +16
-12@@ -68,12 +68,20 @@ static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel)fclose (fp); } +static void subtest_kmem_cache_iter_open_coded (struct kmem_cache_iter *skel) +{ + skel->bss->tgid =getpid (); + +/* To trigger the open coded iterator attached to the syscall */ +syncfs (0 ); + +/* It should be same as we've seen from the explicit iterator */ +ASSERT_EQ (skel->bss->open_coded_seen, skel->bss->kmem_cache_seen,"open_code_seen_eq" ); +} +void test_kmem_cache_iter (void ) { -DECLARE_LIBBPF_OPTS (bpf_iter_attach_opts, opts);struct kmem_cache_iter *skel =NULL ; -union bpf_iter_link_info linfo = {}; -struct bpf_link *link;char buf[256 ];int iter_fd;@@ -81,16 +89,12 @@ void test_kmem_cache_iter(void)if (!ASSERT_OK_PTR (skel,"kmem_cache_iter__open_and_load" ))return ; - opts.link_info = &linfo; - opts.link_info_len =sizeof (linfo); - - link =bpf_program__attach_iter (skel->progs.slab_info_collector, &opts); -if (!ASSERT_OK_PTR (link,"attach_iter" )) +if (!ASSERT_OK (kmem_cache_iter__attach (skel),"skel_attach" )) goto destroy; - iter_fd =bpf_iter_create (bpf_link__fd (link)); + iter_fd =bpf_iter_create (bpf_link__fd (skel->links.slab_info_collector));if (!ASSERT_GE (iter_fd,0 ,"iter_create" )) - goto free_link; + goto destroy;memset (buf,0 ,sizeof (buf));while (read (iter_fd, buf,sizeof (buf) >0 )) {@@ -105,11 +109,11 @@ void test_kmem_cache_iter(void)subtest_kmem_cache_iter_check_task_struct (skel);if (test__start_subtest ("check_slabinfo" ))subtest_kmem_cache_iter_check_slabinfo (skel); +if (test__start_subtest ("open_coded_iter" )) +subtest_kmem_cache_iter_open_coded (skel);close (iter_fd); -free_link: -bpf_link__destroy (link); destroy:kmem_cache_iter__destroy (skel); }tools/testing/selftests/bpf/progs/kmem_cache_iter.c +8
-0@@ -2,6 +2,8 @@/* Copyright (c) 2024 Google */ #include "bpf_iter.h" +#include "bpf_experimental.h" +#include "bpf_misc.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> @@ -30,9 +32,12 @@ struct {extern struct kmem_cache *bpf_get_kmem_cache (u64 addr) __ksym; +unsigned int tgid; +/* Result, will be checked by userspace */ int task_struct_found;int kmem_cache_seen; +int open_coded_seen; SEC("iter/kmem_cache")int slab_info_collector (struct bpf_iter__kmem_cache *ctx)@@ -85,3 +90,26 @@ int BPF_PROG(check_task_struct) task_struct_found =-2 ;return 0 ; } + +SEC("fentry.s/" SYS_PREFIX "sys_syncfs") +int open_coded_iter (const void *ctx)
Quoted 8 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 100
Quoted 5 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +5
-0+{ +struct kmem_cache *s; + +if (tgid !=bpf_get_current_pid_tgid () >>32 ) +return 0 ;Pls use syscall prog type and
prog_run()it. No need to attach to exotic syscalls and filter by pid.
Sure, will update in v3.
Quoted 16 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 112
Quoted 13 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +12
-0+ +bpf_for_each (kmem_cache, s) { +struct kmem_cache_result *r; + + r =bpf_map_lookup_elem (&slab_result, &open_coded_seen); +if (!r) +break ; + + open_coded_seen++; + +if (r->obj_size != s->size) +break ;The order of 'if' and ++ should probably be changed ? Otherwise the last object isn't sufficiently checked.
I don't think so. The last element should be an actual slab cache and then the iterator will return NULL to break the loop. I don't expect it will hit the if statement.
Thanks,
Namhyung
Re: [PATCH v2 bpf-next 2/2] selftests/bpf: Add a test for open coded kmem_cache iter
Quoted 22 lines · Namhyung Kim
Hello,
Quoted 19 lines · Alexei Starovoitov
Quoted 18 lines · Namhyung Kim
The new subtest is attached to sleepable fentry of
syncfs()syscall. It iterates thekmem_cacheusingbpf_for_eachloop and count the number of entries. Finally it checks it with the number of entries from the regular iterator.$ ./vmtest.sh -- ./test_progs -t kmem_cache_iter ... #130/1 kmem_cache_iter/check_task_struct:OK #130/2 kmem_cache_iter/check_slabinfo:OK #130/3 kmem_cache_iter/open_coded_iter:OK #130 kmem_cache_iter:OK Summary: 1/3 PASSED, 0 SKIPPED, 0 FAILEDAlso simplify the code by using attach routine of the skeleton.
Signed-off-by Namhyung Kim <[email protected]>---
[SNIP]
Quoted 30 lines
Quoted 10 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 100
Quoted 7 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +7
-0+SEC("fentry.s/" SYS_PREFIX "sys_syncfs") +int open_coded_iter (const void *ctx) +{ +struct kmem_cache *s; + +if (tgid !=bpf_get_current_pid_tgid () >>32 ) +return 0 ;Pls use syscall prog type and
prog_run()it. No need to attach to exotic syscalls and filter by pid.Sure, will update in v3.
Quoted 16 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c int BPF_PROG(check_task_struct) · line 112
Quoted 13 lines
tools/testing/selftests/bpf/progs/kmem_cache_iter.c +12
-0+ +bpf_for_each (kmem_cache, s) { +struct kmem_cache_result *r; + + r =bpf_map_lookup_elem (&slab_result, &open_coded_seen); +if (!r) +break ; + + open_coded_seen++; + +if (r->obj_size != s->size) +break ;The order of 'if' and ++ should probably be changed ? Otherwise the last object isn't sufficiently checked.
Quoted 3 lines
I don't think so. The last element should be an actual slab cache and then the iterator will return NULL to break the loop. I don't expect it will hit the if statement.
Oh, it seems you meant checking the obj_size. Ok then, I can move the increment after the check.
Thanks,
Namhyung