Allow struct_ops maps with a large number of programs
Subsystems (8)
v3 cover Allow struct_ops maps with a large number of programs
The BPF struct_ops previously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.
The variable st_map->image has been changed to st_map->image_pages, and its type has been changed to an array of pointers to buffers of PAGE_SIZE. Additional pages are allocated when all existing pages are exhausted.
The test case loads a struct_ops maps having 40 programs. Their trampolines takes about 6.6k+ bytes over 1.5 pages on x86.
---
Major differences from v2:
- Move image buffer allocation to bpf_struct_ops_prepare_trampoline().
Major differences from v1:
- Always free pages if failing to update.
- Allocate 8 pages at most.
v2: https://lore.kernel.org/all/[email protected]/
v1: https://lore.kernel.org/all/[email protected]/
Kui-Feng Lee (3):
bpf, net: validate struct_ops when updating value.
bpf: struct_ops supports more than one page for trampolines. selftests/bpf: Test struct_ops maps with a large number of program
links.
include/linux/bpf.h | 3 +- kernel/bpf/bpf_struct_ops.c | 126 ++++++++++++------ net/bpf/bpf_dummy_struct_ops.c | 12 +- net/ipv4/tcp_cong.c | 6 +- .../selftests/bpf/bpf_testmod/bpf_testmod.h | 44 ++++++ .../prog_tests/test_struct_ops_multi_pages.c | 30 +++++ .../bpf/progs/struct_ops_multi_pages.c | 102 ++++++++++++++ 7 files changed, 266 insertions(+), 57 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c
Signature · 1 line
2.34.1
PATCH v3 1/3 bpf, net: validate struct_ops when updating value.
Perform all validations when updating values of struct_ops maps. Doing validation in st_ops->reg() and st_ops->update() is not necessary anymore. However, tcp_register_congestion_control() has been called in various places. It still needs to do validations.
--- kernel/bpf/bpf_struct_ops.c | 11 ++++++----- net/ipv4/tcp_cong.c | 6 +----- 2 files changed, 7 insertions(+), 10 deletions(-)
kernel/bpf/bpf_struct_ops.c +6 -5
@@ -672,13 +672,14 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, *(unsigned long *)(udata + moff) = prog->aux->id; } +if (st_ops->validate) { + err = st_ops->validate (kdata); +if (err) + goto reset_unlock; + } +if (st_map->map.map_flags &BPF_F_LINK ) { err =0 ; -if (st_ops->validate) { - err = st_ops->validate (kdata); -if (err) - goto reset_unlock; - }arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE );/* Let bpf_link handle registration & unregistration. *
net/ipv4/tcp_cong.c +1 -5
@@ -146,11 +146,7 @@ EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control);int tcp_update_congestion_control (struct tcp_congestion_ops *ca,struct tcp_congestion_ops *old_ca) {struct tcp_congestion_ops *existing; -int ret; - - ret =tcp_validate_congestion_control (ca); -if (ret) -return ret; +int ret =0 ; ca->key =jhash (ca->name,sizeof (ca->name),strlen (ca->name));
Signature · 1 line
2.34.1
PATCH v3 2/3 bpf: struct_ops supports more than one page for trampolines.
In reply to Kui-Feng LeeThe BPF struct_ops previously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.
The variable st_map->image has been changed to st_map->image_pages, and its type has been changed to an array of pointers to buffers of PAGE_SIZE. Every struct_ops map can have MAX_IMAGE_PAGES (8) pages for trampolines at most.
--- include/linux/bpf.h | 3 +- kernel/bpf/bpf_struct_ops.c | 115 ++++++++++++++++++++++----------- net/bpf/bpf_dummy_struct_ops.c | 12 ++-- 3 files changed, 83 insertions(+), 47 deletions(-)
include/linux/bpf.h +2 -1
@@ -1763,7 +1763,8 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,struct bpf_tramp_link *link, const struct btf_func_model *model, void *stub_func, - void *image, void *image_end); + void **image, u32 *image_off, + bool allow_alloc);static inline bool bpf_try_module_get (const void *data,struct module *owner) {if (owner ==BPF_MODULE_OWNER )
kernel/bpf/bpf_struct_ops.c +76 -39
220 lines not on this page · raw message
net/bpf/bpf_dummy_struct_ops.c +5 -7
@@ -91,6 +91,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,struct bpf_tramp_link *link =NULL ;void *image =NULL ;unsigned int op_idx; +u32 image_off =0 ;int prog_ret;s32 type_id;int err;
@@ -114,12 +115,6 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, goto out; } - image =arch_alloc_bpf_trampoline (PAGE_SIZE ); -if (!image) { - err = -ENOMEM ; - goto out; - } - link =kzalloc (sizeof (*link),GFP_USER );if (!link) { err = -ENOMEM ;
@@ -133,7 +128,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err =bpf_struct_ops_prepare_trampoline (tlinks, link, &st_ops->func_models[op_idx], &dummy_ops_test_ret_function, - image, image +PAGE_SIZE ); + &image, &image_off, + true);if (err <0 ) goto out;
@@ -147,6 +143,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -EFAULT ; out:kfree (args); +if (image) +bpf_jit_uncharge_modmem (PAGE_SIZE );arch_free_bpf_trampoline (image,PAGE_SIZE );if (link)bpf_link_put (&link->link);
Signature · 1 line
2.34.1
Martin KaFai Lau
reviewerRe: [PATCH bpf-next v3 2/3] bpf: struct_ops supports more than one page for trampolines.
Quoted 13 lines · Kui-Feng Lee
kernel/bpf/bpf_struct_ops.c +13
-0+static void bpf_struct_ops_map_free_image (struct bpf_struct_ops_map *st_map) +{ +int i; +void *image; + +bpf_jit_uncharge_modmem (PAGE_SIZE * st_map->image_pages_cnt); +for (i =0 ; i < st_map->image_pages_cnt; i++) { + image = st_map->image_pages[i]; +arch_free_bpf_trampoline (image,PAGE_SIZE ); + } + st_map->image_pages_cnt =0 ; +} +
[ ... ]
Quoted 11 lines
net/bpf/bpf_dummy_struct_ops.c +2
-1@@ -133,7 +128,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err =bpf_struct_ops_prepare_trampoline (tlinks, link, &st_ops->func_models[op_idx], &dummy_ops_test_ret_function, - image, image +PAGE_SIZE ); + &image, &image_off, + true);if (err <0 ) goto out;
net/bpf/bpf_dummy_struct_ops.c int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, · line 148
Quoted 6 lines
net/bpf/bpf_dummy_struct_ops.c +2
-0@@ -147,6 +143,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -EFAULT ; out:kfree (args); +if (image) +bpf_jit_uncharge_modmem (PAGE_SIZE );arch_free_bpf_trampoline (image,PAGE_SIZE );
It seems my last reply on v2 has crossed over with v3.
The bpf_struct_ops_free_trampoline() highlighted in my last reply should address your concern in v2 that the caller needs to remember the bpf_jit_uncharge_modmem here.
I think the trampoline alloc(aka prepare here)/free pair that you also suggested in v2 discussion is a nice match here and work as a charge+alloc and uncharge+free pair.
Quoted 2 lines
net/bpf/bpf_dummy_struct_ops.c +0
-0if (link)bpf_link_put (&link->link);
Re: [PATCH bpf-next v3 2/3] bpf: struct_ops supports more than one page for trampolines.
Quoted 39 lines · Martin KaFai Lau
Quoted 14 lines · Kui-Feng Lee
+static void
bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)kernel/bpf/bpf_struct_ops.c +12
-0+{ + int i; + void *image; + + bpf_jit_uncharge_modmem(PAGE_SIZE * st_map->image_pages_cnt); + for (i = 0; i < st_map->image_pages_cnt; i++) { + image = st_map->image_pages[i]; + arch_free_bpf_trampoline(image, PAGE_SIZE); + } + st_map->image_pages_cnt = 0; +} +[ ... ]
Quoted 13 lines
net/bpf/bpf_dummy_struct_ops.c +2
-1@@ -133,7 +128,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog,const union bpf_attr *kattr, err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[op_idx], &dummy_ops_test_ret_function, - image, image + PAGE_SIZE); + &image, &image_off, + true); if (err < 0) goto out;@@ -147,6 +143,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog,const union bpf_attr *kattr, err = -EFAULT;Quoted 5 lines
out:
kfree(args);
+ if (image)
+bpf_jit_uncharge_modmem(PAGE_SIZE);
arch_free_bpf_trampoline(image, PAGE_SIZE);It seems my last reply on v2 has crossed over with v3.
Quoted 7 lines
The
bpf_struct_ops_free_trampoline()highlighted in my last reply should address your concern in v2 that the caller needs to remember thebpf_jit_uncharge_modmemhere.I think the trampoline alloc(aka prepare here)/free pair that you also suggested in v2 discussion is a nice match here and work as a charge+alloc and uncharge+free pair.
sure
Quoted 4 lines
Quoted 3 lines
if (link)
bpf_link_put(&link->link);
PATCH v3 3/3 selftests/bpf: Test struct_ops maps with a large number of program links.
In reply to Kui-Feng LeeCreate and load a struct_ops map with a large number of programs to generate trampolines taking a size over multiple pages. The map includes 40 programs. Their trampolines takes 6.6k+, more than 1.5 pages, on x86.
--- .../selftests/bpf/bpf_testmod/bpf_testmod.h | 44 ++++++++ .../prog_tests/test_struct_ops_multi_pages.c | 30 ++++++ .../bpf/progs/struct_ops_multi_pages.c | 102 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +44 -0
@@ -35,6 +35,50 @@ struct bpf_testmod_ops {void (*test_2)(int a,int b);/* Used to test nullable arguments. */ int (*test_maybe_null)(int dummy,struct task_struct *task); + +/* The following pointers are used to test the maps having multiple +* pages of trampolines. +*/ +int (*tramp_1)(int value); +int (*tramp_2)(int value); +int (*tramp_3)(int value); +int (*tramp_4)(int value); +int (*tramp_5)(int value); +int (*tramp_6)(int value); +int (*tramp_7)(int value); +int (*tramp_8)(int value); +int (*tramp_9)(int value); +int (*tramp_10)(int value); +int (*tramp_11)(int value); +int (*tramp_12)(int value); +int (*tramp_13)(int value); +int (*tramp_14)(int value); +int (*tramp_15)(int value); +int (*tramp_16)(int value); +int (*tramp_17)(int value); +int (*tramp_18)(int value); +int (*tramp_19)(int value); +int (*tramp_20)(int value); +int (*tramp_21)(int value); +int (*tramp_22)(int value); +int (*tramp_23)(int value); +int (*tramp_24)(int value); +int (*tramp_25)(int value); +int (*tramp_26)(int value); +int (*tramp_27)(int value); +int (*tramp_28)(int value); +int (*tramp_29)(int value); +int (*tramp_30)(int value); +int (*tramp_31)(int value); +int (*tramp_32)(int value); +int (*tramp_33)(int value); +int (*tramp_34)(int value); +int (*tramp_35)(int value); +int (*tramp_36)(int value); +int (*tramp_37)(int value); +int (*tramp_38)(int value); +int (*tramp_39)(int value); + int (*tramp_40)(int value); };#endif /* _BPF_TESTMOD_H */
tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c new file +30 -0
@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_multi_pages.skel.h" + +static void do_struct_ops_multi_pages (void ) +{ +struct struct_ops_multi_pages *skel; +struct bpf_link *link; + +/* The size of all trampolines of skel->maps.multi_pages should be +* over 1 page (at least for x86). +*/ + skel =struct_ops_multi_pages__open_and_load (); +if (!ASSERT_OK_PTR (skel,"struct_ops_multi_pages_open_and_load" )) +return ; + + link =bpf_map__attach_struct_ops (skel->maps.multi_pages); +ASSERT_OK_PTR (link,"attach_multi_pages" ); + +bpf_link__destroy (link); +struct_ops_multi_pages__destroy (skel); +} + +void test_struct_ops_multi_pages (void ) +{ +if (test__start_subtest ("multi_pages" )) +do_struct_ops_multi_pages (); +}
tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c new file +102 -0
102 lines not on this page · raw message
Signature · 1 line
2.34.1