Allow struct_ops maps with a large number of programs
Subsystems (4)
cover Allow struct_ops maps with a large number of programs
From: Kui-Feng Lee <[email protected]>
The BPF struct_ops previously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.
The variable st_map->image has been changed to st_map->image_pages, and its type has been changed to an array of pointers to buffers of PAGE_SIZE. The array is dynamically resized and additional pages are allocated when all existing pages are exhausted.
The test case loads a struct_ops maps having 40 programs. Their trampolines takes about 6.6k+ bytes over 1.5 pages on x86.
Kui-Feng Lee (2):
bpf: struct_ops supports more than one page for trampolines. selftests/bpf: Test struct_ops maps with a large number of program
links.
kernel/bpf/bpf_struct_ops.c | 99 +++++++++++++---- .../selftests/bpf/bpf_testmod/bpf_testmod.h | 44 ++++++++ .../prog_tests/test_struct_ops_multi_pages.c | 24 +++++ .../bpf/progs/struct_ops_multi_pages.c | 102 ++++++++++++++++++ 4 files changed, 250 insertions(+), 19 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c
Signature · 1 line
2.34.1
PATCH 1/2 bpf: struct_ops supports more than one page for trampolines.
From: Kui-Feng Lee <[email protected]>
The BPF struct_ops previously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.
The variable st_map->image has been changed to st_map->image_pages, and its type has been changed to an array of pointers to buffers of PAGE_SIZE. The array is dynamically resized and additional pages are allocated when all existing pages are exhausted.
--- kernel/bpf/bpf_struct_ops.c | 99 ++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 19 deletions(-)
kernel/bpf/bpf_struct_ops.c +80 -19
@@ -30,12 +30,11 @@ struct bpf_struct_ops_map {*/ struct bpf_link **links;u32 links_cnt; -/* image is a page that has all the trampolines +u32 image_pages_cnt; +/* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. -* A PAGE_SIZE "image" is enough to store all trampoline for -* "links[]". */ -void *image; +void **image_pages;/* The owner moduler's btf. */ struct btf *btf;/* uvalue->data stores the kernel struct
@@ -535,7 +534,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,void *udata, *kdata;int prog_fd, err;void *image, *image_end; -u32 i; +void **image_pages; +u32 i, next_page =0 ;if (flags)return -EINVAL ;
@@ -573,8 +573,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, udata = &uvalue->data; kdata = &kvalue->data; - image = st_map->image; - image_end = st_map->image +PAGE_SIZE ; + image = st_map->image_pages[next_page++]; + image_end = image +PAGE_SIZE ; module_type =btf_type_by_id (btf_vmlinux, st_ops_ids[IDX_MODULE_ID ]);for_each_member (i, t, member) {
@@ -657,6 +657,43 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, &st_ops->func_models[i], *(void **)(st_ops->cfi_stubs + moff), image, image_end); +if (err == -E2BIG ) { +/* Use an additional page to try again. +* +* It may reuse pages allocated for the previous +* failed calls. +*/ +if (next_page >= st_map->image_pages_cnt) { +/* Allocate an additional page */ + image_pages =krealloc (st_map->image_pages, + (st_map->image_pages_cnt +1 ) *sizeof (void *), +GFP_KERNEL ); +if (!image_pages) { + err = -ENOMEM ; + goto reset_unlock; + } + st_map->image_pages = image_pages; + + err =bpf_jit_charge_modmem (PAGE_SIZE ); +if (err) + goto reset_unlock; + + image =arch_alloc_bpf_trampoline (PAGE_SIZE ); +if (!image) { +bpf_jit_uncharge_modmem (PAGE_SIZE ); + err = -ENOMEM ; + goto reset_unlock; + } + st_map->image_pages[st_map->image_pages_cnt++] = image; + } + image = st_map->image_pages[next_page++]; + image_end = image +PAGE_SIZE ; + + err =bpf_struct_ops_prepare_trampoline (tlinks, link, + &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), + image, image_end); + }if (err <0 ) goto reset_unlock;
@@ -667,6 +704,18 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, *(unsigned long *)(udata + moff) = prog->aux->id; } +while (next_page < st_map->image_pages_cnt) { +/* Free unused pages +* +* The value can not be updated anymore if the value is not +* rejected by st_ops->validate() or st_ops->reg(). So, +* there is no reason to keep the unused pages. +*/ +bpf_jit_uncharge_modmem (PAGE_SIZE ); + image = st_map->image_pages[--st_map->image_pages_cnt]; +arch_free_bpf_trampoline (image,PAGE_SIZE ); + } +if (st_map->map.map_flags &BPF_F_LINK ) { err =0 ;if (st_ops->validate) {
@@ -674,7 +723,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,if (err) goto reset_unlock; } -arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_protect_bpf_trampoline (st_map->image_pages[i], +PAGE_SIZE );/* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem().
@@ -683,7 +734,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } -arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_protect_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE ); err = st_ops->reg (kdata);if (likely (!err)) {/* This refcnt increment on the map here after
@@ -706,7 +758,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,* there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ -arch_unprotect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_unprotect_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE ); reset_unlock:bpf_struct_ops_map_put_progs (st_map);
@@ -771,14 +824,15 @@ static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,static void __bpf_struct_ops_map_free (struct bpf_map *map) {struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; +int i;if (st_map->links)bpf_struct_ops_map_put_progs (st_map);bpf_map_area_free (st_map->links); -if (st_map->image) { -arch_free_bpf_trampoline (st_map->image,PAGE_SIZE ); -bpf_jit_uncharge_modmem (PAGE_SIZE ); - } +for (i =0 ; i < st_map->image_pages_cnt; i++) +arch_free_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE ); +bpf_jit_uncharge_modmem (PAGE_SIZE * st_map->image_pages_cnt); +kfree (st_map->image_pages);bpf_map_area_free (st_map->uvalue);bpf_map_area_free (st_map); }
@@ -888,20 +942,27 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) st_map->st_ops_desc = st_ops_desc; map = &st_map->map; + st_map->image_pages =kcalloc (1 ,sizeof (void *),GFP_KERNEL ); +if (!st_map->image_pages) { + ret = -ENOMEM ; + goto errout_free; + } + ret =bpf_jit_charge_modmem (PAGE_SIZE );if (ret) goto errout_free; - st_map->image =arch_alloc_bpf_trampoline (PAGE_SIZE ); -if (!st_map->image) { -/* __bpf_struct_ops_map_free() uses st_map->image as flag -* for "charged or not". In this case, we need to unchange -* here. + st_map->image_pages[0 ] =arch_alloc_bpf_trampoline (PAGE_SIZE ); +if (!st_map->image_pages[0 ]) { +/* __bpf_struct_ops_map_free() uses st_map->image_pages_cnt +* to for uncharging a number of pages. In this case, we +* need to uncharge here. */ bpf_jit_uncharge_modmem (PAGE_SIZE ); ret = -ENOMEM ; goto errout_free; } + st_map->image_pages_cnt =1 ; st_map->uvalue =bpf_map_area_alloc (vt->size,NUMA_NO_NODE ); st_map->links_cnt =btf_type_vlen (t); st_map->links =
Signature · 1 line
2.34.1
Martin KaFai Lau
reviewerRe: [PATCH bpf-next 1/2] bpf: struct_ops supports more than one page for trampolines.
Quoted 60 lines · On 2/16/24 10:28 AM, [email protected] wrote:
From: Kui-Feng Lee <[email protected]>
The BPF
struct_opspreviously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.The variable
st_map->imagehas been changed tost_map->image_pages, and its type has been changed to an array of pointers to buffers ofPAGE_SIZE. The array is dynamically resized and additional pages are allocated when all existing pages are exhausted.Signed-off-by Kui-Feng Lee <[email protected]>--- kernel/bpf/bpf_struct_ops.c | 99 ++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 19 deletions(-)kernel/bpf/bpf_struct_ops.c +7
-7@@ -30,12 +30,11 @@ struct bpf_struct_ops_map {*/ struct bpf_link **links;u32 links_cnt; -/* image is a page that has all the trampolines +u32 image_pages_cnt; +/* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. -* A PAGE_SIZE "image" is enough to store all trampoline for -* "links[]". */ -void *image; +void **image_pages;/* The owner moduler's btf. */ struct btf *btf;/* uvalue->data stores the kernel struct @@ -535,7 +534,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,void *udata, *kdata;int prog_fd, err;void *image, *image_end; -u32 i; +void **image_pages; +u32 i, next_page =0 ;if (flags)return -EINVAL ;@@ -573,8 +573,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, udata = &uvalue->data; kdata = &kvalue->data; - image = st_map->image; - image_end = st_map->image +PAGE_SIZE ; + image = st_map->image_pages[next_page++]; + image_end = image +PAGE_SIZE ; module_type =btf_type_by_id (btf_vmlinux, st_ops_ids[IDX_MODULE_ID ]);for_each_member (i, t, member) {
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 666
Quoted 10 lines
kernel/bpf/bpf_struct_ops.c +7
-0@@ -657,6 +657,43 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, &st_ops->func_models[i], *(void **)(st_ops->cfi_stubs + moff), image, image_end); +if (err == -E2BIG ) { +/* Use an additional page to try again. +* +* It may reuse pages allocated for the previous +* failed calls. +*/ +if (next_page >= st_map->image_pages_cnt) {
This check (more on this later) ...
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 670
Quoted 4 lines
kernel/bpf/bpf_struct_ops.c +4
-0+/* Allocate an additional page */ + image_pages =krealloc (st_map->image_pages, + (st_map->image_pages_cnt +1 ) *sizeof (void *), +GFP_KERNEL );
From the patch 2 test, one page is enough for at least 20 ops. How about keep it simple and allow a max 8 pages which should be much more than enough for sane usage. (i.e. add "void *images[MAX_STRUCT_OPS_PAGES];" to "struct bpf_struct_ops_map").
Quoted 30 lines
kernel/bpf/bpf_struct_ops.c +26
-0+if (!image_pages) { + err = -ENOMEM ; + goto reset_unlock; + } + st_map->image_pages = image_pages; + + err =bpf_jit_charge_modmem (PAGE_SIZE ); +if (err) + goto reset_unlock; + + image =arch_alloc_bpf_trampoline (PAGE_SIZE ); +if (!image) { +bpf_jit_uncharge_modmem (PAGE_SIZE ); + err = -ENOMEM ; + goto reset_unlock; + } + st_map->image_pages[st_map->image_pages_cnt++] = image; + } + image = st_map->image_pages[next_page++]; + image_end = image +PAGE_SIZE ; + + err =bpf_struct_ops_prepare_trampoline (tlinks, link, + &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), + image, image_end); + }if (err <0 ) goto reset_unlock;
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 707
Quoted 4 lines
kernel/bpf/bpf_struct_ops.c +1
-0@@ -667,6 +704,18 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, *(unsigned long *)(udata + moff) = prog->aux->id; } +while (next_page < st_map->image_pages_cnt) {
This check and the above "if (next_page >= st_map->image_pages_cnt)" should not happen for the common case?
Together with the new comment after the above "if (err == -E2BIG)", is it trying to optimize to reuse the pages allocated in the previous error-ed out map_update_elem() call?
How about keep it simple for the common case and always free all pages when map_update_elem() failed?
Also, after this patch, the same calls are used in different places.
arch_alloc_bpf_trampoline() is done in two different places, one in map_alloc() and one in map_update_elem(). How about do all the page alloc in map_update_elem()?
bpf_struct_ops_prepare_trampoline() is also called in two different places within the same map_update_elem(). When looking inside the bpf_struct_ops_prepare_trampoline(), it does call arch_bpf_trampoline_size() to learn the required size first. bpf_struct_ops_prepare_trampoline() should be a better place to call arch_alloc_bpf_trampoline() when needed. Then there is no need to retry bpf_struct_ops_prepare_trampoline() in map_update_elem()?
Quoted 15 lines
kernel/bpf/bpf_struct_ops.c +11
-0+/* Free unused pages +* +* The value can not be updated anymore if the value is not +* rejected by st_ops->validate() or st_ops->reg(). So, +* there is no reason to keep the unused pages. +*/ +bpf_jit_uncharge_modmem (PAGE_SIZE ); + image = st_map->image_pages[--st_map->image_pages_cnt]; +arch_free_bpf_trampoline (image,PAGE_SIZE ); + } +if (st_map->map.map_flags &BPF_F_LINK ) { err =0 ;if (st_ops->validate) {
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 728
Quoted 7 lines
kernel/bpf/bpf_struct_ops.c +3
-1@@ -674,7 +723,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,if (err) goto reset_unlock; } -arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_protect_bpf_trampoline (st_map->image_pages[i], +PAGE_SIZE );
arch_protect_bpf_trampoline() is called here for BPF_F_LINK.
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 738
Quoted 10 lines
kernel/bpf/bpf_struct_ops.c +2
-1/* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem(). @@ -683,7 +734,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } -arch_protect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_protect_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE );
arch_protect_bpf_trampoline() is called here also in the same function for non BPF_F_LINK.
Can this be cleaned up a bit? For example, "st_ops->validate(kdata);" should not be specific to BPF_F_LINK which had been brought up earlier when making the "->validate" optional. It is a good time to clean this up also.
----
pw-bot: cr
Quoted 66 lines
kernel/bpf/bpf_struct_ops.c +19
-10err = st_ops->reg (kdata);if (likely (!err)) {/* This refcnt increment on the map here after @@ -706,7 +758,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,* there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ -arch_unprotect_bpf_trampoline (st_map->image,PAGE_SIZE ); +for (i =0 ; i < next_page; i++) +arch_unprotect_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE ); reset_unlock:bpf_struct_ops_map_put_progs (st_map);@@ -771,14 +824,15 @@ static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,static void __bpf_struct_ops_map_free (struct bpf_map *map) {struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; +int i;if (st_map->links)bpf_struct_ops_map_put_progs (st_map);bpf_map_area_free (st_map->links); -if (st_map->image) { -arch_free_bpf_trampoline (st_map->image,PAGE_SIZE ); -bpf_jit_uncharge_modmem (PAGE_SIZE ); - } +for (i =0 ; i < st_map->image_pages_cnt; i++) +arch_free_bpf_trampoline (st_map->image_pages[i],PAGE_SIZE ); +bpf_jit_uncharge_modmem (PAGE_SIZE * st_map->image_pages_cnt); +kfree (st_map->image_pages);bpf_map_area_free (st_map->uvalue);bpf_map_area_free (st_map); }@@ -888,20 +942,27 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) st_map->st_ops_desc = st_ops_desc; map = &st_map->map; + st_map->image_pages =kcalloc (1 ,sizeof (void *),GFP_KERNEL ); +if (!st_map->image_pages) { + ret = -ENOMEM ; + goto errout_free; + } + ret =bpf_jit_charge_modmem (PAGE_SIZE );if (ret) goto errout_free; - st_map->image =arch_alloc_bpf_trampoline (PAGE_SIZE ); -if (!st_map->image) { -/* __bpf_struct_ops_map_free() uses st_map->image as flag -* for "charged or not". In this case, we need to unchange -* here. + st_map->image_pages[0 ] =arch_alloc_bpf_trampoline (PAGE_SIZE ); +if (!st_map->image_pages[0 ]) { +/* __bpf_struct_ops_map_free() uses st_map->image_pages_cnt +* to for uncharging a number of pages. In this case, we +* need to uncharge here. */ bpf_jit_uncharge_modmem (PAGE_SIZE ); ret = -ENOMEM ; goto errout_free; } + st_map->image_pages_cnt =1 ; st_map->uvalue =bpf_map_area_alloc (vt->size,NUMA_NO_NODE ); st_map->links_cnt =btf_type_vlen (t); st_map->links =
Re: [PATCH bpf-next 1/2] bpf: struct_ops supports more than one page for trampolines.
Quoted 74 lines · Martin KaFai Lau
Quoted 60 lines · On 2/16/24 10:28 AM, [email protected] wrote:
From: Kui-Feng Lee <[email protected]>
The BPF
struct_opspreviously only allowed for one page to be used for the trampolines of all links in a map. However, we have recently run out of space due to the large number of BPF program links. By allocating additional pages when we exhaust an existing page, we can accommodate more links in a single map.The variable
st_map->imagehas been changed tost_map->image_pages, and its type has been changed to an array of pointers to buffers ofPAGE_SIZE. The array is dynamically resized and additional pages are allocated when all existing pages are exhausted.Signed-off-by Kui-Feng Lee <[email protected]>---
kernel/bpf/bpf_struct_ops.c| 99 ++++++++++++++++++++++++++++++-------1 file changed, 80 insertions(+), 19 deletions(-)kernel/bpf/bpf_struct_ops.c +7
-7@@ -30,12 +30,11 @@ struct bpf_struct_ops_map {*/ struct bpf_link **links; u32 links_cnt; - /* image is a page that has all the trampolines + u32 image_pages_cnt; + /* image_pages is an array of pages that has all the trampolines * that stores the func args before calling the bpf_prog. - * A PAGE_SIZE "image" is enough to store all trampoline for - * "links[]". */ - void *image; + void **image_pages; /* The owner moduler's btf. */ struct btf *btf; /* uvalue->data stores the kernel struct@@ -535,7 +534,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, void *udata, *kdata; int prog_fd, err; void *image, *image_end; - u32 i; + void **image_pages; + u32 i, next_page = 0; if (flags) return -EINVAL;@@ -573,8 +573,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, udata = &uvalue->data; kdata = &kvalue->data; - image = st_map->image; - image_end = st_map->image + PAGE_SIZE; + image = st_map->image_pages[next_page++]; + image_end = image + PAGE_SIZE; module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]); for_each_member(i, t, member) {@@ -657,6 +657,43 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct · line 667
Quoted 10 lines
kernel/bpf/bpf_struct_ops.c +7
-0&st_ops->func_models[i], *(void **)(st_ops->cfi_stubs + moff), image, image_end); + if (err == -E2BIG) { + /* Use an additional page to try again. + * + * It may reuse pages allocated for the previous + * failed calls. + */ + if (next_page >= st_map->image_pages_cnt) {This check (more on this later) ...
Quoted 8 lines
kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · line 670
Quoted 5 lines
kernel/bpf/bpf_struct_ops.c +4
-0+ /* Allocate an additional page */ + image_pages = krealloc(st_map->image_pages, + (st_map->image_pages_cnt + 1) *sizeof (void *), + GFP_KERNEL);From the patch 2 test, one page is enough for at least 20 ops. How
about keep it simple and allow a max 8 pages which should be much more than enough for sane usage. (i.e. add "void *images[MAX_STRUCT_OPS_PAGES];" to "structbpf_struct_ops_map").
Ok!
Quoted 37 lines
Quoted 30 lines
kernel/bpf/bpf_struct_ops.c +26
-0+ if (!image_pages) { + err = -ENOMEM; + goto reset_unlock; + } + st_map->image_pages = image_pages; + + err = bpf_jit_charge_modmem(PAGE_SIZE); + if (err) + goto reset_unlock; + + image = arch_alloc_bpf_trampoline(PAGE_SIZE); + if (!image) { + bpf_jit_uncharge_modmem(PAGE_SIZE); + err = -ENOMEM; + goto reset_unlock; + } + st_map->image_pages[st_map->image_pages_cnt++] = image; + } + image = st_map->image_pages[next_page++]; + image_end = image + PAGE_SIZE; + + err = bpf_struct_ops_prepare_trampoline(tlinks, link, + &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), + image, image_end); + } if (err < 0)goto
reset_unlock;kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct · line 707
Quoted 4 lines
kernel/bpf/bpf_struct_ops.c +1
-0@@ -667,6 +704,18 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, *(unsigned long *)(udata + moff) = prog->aux->id; } + while (next_page < st_map->image_pages_cnt) {This check and the above "if (
next_page>=st_map->image_pages_cnt)" should not happen for the common case?
Quoted 3 lines
Together with the new comment after the above "if (err ==
-E2BIG)", is it trying to optimize to reuse the pages allocated in the previous error-ed outmap_update_elem()call?How about keep it simple for the common case and always free all pages when
map_update_elem()failed?
Yes, it is an optimization. So, together with max 8 pages and free all pages when fails, we can remove all these code.
Quoted 6 lines
Also, after this patch, the same calls are used in different places.
arch_alloc_bpf_trampoline()is done in two different places, one inmap_alloc()and one inmap_update_elem(). How about do all the page alloc inmap_update_elem()?
bpf_struct_ops_prepare_trampoline()is also called in two different places within the samemap_update_elem(). When looking inside thebpf_struct_ops_prepare_trampoline(), it does callarch_bpf_trampoline_size()to learn the required size first.bpf_struct_ops_prepare_trampoline()should be a better place to callarch_alloc_bpf_trampoline()when needed. Then there is no need to retrybpf_struct_ops_prepare_trampoline()inmap_update_elem()?
Agree!
Quoted 41 lines
Quoted 17 lines
kernel/bpf/bpf_struct_ops.c +11
-0+ /* Free unused pages + * + * The value can not be updated anymore if the value is not + * rejected by st_ops->validate() or st_ops->reg(). So, + * there is no reason to keep the unused pages. + */ + bpf_jit_uncharge_modmem(PAGE_SIZE); + image = st_map->image_pages[--st_map->image_pages_cnt]; + arch_free_bpf_trampoline(image, PAGE_SIZE); + } + if (st_map->map.map_flags & BPF_F_LINK) { err = 0; if (st_ops->validate) {kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct · line 729
Quoted 8 lines
kernel/bpf/bpf_struct_ops.c +3
-1@@ -674,7 +723,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (err) goto reset_unlock; } - arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); + for (i = 0; i < next_page; i++) + arch_protect_bpf_trampoline(st_map->image_pages[i], + PAGE_SIZE);
arch_protect_bpf_trampoline()is called here forBPF_F_LINK.kernel/bpf/bpf_struct_ops.c static long bpf_struct_ops_map_update_elem(struct · line 738
Quoted 10 lines
kernel/bpf/bpf_struct_ops.c +0
-0/* Let bpf_link handle registration & unregistration.* * Pair with
smp_load_acquire()duringlookup_elem().kernel/bpf/bpf_struct_ops.c +2
-1@@ -683,7 +734,8 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } - arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); + for (i = 0; i < next_page; i++) + arch_protect_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE);
arch_protect_bpf_trampoline()is called here also in the same function for nonBPF_F_LINK.
Quoted 1 line
Can this be cleaned up a bit? For example, "
st_ops->validate(kdata);" should not be specific toBPF_F_LINKwhich had been brought up earlier when making the "->validate" optional. It is a good time to clean this up also.
Ok
Quoted 71 lines
----
pw-bot: crQuoted 66 lines
kernel/bpf/bpf_struct_ops.c +2
-1err = st_ops->reg(kdata); if (likely(!err)) { /* This refcnt increment on the map here after@@ -706,7 +758,8 @@ static long bpf_struct_ops_map_update_elem(structbpf_map *map, void *key, * there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ - arch_unprotect_bpf_trampoline(st_map->image, PAGE_SIZE); + for (i = 0; i < next_page; i++) + arch_unprotect_bpf_trampoline(st_map->image_pages[i],PAGE_SIZE );
reset_unlock:
bpf_struct_ops_map_put_progs(st_map);kernel/bpf/bpf_struct_ops.c +17
-9@@ -771,14 +824,15 @@ static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key, static void __bpf_struct_ops_map_free(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + int i; if (st_map->links) bpf_struct_ops_map_put_progs(st_map); bpf_map_area_free(st_map->links); - if (st_map->image) { - arch_free_bpf_trampoline(st_map->image, PAGE_SIZE); - bpf_jit_uncharge_modmem(PAGE_SIZE); - } + for (i = 0; i < st_map->image_pages_cnt; i++) + arch_free_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE); + bpf_jit_uncharge_modmem(PAGE_SIZE * st_map->image_pages_cnt); + kfree(st_map->image_pages); bpf_map_area_free(st_map->uvalue); bpf_map_area_free(st_map); }@@ -888,20 +942,27 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) st_map->st_ops_desc = st_ops_desc; map = &st_map->map; + st_map->image_pages = kcalloc(1, sizeof(void *), GFP_KERNEL); + if (!st_map->image_pages) { + ret = -ENOMEM; + goto errout_free; + } + ret = bpf_jit_charge_modmem(PAGE_SIZE); if (ret) goto errout_free; - st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); - if (!st_map->image) { - /* __bpf_struct_ops_map_free() uses st_map->image as flag - * for "charged or not". In this case, we need to unchange - * here. + st_map->image_pages[0] = arch_alloc_bpf_trampoline(PAGE_SIZE); + if (!st_map->image_pages[0]) { + /* __bpf_struct_ops_map_free() uses st_map->image_pages_cnt + * to for uncharging a number of pages. In this case, we + * need to uncharge here. */ bpf_jit_uncharge_modmem(PAGE_SIZE); ret = -ENOMEM; goto errout_free; } + st_map->image_pages_cnt = 1; st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE); st_map->links_cnt = btf_type_vlen(t);
st_map->links=
PATCH 2/2 selftests/bpf: Test struct_ops maps with a large number of program links.
In reply to [email protected]From: Kui-Feng Lee <[email protected]>
Create and load a struct_ops map with a large number of programs to generate trampolines taking a size over multiple pages. The map includes 40 programs. Their trampolines takes 6.6k+, more than 1.5 pages, on x86.
--- .../selftests/bpf/bpf_testmod/bpf_testmod.h | 44 ++++++++ .../prog_tests/test_struct_ops_multi_pages.c | 24 +++++ .../bpf/progs/struct_ops_multi_pages.c | 102 ++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +44 -0
@@ -35,6 +35,50 @@ struct bpf_testmod_ops {void (*test_2)(int a,int b);/* Used to test nullable arguments. */ int (*test_maybe_null)(int dummy,struct task_struct *task); + +/* The following pointers are used to test the maps having multiple +* pages of trampolines. +*/ +int (*tramp_1)(int value); +int (*tramp_2)(int value); +int (*tramp_3)(int value); +int (*tramp_4)(int value); +int (*tramp_5)(int value); +int (*tramp_6)(int value); +int (*tramp_7)(int value); +int (*tramp_8)(int value); +int (*tramp_9)(int value); +int (*tramp_10)(int value); +int (*tramp_11)(int value); +int (*tramp_12)(int value); +int (*tramp_13)(int value); +int (*tramp_14)(int value); +int (*tramp_15)(int value); +int (*tramp_16)(int value); +int (*tramp_17)(int value); +int (*tramp_18)(int value); +int (*tramp_19)(int value); +int (*tramp_20)(int value); +int (*tramp_21)(int value); +int (*tramp_22)(int value); +int (*tramp_23)(int value); +int (*tramp_24)(int value); +int (*tramp_25)(int value); +int (*tramp_26)(int value); +int (*tramp_27)(int value); +int (*tramp_28)(int value); +int (*tramp_29)(int value); +int (*tramp_30)(int value); +int (*tramp_31)(int value); +int (*tramp_32)(int value); +int (*tramp_33)(int value); +int (*tramp_34)(int value); +int (*tramp_35)(int value); +int (*tramp_36)(int value); +int (*tramp_37)(int value); +int (*tramp_38)(int value); +int (*tramp_39)(int value); + int (*tramp_40)(int value); };#endif /* _BPF_TESTMOD_H */
tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_pages.c new file +24 -0
@@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_multi_pages.skel.h" + +void test_struct_ops_multi_pages (void ) +{ +struct struct_ops_multi_pages *skel; +struct bpf_link *link; + +/* The size of all trampolines of skel->maps.multi_pages should be +* over 1 page (at least for x86). +*/ + skel =struct_ops_multi_pages__open_and_load (); +if (!ASSERT_OK_PTR (skel,"struct_ops_multi_pages_open_and_load" )) +return ; + + link =bpf_map__attach_struct_ops (skel->maps.multi_pages); +ASSERT_OK_PTR (link,"attach_multi_pages" ); + +bpf_link__destroy (link); +struct_ops_multi_pages__destroy (skel); +}
tools/testing/selftests/bpf/progs/struct_ops_multi_pages.c new file +102 -0
102 lines not on this page · raw message
Signature · 1 line
2.34.1