bpf: select proper size for bpf_prog_pack
Subsystems (4)
Song Liu
reviewerPATCH v4 1/1 bpf: select proper size for bpf_prog_pack
Using HPAGE_PMD_SIZE as the size for bpf_prog_pack is not ideal in some cases. Specifically, for NUMA systems, __vmalloc_node_range requires PMD_SIZE * num_online_nodes() to allocate huge pages. Also, if the system does not support huge pages (i.e., with cmdline option nohugevmalloc), it is better to use PAGE_SIZE packs.
Add logic to select proper size for bpf_prog_pack. This solution is not ideal, as it makes assumption about the behavior of module_alloc and __vmalloc_node_range. However, it appears to be the easiest solution as it doesn't require changes in module_alloc and vmalloc code.
---
Changes v3 => v4:
1. Fix a race condition reported by kernel test robot <[email protected]>.
Changes v2 => v3:
1. Remove a leftover debug change.
2. Shuffle alloc_new_pack() for cleaner patch.
Changes v1 => v2:
1. Fix case with first program > PAGE_SIZE. (Daniel)
2. Add Fixes tag.
3. Remove a inline to avoid netdev/source_inline error.
--- kernel/bpf/core.c | 70 +++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 23 deletions(-)
kernel/bpf/core.c +47 -23
@@ -33,6 +33,7 @@#include <linux/extable.h> #include <linux/log2.h> #include <linux/bpf_verifier.h> +#include <linux/nodemask.h> #include <asm/barrier.h> #include <asm/unaligned.h>
@@ -814,15 +815,9 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,* allocator. The prog_pack allocator uses HPAGE_PMD_SIZE page (2MB on x86) * to host BPF programs. */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define BPF_PROG_PACK_SIZE HPAGE_PMD_SIZE -#else -#define BPF_PROG_PACK_SIZE PAGE_SIZE -#endif #define BPF_PROG_CHUNK_SHIFT 6#define BPF_PROG_CHUNK_SIZE (1 << BPF_PROG_CHUNK_SHIFT)#define BPF_PROG_CHUNK_MASK (~(BPF_PROG_CHUNK_SIZE - 1)) -#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)struct bpf_prog_pack {struct list_head list;
@@ -830,30 +825,56 @@ struct bpf_prog_pack {unsigned long bitmap[]; }; -#define BPF_PROG_MAX_PACK_PROG_SIZE BPF_PROG_PACK_SIZE#define BPF_PROG_SIZE_TO_NBITS (size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) +static size_t bpf_prog_pack_size =-1 ; + +static int bpf_prog_chunk_count (void ) +{ +WARN_ON_ONCE (bpf_prog_pack_size ==-1 ); +return bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE ; +} +static DEFINE_MUTEX (pack_mutex);static LIST_HEAD (pack_list); +static size_t select_bpf_prog_pack_size (void ) +{ +size_t size; +void *ptr; + + size =PMD_SIZE *num_online_nodes (); + ptr =module_alloc (size); + +/* Test whether we can get huge pages. If not just use PAGE_SIZE +* packs. +*/ +if (!ptr || !is_vm_area_hugepages (ptr)) + size =PAGE_SIZE ; + +vfree (ptr); +return size; +} +static struct bpf_prog_pack *alloc_new_pack (void ) {struct bpf_prog_pack *pack; - pack =kzalloc (sizeof (*pack) +BITS_TO_BYTES (BPF_PROG_CHUNK_COUNT ),GFP_KERNEL ); + pack =kzalloc (struct_size (pack, bitmap,BITS_TO_LONGS (bpf_prog_chunk_count ())), +GFP_KERNEL );if (!pack)return NULL ; - pack->ptr =module_alloc (BPF_PROG_PACK_SIZE ); + pack->ptr =module_alloc (bpf_prog_pack_size);if (!pack->ptr) {kfree (pack);return NULL ; } -bitmap_zero (pack->bitmap,BPF_PROG_PACK_SIZE /BPF_PROG_CHUNK_SIZE ); +bitmap_zero (pack->bitmap, bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE );list_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_ro ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); +set_memory_x ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE );return pack; }
@@ -864,7 +885,11 @@ static void *bpf_prog_pack_alloc(u32 size)unsigned long pos;void *ptr =NULL ; -if (size >BPF_PROG_MAX_PACK_PROG_SIZE ) { +mutex_lock (&pack_mutex); +if (bpf_prog_pack_size ==-1 ) + bpf_prog_pack_size =select_bpf_prog_pack_size (); + +if (size > bpf_prog_pack_size) { size =round_up (size,PAGE_SIZE ); ptr =module_alloc (size);if (ptr) {
@@ -872,13 +897,12 @@ static void *bpf_prog_pack_alloc(u32 size)set_memory_ro ((unsigned long )ptr, size /PAGE_SIZE );set_memory_x ((unsigned long )ptr, size /PAGE_SIZE ); } -return ptr; + goto out; } -mutex_lock (&pack_mutex);list_for_each_entry (pack, &pack_list, list) { - pos =bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , + pos =bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 , nbits,0 ); -if (pos <BPF_PROG_CHUNK_COUNT ) +if (pos <bpf_prog_chunk_count ()) goto found_free_area; }
@@ -904,13 +928,13 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)unsigned long pos;void *pack_ptr; -if (hdr->size >BPF_PROG_MAX_PACK_PROG_SIZE ) { +mutex_lock (&pack_mutex); +if (hdr->size > bpf_prog_pack_size) {module_memfree (hdr); -return ; + goto out; } - pack_ptr = (void *)((unsigned long )hdr & ~(BPF_PROG_PACK_SIZE -1 )); -mutex_lock (&pack_mutex); + pack_ptr = (void *)((unsigned long )hdr & ~(bpf_prog_pack_size -1 ));list_for_each_entry (tmp, &pack_list, list) {if (tmp->ptr == pack_ptr) {
@@ -926,8 +950,8 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) pos = ((unsigned long )hdr - (unsigned long )pack_ptr) >>BPF_PROG_CHUNK_SHIFT ;bitmap_clear (pack->bitmap, pos, nbits); -if (bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , -BPF_PROG_CHUNK_COUNT ,0 ) ==0 ) { +if (bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 , +bpf_prog_chunk_count (),0 ) ==0 ) {list_del (&pack->list);module_memfree (pack->ptr);kfree (pack);
Signature · 1 line
2.30.2
This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Quoted 6 lines · On Fri, 11 Mar 2022 12:11:35 -0800 you wrote:
Using
HPAGE_PMD_SIZEas the size forbpf_prog_packis not ideal in some cases. Specifically, for NUMA systems,__vmalloc_node_rangerequiresPMD_SIZE*num_online_nodes()to allocate huge pages. Also, if the system does not support huge pages (i.e., with cmdline option nohugevmalloc), it is better to usePAGE_SIZEpacks.
Quoted 6 lines
Add logic to select proper size for
bpf_prog_pack. This solution is not ideal, as it makes assumption about the behavior ofmodule_allocand__vmalloc_node_range. However, it appears to be the easiest solution as it doesn't require changes inmodule_allocand vmalloc code.[...]
Here is the summary with links:
- [v4,bpf-next] bpf: select proper size for bpf_prog_pack
https://git.kernel.org/bpf/bpf-next/c/ef078600eec2
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html