bpf: simplify bpf_prog_pack_[size|mask]
Song Liu
reviewerPATCH 1/1 bpf: simplify bpf_prog_pack_[size|mask]
Simplify the logic that selects bpf_prog_pack_size, and always use (PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [1].
Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore, we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of bpf_prog_pack. Fix this by checking the header address falls in the range of pack->ptr and (pack->ptr + bpf_prog_pack_size).
[1] https://lore.kernel.org/bpf/[email protected]/
--- kernel/bpf/core.c | 71 ++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 54 deletions(-)
kernel/bpf/core.c +17 -54
@@ -825,15 +825,6 @@ struct bpf_prog_pack {#define BPF_PROG_SIZE_TO_NBITS (size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) -static size_t bpf_prog_pack_size =-1 ; -static size_t bpf_prog_pack_mask =-1 ; - -static int bpf_prog_chunk_count (void ) -{ -WARN_ON_ONCE (bpf_prog_pack_size ==-1 ); -return bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE ; -} -static DEFINE_MUTEX (pack_mutex);static LIST_HEAD (pack_list);
@@ -841,55 +832,33 @@ static LIST_HEAD(pack_list);* CONFIG_MMU=n. Use PAGE_SIZE in these cases. */ #ifdef PMD_SIZE -#define BPF_HPAGE_SIZE PMD_SIZE -#define BPF_HPAGE_MASK PMD_MASK +#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes())#else -#define BPF_HPAGE_SIZE PAGE_SIZE -#define BPF_HPAGE_MASK PAGE_MASK +#define BPF_PROG_PACK_SIZE PAGE_SIZE#endif -static size_t select_bpf_prog_pack_size (void ) -{ -size_t size; -void *ptr; - - size =BPF_HPAGE_SIZE *num_online_nodes (); - ptr =module_alloc (size); - -/* Test whether we can get huge pages. If not just use PAGE_SIZE -* packs. -*/ -if (!ptr || !is_vm_area_hugepages (ptr)) { - size =PAGE_SIZE ; - bpf_prog_pack_mask =PAGE_MASK ; - }else { - bpf_prog_pack_mask =BPF_HPAGE_MASK ; - } - -vfree (ptr); -return size; -} +#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)static struct bpf_prog_pack *alloc_new_pack (bpf_jit_fill_hole_t bpf_fill_ill_insns) {struct bpf_prog_pack *pack; - pack =kzalloc (struct_size (pack, bitmap,BITS_TO_LONGS (bpf_prog_chunk_count ())), + pack =kzalloc (struct_size (pack, bitmap,BITS_TO_LONGS (BPF_PROG_CHUNK_COUNT )),GFP_KERNEL );if (!pack)return NULL ; - pack->ptr =module_alloc (bpf_prog_pack_size); + pack->ptr =module_alloc (BPF_PROG_PACK_SIZE );if (!pack->ptr) {kfree (pack);return NULL ; } -bpf_fill_ill_insns (pack->ptr, bpf_prog_pack_size); -bitmap_zero (pack->bitmap, bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE ); +bpf_fill_ill_insns (pack->ptr,BPF_PROG_PACK_SIZE ); +bitmap_zero (pack->bitmap,BPF_PROG_PACK_SIZE /BPF_PROG_CHUNK_SIZE );list_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); +set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE );return pack; }
@@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insnvoid *ptr =NULL ;mutex_lock (&pack_mutex); -if (bpf_prog_pack_size ==-1 ) - bpf_prog_pack_size =select_bpf_prog_pack_size (); - -if (size > bpf_prog_pack_size) { +if (size >BPF_PROG_PACK_SIZE ) { size =round_up (size,PAGE_SIZE ); ptr =module_alloc (size);if (ptr) {
@@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn goto out; }list_for_each_entry (pack, &pack_list, list) { - pos =bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 , + pos =bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , nbits,0 ); -if (pos <bpf_prog_chunk_count ()) +if (pos <BPF_PROG_CHUNK_COUNT ) goto found_free_area; }
@@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)struct bpf_prog_pack *pack =NULL , *tmp;unsigned int nbits;unsigned long pos; -void *pack_ptr;mutex_lock (&pack_mutex); -if (hdr->size > bpf_prog_pack_size) { +if (hdr->size >BPF_PROG_PACK_SIZE ) {module_memfree (hdr); goto out; } - pack_ptr = (void *)((unsigned long )hdr & bpf_prog_pack_mask); -list_for_each_entry (tmp, &pack_list, list) { -if (tmp->ptr == pack_ptr) { +if ((void *)hdr >= tmp->ptr && (tmp->ptr +BPF_PROG_PACK_SIZE ) > (void *)hdr) { pack = tmp;break ; }
@@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) goto out; nbits =BPF_PROG_SIZE_TO_NBITS (hdr->size); - pos = ((unsigned long )hdr - (unsigned long )pack_ptr) >>BPF_PROG_CHUNK_SHIFT ; + pos = ((unsigned long )hdr - (unsigned long )pack->ptr) >>BPF_PROG_CHUNK_SHIFT ;WARN_ONCE (bpf_arch_text_invalidate (hdr, hdr->size),"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n" );bitmap_clear (pack->bitmap, pos, nbits); -if (bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 , -bpf_prog_chunk_count (),0 ) ==0 ) { +if (bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , +BPF_PROG_CHUNK_COUNT ,0 ) ==0 ) {list_del (&pack->list);module_memfree (pack->ptr);kfree (pack);
Signature · 1 line
2.30.2
Quoted 11 lines · Song Liu
Simplify the logic that selects
bpf_prog_pack_size, and always use (PMD_SIZE*num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [1].Also,
module_alloc(4MB)may not allocate 4MB aligned memory. Therefore, we cannot use (ptr &bpf_prog_pack_mask) to find the correct address ofbpf_prog_pack. Fix this by checking the header address falls in the range ofpack->ptrand (pack->ptr+bpf_prog_pack_size).[1] https://lore.kernel.org/bpf/[email protected]/
Signed-off-by Song Liu <[email protected]>
(purely from comparing before/after, ignoring larger context)
Quoted 166 lines
--- kernel/bpf/core.c | 71 ++++++++++++----------------------------------- 1 file changed, 17 insertions(+), 54 deletions(-)kernel/bpf/core.c +9
-40@@ -825,15 +825,6 @@ struct bpf_prog_pack {#define BPF_PROG_SIZE_TO_NBITS (size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) -static size_t bpf_prog_pack_size =-1 ; -static size_t bpf_prog_pack_mask =-1 ; - -static int bpf_prog_chunk_count (void ) -{ -WARN_ON_ONCE (bpf_prog_pack_size ==-1 ); -return bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE ; -} -static DEFINE_MUTEX (pack_mutex);static LIST_HEAD (pack_list);@@ -841,55 +832,33 @@ static LIST_HEAD(pack_list);* CONFIG_MMU=n. Use PAGE_SIZE in these cases. */ #ifdef PMD_SIZE -#define BPF_HPAGE_SIZE PMD_SIZE -#define BPF_HPAGE_MASK PMD_MASK +#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes())#else -#define BPF_HPAGE_SIZE PAGE_SIZE -#define BPF_HPAGE_MASK PAGE_MASK +#define BPF_PROG_PACK_SIZE PAGE_SIZE#endif -static size_t select_bpf_prog_pack_size (void ) -{ -size_t size; -void *ptr; - - size =BPF_HPAGE_SIZE *num_online_nodes (); - ptr =module_alloc (size); - -/* Test whether we can get huge pages. If not just use PAGE_SIZE -* packs. -*/ -if (!ptr || !is_vm_area_hugepages (ptr)) { - size =PAGE_SIZE ; - bpf_prog_pack_mask =PAGE_MASK ; - }else { - bpf_prog_pack_mask =BPF_HPAGE_MASK ; - } - -vfree (ptr); -return size; -} +#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)static struct bpf_prog_pack *alloc_new_pack (bpf_jit_fill_hole_t bpf_fill_ill_insns) {struct bpf_prog_pack *pack; - pack =kzalloc (struct_size (pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())), + pack =kzalloc (struct_size (pack, bitmap,BITS_TO_LONGS (BPF_PROG_CHUNK_COUNT )),GFP_KERNEL );if (!pack)return NULL ; - pack->ptr =module_alloc (bpf_prog_pack_size); + pack->ptr =module_alloc (BPF_PROG_PACK_SIZE );if (!pack->ptr) {kfree (pack);return NULL ; } -bpf_fill_ill_insns (pack->ptr, bpf_prog_pack_size); -bitmap_zero (pack->bitmap, bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE ); +bpf_fill_ill_insns (pack->ptr,BPF_PROG_PACK_SIZE ); +bitmap_zero (pack->bitmap,BPF_PROG_PACK_SIZE /BPF_PROG_CHUNK_SIZE );list_add_tail (&pack->list, &pack_list);set_vm_flush_reset_perms (pack->ptr); -set_memory_ro ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); -set_memory_x ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); +set_memory_ro ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE ); +set_memory_x ((unsigned long )pack->ptr,BPF_PROG_PACK_SIZE /PAGE_SIZE );return pack; }kernel/bpf/core.c +0
-0@@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size,
bpf_jit_fill_hole_tbpf_fill_ill_insn
void *ptr = NULL;mutex_lock(&pack_mutex);kernel/bpf/core.c +1
-4-if (bpf_prog_pack_size ==-1 ) - bpf_prog_pack_size =select_bpf_prog_pack_size (); - -if (size > bpf_prog_pack_size) { +if (size >BPF_PROG_PACK_SIZE ) { size =round_up (size,PAGE_SIZE ); ptr =module_alloc (size);if (ptr) {@@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size,
bpf_jit_fill_hole_tbpf_fill_ill_insn
goto out;
}
list_for_each_entry(pack, &pack_list, list){kernel/bpf/core.c +0
-1- pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(),0,
kernel/bpf/core.c +4
-6+ pos =bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , nbits,0 ); -if (pos <bpf_prog_chunk_count ()) +if (pos <BPF_PROG_CHUNK_COUNT ) goto found_free_area; }@@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)struct bpf_prog_pack *pack =NULL , *tmp;unsigned int nbits;unsigned long pos; -void *pack_ptr;mutex_lock (&pack_mutex); -if (hdr->size > bpf_prog_pack_size) { +if (hdr->size >BPF_PROG_PACK_SIZE ) {module_memfree (hdr); goto out; } - pack_ptr = (void *)((unsigned long )hdr & bpf_prog_pack_mask); -list_for_each_entry (tmp, &pack_list, list) { -if (tmp->ptr == pack_ptr) { +if ((void *)hdr >= tmp->ptr && (tmp->ptr +BPF_PROG_PACK_SIZE ) > (void *)hdr) { pack = tmp;break; }kernel/bpf/core.c +3
-3@@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) goto out; nbits =BPF_PROG_SIZE_TO_NBITS (hdr->size); - pos = ((unsigned long )hdr - (unsigned long )pack_ptr) >>BPF_PROG_CHUNK_SHIFT ; + pos = ((unsigned long )hdr - (unsigned long )pack->ptr) >>BPF_PROG_CHUNK_SHIFT ;WARN_ONCE (bpf_arch_text_invalidate (hdr, hdr->size),"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n" );bitmap_clear (pack->bitmap, pos, nbits); -if (bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 , -bpf_prog_chunk_count (),0 ) ==0 ) { +if (bitmap_find_next_zero_area (pack->bitmap,BPF_PROG_CHUNK_COUNT ,0 , +BPF_PROG_CHUNK_COUNT ,0 ) ==0 ) {
list_del(&pack->list);
module_memfree(pack->ptr);
kfree(pack);
--
2.30.2
This patch was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Hello:
This patch was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:
Quoted 10 lines · On Wed, 13 Jul 2022 13:49:50 -0700 you wrote:
Simplify the logic that selects
bpf_prog_pack_size, and always use (PMD_SIZE*num_possible_nodes()). This is a good tradeoff, as most of the performance benefit observed is from less direct map fragmentation [1].Also,
module_alloc(4MB)may not allocate 4MB aligned memory. Therefore, we cannot use (ptr &bpf_prog_pack_mask) to find the correct address ofbpf_prog_pack. Fix this by checking the header address falls in the range ofpack->ptrand (pack->ptr+bpf_prog_pack_size).[...]
Here is the summary with links:
- [bpf-next] bpf: simplify bpf_prog_pack_[size|mask]
https://git.kernel.org/bpf/bpf-next/c/ea2babac63d4
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html