bpf_prog_pack and vmalloc-on-huge-page fixes
Subsystems (7)
Song Liu
reviewercover bpf_prog_pack and vmalloc-on-huge-page fixes
NOTE: This set is based on Linus' master branch (d569e86915b7), not bpf/master.
There are various discussion about these changes, check out [1] [2]. I guess we can use this thread to discuss which patches should go in 5.18. AFAICT, 1/4 need to with 5.18; 2/4 seems safe to go as well; 3/4 and 4/4 may still need more work/discussion.
Thanks!
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/
Song Liu (4):
bpf: invalidate unused part of bpf_prog_pack
page_alloc: use vmalloc_huge for large system hash
module: introduce module_alloc_huge
bpf: use module_alloc_huge for bpf_prog_pack
arch/x86/kernel/module.c | 21 +++++++++++++++++++++ arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++ include/linux/bpf.h | 2 ++ include/linux/moduleloader.h | 5 +++++ kernel/bpf/core.c | 28 +++++++++++++++++++++------- kernel/module.c | 8 ++++++++ mm/page_alloc.c | 2 +- 7 files changed, 80 insertions(+), 8 deletions(-)
--
2.30.2
Song Liu
reviewerPATCH 1/4 bpf: invalidate unused part of bpf_prog_pack
bpf_prog_pack enables sharing huge pages among multiple BPF programs. These pages are marked as executable, but some part of these huge page may not contain proper BPF programs. To make these pages safe, fill such unused part of these pages with invalid instructions. This is done when a pack is first allocated, and when a bpf program is freed..
--- arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++ include/linux/bpf.h | 2 ++ kernel/bpf/core.c | 21 +++++++++++++++++---- 3 files changed, 41 insertions(+), 4 deletions(-)
arch/x86/net/bpf_jit_comp.c +22 -0
@@ -228,6 +228,28 @@ static void jit_fill_hole(void *area, unsigned int size)memset (area,0xcc , size); } +#define INVALID_BUF_SIZE PAGE_SIZE +static char invalid_insn_buf[INVALID_BUF_SIZE ]; + +static int __init bpf_init_invalid_insn_buf (void ) +{ +jit_fill_hole (invalid_insn_buf,INVALID_BUF_SIZE ); +return 0 ; +} +pure_initcall (bpf_init_invalid_insn_buf); + +void bpf_arch_invalidate_text (void *dst,size_t len) +{ +size_t i =0 ; + +while (i < len) { +size_t s =min_t (size_t, len - i,INVALID_BUF_SIZE ); + +bpf_arch_text_copy (dst + i, invalid_insn_buf, s); + i += s; + } +} +struct jit_context {int cleanup_addr;/* Epilogue code offset */
include/linux/bpf.h +2 -0
@@ -2382,6 +2382,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,void *bpf_arch_text_copy (void *dst,void *src,size_t len); +void bpf_arch_invalidate_text (void *dst,size_t len); +struct btf_id_set ;bool btf_id_set_contains (const struct btf_id_set *set,u32 id);
kernel/bpf/core.c +17 -4
@@ -873,7 +873,7 @@ static size_t select_bpf_prog_pack_size(void)return size; } -static struct bpf_prog_pack *alloc_new_pack (void ) +static struct bpf_prog_pack *alloc_new_pack (bpf_jit_fill_hole_t bpf_fill_ill_insns) {struct bpf_prog_pack *pack;
@@ -886,6 +886,7 @@ static struct bpf_prog_pack *alloc_new_pack(void)kfree (pack);return NULL ; } +bpf_fill_ill_insns (pack->ptr, bpf_prog_pack_size);bitmap_zero (pack->bitmap, bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE );list_add_tail (&pack->list, &pack_list);
@@ -895,7 +896,7 @@ static struct bpf_prog_pack *alloc_new_pack(void)return pack; } -static void *bpf_prog_pack_alloc (u32 size) +static void *bpf_prog_pack_alloc (u32 size,bpf_jit_fill_hole_t bpf_fill_ill_insns) {unsigned int nbits =BPF_PROG_SIZE_TO_NBITS (size);struct bpf_prog_pack *pack;
@@ -910,6 +911,7 @@ static void *bpf_prog_pack_alloc(u32 size) size =round_up (size,PAGE_SIZE ); ptr =module_alloc (size);if (ptr) { +bpf_fill_ill_insns (ptr, size);set_vm_flush_reset_perms (ptr);set_memory_ro ((unsigned long )ptr, size /PAGE_SIZE );set_memory_x ((unsigned long )ptr, size /PAGE_SIZE );
@@ -923,7 +925,7 @@ static void *bpf_prog_pack_alloc(u32 size) goto found_free_area; } - pack =alloc_new_pack (); + pack =alloc_new_pack (bpf_fill_ill_insns);if (!pack) goto out;
@@ -966,6 +968,7 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) nbits =BPF_PROG_SIZE_TO_NBITS (hdr->size); pos = ((unsigned long )hdr - (unsigned long )pack_ptr) >>BPF_PROG_CHUNK_SHIFT ; +bpf_arch_invalidate_text (hdr, hdr->size);bitmap_clear (pack->bitmap, pos, nbits);if (bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 ,bpf_prog_chunk_count (),0 ) ==0 ) {
@@ -1102,7 +1105,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,if (bpf_jit_charge_modmem (size))return NULL ; - ro_header =bpf_prog_pack_alloc (size); + ro_header =bpf_prog_pack_alloc (size, bpf_fill_ill_insns);if (!ro_header) {bpf_jit_uncharge_modmem (size);return NULL ;
@@ -1203,6 +1206,16 @@ void __weak bpf_jit_free(struct bpf_prog *fp)bpf_prog_unlock_free (fp); } +void __weak bpf_arch_invalidate_text (void *dst,size_t len) +{ +char buf[1 ] = {}; +int i; + +WARN_ONCE (1 ,"Please override %s for bpf_prog_pack\n" , __func__); +for (i =0 ; i < len; i++) +bpf_arch_text_copy (dst + i, buf,1 ); +} +int bpf_jit_get_func_addr (const struct bpf_prog *prog,const struct bpf_insn *insn,bool extra_pass,u64 *func_addr,bool *func_addr_fixed)
Signature · 1 line
2.30.2
Song Liu
reviewerPATCH 2/4 page_alloc: use vmalloc_huge for large system hash
In reply to Song LiuUse vmalloc_huge() in alloc_large_system_hash() so that large system hash (>= PMD_SIZE) could benefit from huge pages. Note that vmalloc_huge only allocates huge pages for systems with HAVE_ARCH_HUGE_VMALLOC.
--- mm/page_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/page_alloc.c +1 -1
@@ -8919,7 +8919,7 @@ void *__init alloc_large_system_hash(const char *tablename, table =memblock_alloc_raw (size,SMP_CACHE_BYTES ); }else if (get_order (size) >=MAX_ORDER || hashdist) { - table =__vmalloc (size, gfp_flags); + table =vmalloc_huge (size, gfp_flags); virt = true;if (table) huge =is_vm_area_hugepages (table);
Signature · 1 line
2.30.2
kernel test robot
build botRe: [PATCH bpf 2/4] page_alloc: use vmalloc_huge for large system hash
I love your patch! Yet something to improve:
Hi Song,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/intel-lab-lkp/linux/commits/Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master config: i386-randconfig-a001 (https://download.01.org/0day-ci/archive/20220422/[email protected]/config) compiler: gcc-11 (Debian 11.2.0-20) 11.2.0 reproduce (this is a W=1 build): # https://github.com/intel-lab-lkp/linux/commit/239fb9ca743cf33db8d56df7957726e19aea87d5 git remote add linux-review https://github.com/intel-lab-lkp/linux git fetch --no-tags linux-review Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605 git checkout 239fb9ca743cf33db8d56df7957726e19aea87d5 # save the config file mkdir build_dir && cp config build_dir/.config make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
All error/warnings (new ones prefixed by >>):
mm/page_alloc.c: In function 'alloc_large_system_hash':
>> mm/page_alloc.c:8921:33: error: implicit declaration of function 'vmalloc_huge'; did you mean 'vmalloc_no_huge'? [-Werror=implicit-function-declaration]
8921 | table = vmalloc_huge(size, gfp_flags);
| ^~~~~~~~~~~~
| vmalloc_no_huge
>> mm/page_alloc.c:8921:31: warning: assignment to 'void *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
8921 | table = vmalloc_huge(size, gfp_flags);
| ^cc1: some warnings being treated as errors
vim +8921 mm/page_alloc.c
8876
8877 /* limit to 1 bucket per 2^scale bytes of low memory */
8878 if (scale > PAGE_SHIFT)
8879 numentries >>= (scale - PAGE_SHIFT);
8880 else
8881 numentries <<= (PAGE_SHIFT - scale);
8882
8883 /* Make sure we've got at least a 0-order allocation.. */
8884 if (unlikely(flags & HASH_SMALL)) {
8885 /* Makes no sense without HASH_EARLY */
8886 WARN_ON(!(flags & HASH_EARLY));
8887 if (!(numentries >> *_hash_shift)) {
8888 numentries = 1UL << *_hash_shift;
8889 BUG_ON(!numentries);
8890 }
8891 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
8892 numentries = PAGE_SIZE / bucketsize;
8893 }
8894 numentries = roundup_pow_of_two(numentries);
8895
8896 /* limit allocation size to 1/16 total memory by default */
8897 if (max == 0) {
8898 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8899 do_div(max, bucketsize);
8900 }
8901 max = min(max, 0x80000000ULL);
8902
8903 if (numentries < low_limit)
8904 numentries = low_limit;
8905 if (numentries > max)
8906 numentries = max;
8907
8908 log2qty = ilog2(numentries);
8909
8910 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
8911 do {
8912 virt = false;
8913 size = bucketsize << log2qty;
8914 if (flags & HASH_EARLY) {
8915 if (flags & HASH_ZERO)
8916 table = memblock_alloc(size, SMP_CACHE_BYTES);
8917 else
8918 table = memblock_alloc_raw(size,
8919 SMP_CACHE_BYTES);
8920 } else if (get_order(size) >= MAX_ORDER || hashdist) {
> 8921 table = vmalloc_huge(size, gfp_flags);
8922 virt = true;
8923 if (table)
8924 huge = is_vm_area_hugepages(table);
8925 } else {
8926 /*
8927 * If bucketsize is not a power-of-two, we may free
8928 * some pages at the end of hash table which
8929 * alloc_pages_exact() automatically does
8930 */
8931 table = alloc_pages_exact(size, gfp_flags);
8932 kmemleak_alloc(table, size, 1, gfp_flags);
8933 }
8934 } while (!table && size > PAGE_SIZE && --log2qty);
8935
8936 if (!table)
8937 panic("Failed to allocate %s hash table\n", tablename);
8938
8939 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8940 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
8941 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
8942
8943 if (_hash_shift)
8944 *_hash_shift = log2qty;
8945 if (_hash_mask)
8946 *_hash_mask = (1 << log2qty) - 1;
8947
8948 return table;
8949 }
8950
Signature · 2 lines
0-DAY CI Kernel Test Service https://01.org/lkp
Song Liu
reviewerPATCH 3/4 module: introduce module_alloc_huge
In reply to Song LiuIntroduce module_alloc_huge, which allocates huge page backed memory in module memory space. The primary user of this memory is bpf_prog_pack (multiple BPF programs sharing a huge page).
--- arch/x86/kernel/module.c | 21 +++++++++++++++++++++ include/linux/moduleloader.h | 5 +++++ kernel/module.c | 8 ++++++++ 3 files changed, 34 insertions(+)
arch/x86/kernel/module.c +21 -0
@@ -86,6 +86,27 @@ void *module_alloc(unsigned long size)return p; } +void *module_alloc_huge (unsigned long size) +{ +gfp_t gfp_mask =GFP_KERNEL ; +void *p; + +if (PAGE_ALIGN (size) >MODULES_LEN ) +return NULL ; + + p =__vmalloc_node_range (size,MODULE_ALIGN , +MODULES_VADDR +get_module_load_offset (), +MODULES_END , gfp_mask,PAGE_KERNEL , +VM_DEFER_KMEMLEAK |VM_ALLOW_HUGE_VMAP , +NUMA_NO_NODE ,__builtin_return_address (0 )); +if (p && (kasan_alloc_module_shadow (p, size, gfp_mask) <0 )) { +vfree (p); +return NULL ; + } + +return p; +} +#ifdef CONFIG_X86_32 int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab,
include/linux/moduleloader.h +5 -0
@@ -26,6 +26,11 @@ unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section);sections. Returns NULL on failure. */ void *module_alloc (unsigned long size); +/* Allocator used for allocating memory in module memory space. If size is +* greater than PMD_SIZE, allow using huge pages. Returns NULL on failure. +*/ +void *module_alloc_huge (unsigned long size); +/* Free memory returned from module_alloc. */ void module_memfree (void *module_region);
kernel/module.c +8 -0
@@ -2839,6 +2839,14 @@ void * __weak module_alloc(unsigned long size) NUMA_NO_NODE, __builtin_return_address(0)); } +void *__weak module_alloc_huge (unsigned long size) +{ +return __vmalloc_node_range (size,1 ,VMALLOC_START ,VMALLOC_END , +GFP_KERNEL ,PAGE_KERNEL_EXEC , +VM_FLUSH_RESET_PERMS |VM_ALLOW_HUGE_VMAP , +NUMA_NO_NODE ,__builtin_return_address (0 )); +} +bool __weak module_init_section (const char *name) {return strstarts (name,".init" );
Signature · 1 line
2.30.2
kernel test robot
build botRe: [PATCH bpf 3/4] module: introduce module_alloc_huge
I love your patch! Yet something to improve:
Hi Song,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/intel-lab-lkp/linux/commits/Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605 base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master config: x86_64-randconfig-a004 (https://download.01.org/0day-ci/archive/20220422/[email protected]/config) compiler: gcc-11 (Debian 11.2.0-20) 11.2.0 reproduce (this is a W=1 build): # https://github.com/intel-lab-lkp/linux/commit/8a0dfde5aef7e95487be2f6e3ff9487d79a30714 git remote add linux-review https://github.com/intel-lab-lkp/linux git fetch --no-tags linux-review Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605 git checkout 8a0dfde5aef7e95487be2f6e3ff9487d79a30714 # save the config file mkdir build_dir && cp config build_dir/.config make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
arch/x86/kernel/module.c: In function 'module_alloc_huge':
>> arch/x86/kernel/module.c:100:54: error: 'VM_ALLOW_HUGE_VMAP' undeclared (first use in this function); did you mean 'VM_NO_HUGE_VMAP'?
100 | VM_DEFER_KMEMLEAK | VM_ALLOW_HUGE_VMAP,
| ^~~~~~~~~~~~~~~~~~
| VM_NO_HUGE_VMAP arch/x86/kernel/module.c:100:54: note: each undeclared identifier is reported only once for each function it appears in
vim +100 arch/x86/kernel/module.c
88
89 void *module_alloc_huge(unsigned long size)
90 {
91 gfp_t gfp_mask = GFP_KERNEL;
92 void *p;
93
94 if (PAGE_ALIGN(size) > MODULES_LEN)
95 return NULL;
96
97 p = __vmalloc_node_range(size, MODULE_ALIGN,
98 MODULES_VADDR + get_module_load_offset(),
99 MODULES_END, gfp_mask, PAGE_KERNEL,
> 100 VM_DEFER_KMEMLEAK | VM_ALLOW_HUGE_VMAP, 101 NUMA_NO_NODE, __builtin_return_address(0));
102 if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
103 vfree(p);
104 return NULL;
105 }
106
107 return p;
108 }
109
Signature · 2 lines
0-DAY CI Kernel Test Service https://01.org/lkp
Song Liu
reviewerPATCH 4/4 bpf: use module_alloc_huge for bpf_prog_pack
In reply to Song LiuUse module_alloc_huge for bpf_prog_pack so that BPF programs sit on PMD_SIZE pages. This benefits system performance by reducing iTLB miss rate.
Also, remove set_vm_flush_reset_perms() from alloc_new_pack() and use set_memory_[nx|rw] in bpf_prog_pack_free(). This is because VM_FLUSH_RESET_PERMS does not work with huge pages yet.
--- kernel/bpf/core.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
kernel/bpf/core.c +4 -3
@@ -857,7 +857,7 @@ static size_t select_bpf_prog_pack_size(void)void *ptr; size =BPF_HPAGE_SIZE *num_online_nodes (); - ptr =module_alloc (size); + ptr =module_alloc_huge (size);/* Test whether we can get huge pages. If not just use PAGE_SIZE * packs.
@@ -881,7 +881,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins GFP_KERNEL);if (!pack)return NULL ; - pack->ptr =module_alloc (bpf_prog_pack_size); + pack->ptr =module_alloc_huge (bpf_prog_pack_size);if (!pack->ptr) {kfree (pack);return NULL ;
@@ -890,7 +890,6 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insbitmap_zero (pack->bitmap, bpf_prog_pack_size /BPF_PROG_CHUNK_SIZE );list_add_tail (&pack->list, &pack_list); -set_vm_flush_reset_perms (pack->ptr);set_memory_ro ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE );set_memory_x ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE );return pack;
@@ -973,6 +972,8 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)if (bitmap_find_next_zero_area (pack->bitmap,bpf_prog_chunk_count (),0 ,bpf_prog_chunk_count (),0 ) ==0 ) {list_del (&pack->list); +set_memory_nx ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE ); +set_memory_rw ((unsigned long )pack->ptr, bpf_prog_pack_size /PAGE_SIZE );module_memfree (pack->ptr);kfree (pack); }
Signature · 1 line
2.30.2
Luis Chamberlain
maintainerQuoted 6 lines · Song Liu
NOTE: This set is based on Linus' master branch (
d569e86915b7), not bpf/master.There are various discussion about these changes, check out [1] [2]. I guess we can use this thread to discuss which patches should go in 5.18. AFAICT, 1/4 need to with 5.18;
Since huge pages are effectively disabled on v5.18 I can't see why.
Quoted 1 line
2/4 seems safe to go as well;
My impression on the discussion was that huge pages design was broken and evidence for this came up after x86 finally enabled *a small* portion use case of it. This revealed how broken huge pages were not just for x86 but for other architectures. And so I can't see why we'd enable for v5.18 huge pages for the large system hash.
Quoted 2 lines
3/4 and 4/4
may still need more work/discussion.
Happy to review these but if huge pages are disabled I don't see the point in a module_alloc_huge() yet.
Luis
Quoted 23 lines
Thanks!
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/Song Liu (4):
bpf: invalidate unused part ofbpf_prog_pack
page_alloc: usevmalloc_hugefor large system hash
module: introducemodule_alloc_huge
bpf: usemodule_alloc_hugeforbpf_prog_packarch/x86/kernel/module.c | 21 +++++++++++++++++++++ arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++ include/linux/bpf.h | 2 ++ include/linux/moduleloader.h | 5 +++++ kernel/bpf/core.c | 28 +++++++++++++++++++++------- kernel/module.c | 8 ++++++++ mm/page_alloc.c | 2 +- 7 files changed, 80 insertions(+), 8 deletions(-)--
2.30.2