Skip to contentSkip to conversation

bpf_prog_pack and vmalloc-on-huge-page fixes

PATCH Superseded Song Liu 3 participants 8 messages Apr 22, 2022

Subsystems (7)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Song Liu

author reviewer
#0 lore raw

cover bpf_prog_pack and vmalloc-on-huge-page fixes

NOTE: This set is based on Linus' master branch (d569e86915b7), not bpf/master.

There are various discussion about these changes, check out [1] [2]. I guess we can use this thread to discuss which patches should go in 5.18. AFAICT, 1/4 need to with 5.18; 2/4 seems safe to go as well; 3/4 and 4/4 may still need more work/discussion.

Thanks!

[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/

Song Liu (4):
bpf: invalidate unused part of bpf_prog_pack
page_alloc: use vmalloc_huge for large system hash
module: introduce module_alloc_huge
bpf: use module_alloc_huge for bpf_prog_pack

 arch/x86/kernel/module.c     | 21 +++++++++++++++++++++
 arch/x86/net/bpf_jit_comp.c  | 22 ++++++++++++++++++++++
 include/linux/bpf.h          |  2 ++
 include/linux/moduleloader.h |  5 +++++
 kernel/bpf/core.c            | 28 +++++++++++++++++++++-------
 kernel/module.c              |  8 ++++++++
 mm/page_alloc.c              |  2 +-
 7 files changed, 80 insertions(+), 8 deletions(-)

--
2.30.2

author reviewer
#1 lore raw

PATCH 1/4 bpf: invalidate unused part of bpf_prog_pack

bpf_prog_pack enables sharing huge pages among multiple BPF programs. These pages are marked as executable, but some part of these huge page may not contain proper BPF programs. To make these pages safe, fill such unused part of these pages with invalid instructions. This is done when a pack is first allocated, and when a bpf program is freed..

Fixes 57631054fae6 ("bpf: Introduce bpf_prog_pack allocator")
Fixes 33c9805860e5 ("bpf: Introduce bpf_jit_binary_pack_[alloc|finalize|free]")
---
 arch/x86/net/bpf_jit_comp.c | 22 ++++++++++++++++++++++
 include/linux/bpf.h         |  2 ++
 kernel/bpf/core.c           | 21 +++++++++++++++++----
 3 files changed, 41 insertions(+), 4 deletions(-)
arch/x86/net/bpf_jit_comp.c +22 -0
index 16b6efacf7c6..ff3f0811c9a1 100644
@@ -228,6 +228,28 @@ static void jit_fill_hole(void *area, unsigned int size)
 	memset(area, 0xcc, size);
 }
 
+#define INVALID_BUF_SIZE PAGE_SIZE
+static char invalid_insn_buf[INVALID_BUF_SIZE];
+
+static int __init bpf_init_invalid_insn_buf(void)
+{
+	jit_fill_hole(invalid_insn_buf, INVALID_BUF_SIZE);
+	return 0;
+}
+pure_initcall(bpf_init_invalid_insn_buf);
+
+void bpf_arch_invalidate_text(void *dst, size_t len)
+{
+	size_t i = 0;
+
+	while (i < len) {
+		size_t s = min_t(size_t, len - i, INVALID_BUF_SIZE);
+
+		bpf_arch_text_copy(dst + i, invalid_insn_buf, s);
+		i += s;
+	}
+}
+
 struct jit_context {
 	int cleanup_addr; /* Epilogue code offset */
 
include/linux/bpf.h +2 -0
index bdb5298735ce..2157392a94d1 100644
@@ -2382,6 +2382,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
 
 void *bpf_arch_text_copy(void *dst, void *src, size_t len);
 
+void bpf_arch_invalidate_text(void *dst, size_t len);
+
 struct btf_id_set;
 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
 
kernel/bpf/core.c +17 -4
index 13e9dbeeedf3..2be2d73fb6a8 100644
@@ -873,7 +873,7 @@ static size_t select_bpf_prog_pack_size(void)
 	return size;
 }
 
-static struct bpf_prog_pack *alloc_new_pack(void)
+static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
 {
 	struct bpf_prog_pack *pack;
 
@@ -886,6 +886,7 @@ static struct bpf_prog_pack *alloc_new_pack(void)
 		kfree(pack);
 		return NULL;
 	}
+	bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size);
 	bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE);
 	list_add_tail(&pack->list, &pack_list);
 
@@ -895,7 +896,7 @@ static struct bpf_prog_pack *alloc_new_pack(void)
 	return pack;
 }
 
-static void *bpf_prog_pack_alloc(u32 size)
+static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
 {
 	unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);
 	struct bpf_prog_pack *pack;
@@ -910,6 +911,7 @@ static void *bpf_prog_pack_alloc(u32 size)
 		size = round_up(size, PAGE_SIZE);
 		ptr = module_alloc(size);
 		if (ptr) {
+			bpf_fill_ill_insns(ptr, size);
 			set_vm_flush_reset_perms(ptr);
 			set_memory_ro((unsigned long)ptr, size / PAGE_SIZE);
 			set_memory_x((unsigned long)ptr, size / PAGE_SIZE);
@@ -923,7 +925,7 @@ static void *bpf_prog_pack_alloc(u32 size)
 			goto found_free_area;
 	}
 
-	pack = alloc_new_pack();
+	pack = alloc_new_pack(bpf_fill_ill_insns);
 	if (!pack)
 		goto out;
 
@@ -966,6 +968,7 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)
 	nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size);
 	pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT;
 
+	bpf_arch_invalidate_text(hdr, hdr->size);
 	bitmap_clear(pack->bitmap, pos, nbits);
 	if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,
 				       bpf_prog_chunk_count(), 0) == 0) {
@@ -1102,7 +1105,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,
 
 	if (bpf_jit_charge_modmem(size))
 		return NULL;
-	ro_header = bpf_prog_pack_alloc(size);
+	ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
 	if (!ro_header) {
 		bpf_jit_uncharge_modmem(size);
 		return NULL;
@@ -1203,6 +1206,16 @@ void __weak bpf_jit_free(struct bpf_prog *fp)
 	bpf_prog_unlock_free(fp);
 }
 
+void __weak bpf_arch_invalidate_text(void *dst, size_t len)
+{
+	char buf[1] = {};
+	int i;
+
+	WARN_ONCE(1, "Please override %s for bpf_prog_pack\n", __func__);
+	for (i = 0; i < len; i++)
+		bpf_arch_text_copy(dst + i, buf, 1);
+}
+
 int bpf_jit_get_func_addr(const struct bpf_prog *prog,
 			  const struct bpf_insn *insn, bool extra_pass,
 			  u64 *func_addr, bool *func_addr_fixed)
Signature · 1 line
2.30.2
author reviewer
#2 lore raw

PATCH 2/4 page_alloc: use vmalloc_huge for large system hash

In reply to Song Liu

Use vmalloc_huge() in alloc_large_system_hash() so that large system hash (>= PMD_SIZE) could benefit from huge pages. Note that vmalloc_huge only allocates huge pages for systems with HAVE_ARCH_HUGE_VMALLOC.

---
 mm/page_alloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
mm/page_alloc.c +1 -1
index 33ca8cab21e6..0e42038382c1 100644
@@ -8919,7 +8919,7 @@ void *__init alloc_large_system_hash(const char *tablename,
 				table = memblock_alloc_raw(size,
 							   SMP_CACHE_BYTES);
 		} else if (get_order(size) >= MAX_ORDER || hashdist) {
-			table = __vmalloc(size, gfp_flags);
+			table = vmalloc_huge(size, gfp_flags);
 			virt = true;
 			if (table)
 				huge = is_vm_area_hugepages(table);
Signature · 1 line
2.30.2
#3 lore raw

Re: [PATCH bpf 2/4] page_alloc: use vmalloc_huge for large system hash

I love your patch! Yet something to improve:

Hi Song,

I love your patch! Yet something to improve:

[auto build test ERROR on bpf/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master
config: i386-randconfig-a001 (https://download.01.org/0day-ci/archive/20220422/[email protected]/config)
compiler: gcc-11 (Debian 11.2.0-20) 11.2.0
reproduce (this is a W=1 build):
        # https://github.com/intel-lab-lkp/linux/commit/239fb9ca743cf33db8d56df7957726e19aea87d5
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605
        git checkout 239fb9ca743cf33db8d56df7957726e19aea87d5
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        make W=1 O=build_dir ARCH=i386 SHELL=/bin/bash

If you fix the issue, kindly add following tag as appropriate

All error/warnings (new ones prefixed by >>):

mm/page_alloc.c: In function 'alloc_large_system_hash':

>> mm/page_alloc.c:8921:33: error: implicit declaration of function 'vmalloc_huge'; did you mean 'vmalloc_no_huge'? [-Werror=implicit-function-declaration]
    8921 |                         table = vmalloc_huge(size, gfp_flags);
         |                                 ^~~~~~~~~~~~
         |                                 vmalloc_no_huge
>> mm/page_alloc.c:8921:31: warning: assignment to 'void *' from 'int' makes pointer from integer without a cast [-Wint-conversion]
    8921 |                         table = vmalloc_huge(size, gfp_flags);
         |                               ^

cc1: some warnings being treated as errors

vim +8921 mm/page_alloc.c

8876
8877 /* limit to 1 bucket per 2^scale bytes of low memory */
8878 if (scale > PAGE_SHIFT)
8879 numentries >>= (scale - PAGE_SHIFT);
8880 else
8881 numentries <<= (PAGE_SHIFT - scale);
8882
8883 /* Make sure we've got at least a 0-order allocation.. */
8884 if (unlikely(flags & HASH_SMALL)) {
8885 /* Makes no sense without HASH_EARLY */
8886 WARN_ON(!(flags & HASH_EARLY));
8887 if (!(numentries >> *_hash_shift)) {
8888 numentries = 1UL << *_hash_shift;
8889 BUG_ON(!numentries);
8890 }
8891 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
8892 numentries = PAGE_SIZE / bucketsize;
8893 }
8894 numentries = roundup_pow_of_two(numentries);
8895
8896 /* limit allocation size to 1/16 total memory by default */
8897 if (max == 0) {
8898 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8899 do_div(max, bucketsize);
8900 }
8901 max = min(max, 0x80000000ULL);
8902
8903 if (numentries < low_limit)
8904 numentries = low_limit;
8905 if (numentries > max)
8906 numentries = max;
8907
8908 log2qty = ilog2(numentries);
8909
8910 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
8911 do {
8912 virt = false;
8913 size = bucketsize << log2qty;
8914 if (flags & HASH_EARLY) {
8915 if (flags & HASH_ZERO)
8916 table = memblock_alloc(size, SMP_CACHE_BYTES);
8917 else
8918 table = memblock_alloc_raw(size,
8919 SMP_CACHE_BYTES);
8920 } else if (get_order(size) >= MAX_ORDER || hashdist) {

> 8921				table = vmalloc_huge(size, gfp_flags);

8922 virt = true;
8923 if (table)
8924 huge = is_vm_area_hugepages(table);
8925 } else {
8926 /*
8927 * If bucketsize is not a power-of-two, we may free
8928 * some pages at the end of hash table which
8929 * alloc_pages_exact() automatically does
8930 */
8931 table = alloc_pages_exact(size, gfp_flags);
8932 kmemleak_alloc(table, size, 1, gfp_flags);
8933 }
8934 } while (!table && size > PAGE_SIZE && --log2qty);
8935
8936 if (!table)
8937 panic("Failed to allocate %s hash table\n", tablename);
8938
8939 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8940 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
8941 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
8942
8943 if (_hash_shift)
8944 *_hash_shift = log2qty;
8945 if (_hash_mask)
8946 *_hash_mask = (1 << log2qty) - 1;
8947
8948 return table;
8949 }
8950

Signature · 2 lines
0-DAY CI Kernel Test Service
https://01.org/lkp
author reviewer
#4 lore raw

PATCH 3/4 module: introduce module_alloc_huge

In reply to Song Liu

Introduce module_alloc_huge, which allocates huge page backed memory in module memory space. The primary user of this memory is bpf_prog_pack (multiple BPF programs sharing a huge page).

---
 arch/x86/kernel/module.c     | 21 +++++++++++++++++++++
 include/linux/moduleloader.h |  5 +++++
 kernel/module.c              |  8 ++++++++
 3 files changed, 34 insertions(+)
arch/x86/kernel/module.c +21 -0
index b98ffcf4d250..63f6a16c70dc 100644
@@ -86,6 +86,27 @@ void *module_alloc(unsigned long size)
 	return p;
 }
 
+void *module_alloc_huge(unsigned long size)
+{
+	gfp_t gfp_mask = GFP_KERNEL;
+	void *p;
+
+	if (PAGE_ALIGN(size) > MODULES_LEN)
+		return NULL;
+
+	p = __vmalloc_node_range(size, MODULE_ALIGN,
+				 MODULES_VADDR + get_module_load_offset(),
+				 MODULES_END, gfp_mask, PAGE_KERNEL,
+				 VM_DEFER_KMEMLEAK | VM_ALLOW_HUGE_VMAP,
+				 NUMA_NO_NODE, __builtin_return_address(0));
+	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
+		vfree(p);
+		return NULL;
+	}
+
+	return p;
+}
+
 #ifdef CONFIG_X86_32
 int apply_relocate(Elf32_Shdr *sechdrs,
 		   const char *strtab,
include/linux/moduleloader.h +5 -0
index 9e09d11ffe5b..d34743a88938 100644
@@ -26,6 +26,11 @@ unsigned int arch_mod_section_prepend(struct module *mod, unsigned int section);
    sections.  Returns NULL on failure. */
 void *module_alloc(unsigned long size);
 
+/* Allocator used for allocating memory in module memory space. If size is
+ * greater than PMD_SIZE, allow using huge pages. Returns NULL on failure.
+ */
+void *module_alloc_huge(unsigned long size);
+
 /* Free memory returned from module_alloc. */
 void module_memfree(void *module_region);
 
kernel/module.c +8 -0
index 6cea788fd965..2af20ac3209c 100644
@@ -2839,6 +2839,14 @@ void * __weak module_alloc(unsigned long size)
 			NUMA_NO_NODE, __builtin_return_address(0));
 }
 
+void * __weak module_alloc_huge(unsigned long size)
+{
+	return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
+				    GFP_KERNEL, PAGE_KERNEL_EXEC,
+				    VM_FLUSH_RESET_PERMS | VM_ALLOW_HUGE_VMAP,
+				    NUMA_NO_NODE, __builtin_return_address(0));
+}
+
 bool __weak module_init_section(const char *name)
 {
 	return strstarts(name, ".init");
Signature · 1 line
2.30.2
#5 lore raw

Re: [PATCH bpf 3/4] module: introduce module_alloc_huge

I love your patch! Yet something to improve:

Hi Song,

I love your patch! Yet something to improve:

[auto build test ERROR on bpf/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master
config: x86_64-randconfig-a004 (https://download.01.org/0day-ci/archive/20220422/[email protected]/config)
compiler: gcc-11 (Debian 11.2.0-20) 11.2.0
reproduce (this is a W=1 build):
        # https://github.com/intel-lab-lkp/linux/commit/8a0dfde5aef7e95487be2f6e3ff9487d79a30714
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Song-Liu/bpf_prog_pack-and-vmalloc-on-huge-page-fixes/20220422-133605
        git checkout 8a0dfde5aef7e95487be2f6e3ff9487d79a30714
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

arch/x86/kernel/module.c: In function 'module_alloc_huge':

>> arch/x86/kernel/module.c:100:54: error: 'VM_ALLOW_HUGE_VMAP' undeclared (first use in this function); did you mean 'VM_NO_HUGE_VMAP'?
     100 |                                  VM_DEFER_KMEMLEAK | VM_ALLOW_HUGE_VMAP,
         |                                                      ^~~~~~~~~~~~~~~~~~
         |                                                      VM_NO_HUGE_VMAP

arch/x86/kernel/module.c:100:54: note: each undeclared identifier is reported only once for each function it appears in

vim +100 arch/x86/kernel/module.c

    88	
    89	void *module_alloc_huge(unsigned long size)
    90	{
    91		gfp_t gfp_mask = GFP_KERNEL;
    92		void *p;
    93	
    94		if (PAGE_ALIGN(size) > MODULES_LEN)
    95			return NULL;
    96	
    97		p = __vmalloc_node_range(size, MODULE_ALIGN,
    98					 MODULES_VADDR + get_module_load_offset(),
    99					 MODULES_END, gfp_mask, PAGE_KERNEL,
 > 100					 VM_DEFER_KMEMLEAK | VM_ALLOW_HUGE_VMAP,

101 NUMA_NO_NODE, __builtin_return_address(0));
102 if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
103 vfree(p);
104 return NULL;
105 }
106
107 return p;
108 }
109

Signature · 2 lines
0-DAY CI Kernel Test Service
https://01.org/lkp
author reviewer
#6 lore raw

PATCH 4/4 bpf: use module_alloc_huge for bpf_prog_pack

In reply to Song Liu

Use module_alloc_huge for bpf_prog_pack so that BPF programs sit on PMD_SIZE pages. This benefits system performance by reducing iTLB miss rate.

Also, remove set_vm_flush_reset_perms() from alloc_new_pack() and use set_memory_[nx|rw] in bpf_prog_pack_free(). This is because VM_FLUSH_RESET_PERMS does not work with huge pages yet.

---
 kernel/bpf/core.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
kernel/bpf/core.c +4 -3
index 2be2d73fb6a8..9acfa8e21fda 100644
@@ -857,7 +857,7 @@ static size_t select_bpf_prog_pack_size(void)
 	void *ptr;
 
 	size = BPF_HPAGE_SIZE * num_online_nodes();
-	ptr = module_alloc(size);
+	ptr = module_alloc_huge(size);
 
 	/* Test whether we can get huge pages. If not just use PAGE_SIZE
 	 * packs.
@@ -881,7 +881,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
 		       GFP_KERNEL);
 	if (!pack)
 		return NULL;
-	pack->ptr = module_alloc(bpf_prog_pack_size);
+	pack->ptr = module_alloc_huge(bpf_prog_pack_size);
 	if (!pack->ptr) {
 		kfree(pack);
 		return NULL;
@@ -890,7 +890,6 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
 	bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE);
 	list_add_tail(&pack->list, &pack_list);
 
-	set_vm_flush_reset_perms(pack->ptr);
 	set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
 	set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
 	return pack;
@@ -973,6 +972,8 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)
 	if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0,
 				       bpf_prog_chunk_count(), 0) == 0) {
 		list_del(&pack->list);
+		set_memory_nx((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
+		set_memory_rw((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE);
 		module_memfree(pack->ptr);
 		kfree(pack);
 	}
Signature · 1 line
2.30.2
maintainer
#7 lore raw
In reply to Song Liu
Quoted 6 lines · Song Liu

NOTE: This set is based on Linus' master branch (d569e86915b7), not bpf/master.

There are various discussion about these changes, check out [1] [2]. I guess we can use this thread to discuss which patches should go in 5.18. AFAICT, 1/4 need to with 5.18;

Since huge pages are effectively disabled on v5.18 I can't see why.

Quoted 1 line

2/4 seems safe to go as well;

My impression on the discussion was that huge pages design was broken and evidence for this came up after x86 finally enabled *a small* portion use case of it. This revealed how broken huge pages were not just for x86 but for other architectures. And so I can't see why we'd enable for v5.18 huge pages for the large system hash.

Quoted 2 lines

3/4 and 4/4
may still need more work/discussion.

Happy to review these but if huge pages are disabled I don't see the point in a module_alloc_huge() yet.

Luis

Quoted 23 lines

Thanks!

[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/

Song Liu (4):
bpf: invalidate unused part of bpf_prog_pack
page_alloc: use vmalloc_huge for large system hash
module: introduce module_alloc_huge
bpf: use module_alloc_huge for bpf_prog_pack

 arch/x86/kernel/module.c     | 21 +++++++++++++++++++++
 arch/x86/net/bpf_jit_comp.c  | 22 ++++++++++++++++++++++
 include/linux/bpf.h          |  2 ++
 include/linux/moduleloader.h |  5 +++++
 kernel/bpf/core.c            | 28 +++++++++++++++++++++-------
 kernel/module.c              |  8 ++++++++
 mm/page_alloc.c              |  2 +-
 7 files changed, 80 insertions(+), 8 deletions(-)

--
2.30.2