mm: large system hash use vmalloc for size > MAX_ORDER when !hashdist
Subsystems (4)
Nicholas Piggin
reviewerRFC PATCH 1/5 mm: large system hash use vmalloc for size > MAX_ORDER when !hashdist
The kernel currently clamps large system hashes to MAX_ORDER when hashdist is not set, which is rather arbitrary.
vmalloc space is limited on 32-bit machines, but this shouldn't result in much more used because of small physical memory.
--- mm/page_alloc.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
mm/page_alloc.c +3 -5
@@ -7978,7 +7978,7 @@ void *__init alloc_large_system_hash(const char *tablename,else table =memblock_alloc_raw (size,SMP_CACHE_BYTES ); - }else if (hashdist) { + }else if (get_order (size) >=MAX_ORDER || hashdist) { table =__vmalloc (size, gfp_flags,PAGE_KERNEL ); }else {/*
@@ -7986,10 +7986,8 @@ void *__init alloc_large_system_hash(const char *tablename,* some pages at the end of hash table which * alloc_pages_exact() automatically does */ -if (get_order (size) <MAX_ORDER ) { - table =alloc_pages_exact (size, gfp_flags); -kmemleak_alloc (table, size,1 , gfp_flags); - } + table =alloc_pages_exact (size, gfp_flags); +kmemleak_alloc (table, size,1 , gfp_flags); } }while (!table && size >PAGE_SIZE && --log2qty);
Signature · 1 line
2.20.1
Nicholas Piggin
reviewerRFC PATCH 2/5 mm: large system hash avoid vmap for non-NUMA machines when hashdist
hashdist currently always uses vmalloc when hashdist is true. When there is only 1 online node and size <= MAX_ORDER, vmalloc can be avoided.
--- mm/page_alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
mm/page_alloc.c +2 -1
@@ -7978,7 +7978,8 @@ void *__init alloc_large_system_hash(const char *tablename,else table =memblock_alloc_raw (size,SMP_CACHE_BYTES ); - }else if (get_order (size) >=MAX_ORDER || hashdist) { + }else if (get_order (size) >=MAX_ORDER || + (hashdist &&num_online_nodes () >1 )) { table =__vmalloc (size, gfp_flags,PAGE_KERNEL ); }else {/*
Signature · 1 line
2.20.1
Nicholas Piggin
reviewerRFC PATCH 3/5 mm/vmalloc: Hugepage vmalloc mappings
In reply to Nicholas PigginThis appears to help cached git diff performance by about 5% on a POWER9 (with 32MB dentry cache hash).
Profiling git diff dTLB misses with a vanilla kernel:
81.75% git [kernel.vmlinux] [k] __d_lookup_rcu 7.21% git [kernel.vmlinux] [k] strncpy_from_user 1.77% git [kernel.vmlinux] [k] find_get_entry 1.59% git [kernel.vmlinux] [k] kmem_cache_free
40,168 dTLB-miss
0.100342754 seconds time elapsedAfter this patch (and the subsequent powerpc HUGE_VMAP patches), the dentry cache hash gets mapped with 2MB pages:
2,987 dTLB-miss
0.095933138 seconds time elapsedelapsed time improvement isn't too scientific but seems consistent, TLB misses certainly improves an order of magnitude. My laptop takes a lot of misses here too, so x86 would be interesting to test, I think it should just work there.
--- include/linux/vmalloc.h | 1 + mm/vmalloc.c | 87 +++++++++++++++++++++++++++-------------- 2 files changed, 59 insertions(+), 29 deletions(-)
include/linux/vmalloc.h +1 -0
@@ -42,6 +42,7 @@ struct vm_struct {unsigned long size;unsigned long flags;struct page **pages; +unsigned int page_shift;unsigned int nr_pages;phys_addr_t phys_addr;const void *caller;
mm/vmalloc.c +58 -29
185 lines not on this page · raw message
Signature · 1 line
2.20.1
Nicholas Piggin
reviewerRFC PATCH 4/5 powerpc/64s/radix: Enable HAVE_ARCH_HUGE_VMAP
In reply to Nicholas PigginThis does not actually enable huge vmap mappings, because powerpc/64 ioremap does not call ioremap_page_range, but it is required before implementing huge mappings in ioremap, because the generic vunmap code needs to cope with them.
--- arch/powerpc/Kconfig | 1 + arch/powerpc/mm/book3s64/radix_pgtable.c | 93 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+)
arch/powerpc/Kconfig +1 -0
@@ -166,6 +166,7 @@ config PPC select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_HUGE_VMAP if PPC_BOOK3S_64 && PPC_RADIX_MMU select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if PPC32 select HAVE_ARCH_KGDB
arch/powerpc/mm/book3s64/radix_pgtable.c +93 -0
96 lines not on this page · raw message
Signature · 1 line
2.20.1
Nicholas Piggin
reviewerRFC PATCH 5/5 powerpc/64s/radix: iomap use huge page mappings
In reply to Nicholas Piggin--- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 +++ arch/powerpc/mm/pgtable_64.c | 54 +++++++++++++++++--- 2 files changed, 56 insertions(+), 6 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +8 -0
@@ -274,6 +274,14 @@ extern unsigned long __vmalloc_end;#define VMALLOC_START __vmalloc_start#define VMALLOC_END __vmalloc_end +static inline unsigned int ioremap_max_order (void ) +{ +if (radix_enabled ()) +return PUD_SHIFT ; +return 7 +PAGE_SHIFT ;/* default from linux/vmalloc.h */ +} +#define IOREMAP_MAX_ORDER ({ ioremap_max_order();}) +extern unsigned long __kernel_virt_start;extern unsigned long __kernel_virt_size;extern unsigned long __kernel_io_start;
arch/powerpc/mm/pgtable_64.c +48 -6
72 lines not on this page · raw message
Signature · 1 line
2.20.1