powerpc/64s/radix: Enable HAVE_ARCH_HUGE_VMAP
Subsystems (2)
Nicholas Piggin
reviewerPATCH 1/2 powerpc/64s/radix: Enable HAVE_ARCH_HUGE_VMAP
This sets the HAVE_ARCH_HUGE_VMAP option, and defines the required page table functions.
This will not enable huge iomaps, because powerpc/64 ioremap does not call ioremap_page_range. That is done in a later change.
HAVE_ARCH_HUGE_VMAP facilities will be used to enable huge pages for vmalloc memory in a set of generic kernel changes. Combined, this improves cached git diff performance by about 5% on a 2-node POWER9 with 32MB dentry cache hash, by allowing the dentry/inode hashes to be mapped with 2MB pages:
Profiling git diff dTLB misses with a vanilla kernel:
81.75% git [kernel.vmlinux] [k] __d_lookup_rcu 7.21% git [kernel.vmlinux] [k] strncpy_from_user 1.77% git [kernel.vmlinux] [k] find_get_entry 1.59% git [kernel.vmlinux] [k] kmem_cache_free
40,168 dTLB-miss
0.100342754 seconds time elapsedWith powerpc huge vmap and generic huge vmap vmalloc:
2,987 dTLB-miss
0.095933138 seconds time elapsed--- arch/powerpc/Kconfig | 1 + arch/powerpc/mm/book3s64/radix_pgtable.c | 93 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+)
arch/powerpc/Kconfig +1 -0
@@ -167,6 +167,7 @@ config PPC select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_HUGE_VMAP if PPC_BOOK3S_64 && PPC_RADIX_MMU select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if PPC32 select HAVE_ARCH_KGDB
arch/powerpc/mm/book3s64/radix_pgtable.c +93 -0
96 lines not on this page · raw message
Signature · 1 line
2.20.1
Nicholas Piggin
reviewerPATCH 2/2 powerpc/64s/radix: ioremap use huge page mappings
powerpc/64s does not use ioremap_page_range, so it does not get huge vmap iomap mappings automatically. The radix kernel mapping function already allows larger page mappings that work with huge vmap, so wire that up to allow huge pages to be used for ioremap mappings.
--- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 +++ arch/powerpc/mm/pgtable_64.c | 58 ++++++++++++++++++-- include/linux/io.h | 1 + lib/ioremap.c | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-)
arch/powerpc/include/asm/book3s/64/pgtable.h +8 -0
@@ -274,6 +274,14 @@ extern unsigned long __vmalloc_end;#define VMALLOC_START __vmalloc_start#define VMALLOC_END __vmalloc_end +static inline unsigned int ioremap_max_order (void ) +{ +if (radix_enabled ()) +return PUD_SHIFT ; +return 7 +PAGE_SHIFT ;/* default from linux/vmalloc.h */ +} +#define IOREMAP_MAX_ORDER ({ ioremap_max_order();}) +extern unsigned long __kernel_virt_start;extern unsigned long __kernel_virt_size;extern unsigned long __kernel_io_start;
arch/powerpc/mm/pgtable_64.c +52 -6
76 lines not on this page · raw message
include/linux/io.h +1 -0
@@ -44,6 +44,7 @@ static inline int ioremap_page_range(unsigned long addr, unsigned long end,#endif #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP +extern int ioremap_huge_disabled;void __init ioremap_huge_init (void );int arch_ioremap_pud_supported (void );int arch_ioremap_pmd_supported (void );
lib/ioremap.c +1 -1
@@ -18,7 +18,7 @@static int __read_mostly ioremap_p4d_capable;static int __read_mostly ioremap_pud_capable;static int __read_mostly ioremap_pmd_capable; -static int __read_mostly ioremap_huge_disabled; +int __read_mostly ioremap_huge_disabled;static int __init set_nohugeiomap (char *str) {
Signature · 1 line
2.20.1
Re: [PATCH 2/2] powerpc/64s/radix: ioremap use huge page mappings
Le 07/06/2019 à 08:19, Nicholas Piggin a écrit :
Quoted 4 lines
powerpc/64s does not use
ioremap_page_range, so it does not get huge vmap iomap mappings automatically. The radix kernel mapping function already allows larger page mappings that work with huge vmap, so wire that up to allow huge pages to be used for ioremap mappings.
Argh ... I was on the way to merge pgtable_64.c and pgtable_32.c. This will complicate the task ... Anyway this looks a good improvment.
Any reason to limit that to Radix ?
Quoted 14 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 +++ arch/powerpc/mm/pgtable_64.c | 58 ++++++++++++++++++-- include/linux/io.h | 1 + lib/ioremap.c | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-)arch/powerpc/include/asm/book3s/64/pgtable.h +0
-0
arch/powerpc/include/asm/book3s/64/pgtable.h extern unsigned long __vmalloc_end; · line 283
Quoted 10 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +7
-0@@ -274,6 +274,14 @@ extern unsigned long __vmalloc_end;#define VMALLOC_START __vmalloc_start#define VMALLOC_END __vmalloc_end +static inline unsigned int ioremap_max_order (void ) +{ +if (radix_enabled ()) +return PUD_SHIFT ; +return 7 +PAGE_SHIFT ;/* default from linux/vmalloc.h */ +} +#define IOREMAP_MAX_ORDER ({ ioremap_max_order();})
Following form doesn't work ?
#define IOREMAP_MAX_ORDER ioremap_max_order()
Quoted 9 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +1
-0+extern unsigned long __kernel_virt_start;extern unsigned long __kernel_virt_size;extern unsigned long __kernel_io_start;arch/powerpc/mm/pgtable_64.c +0
-0
arch/powerpc/mm/pgtable_64.c unsigned long ioremap_bot = IOREMAP_BASE; · line 115
Quoted 5 lines
arch/powerpc/mm/pgtable_64.c +1
-1@@ -112,7 +112,7 @@ unsigned long ioremap_bot = IOREMAP_BASE;* __ioremap_at - Low level function to establish the page tables * for an IO mapping */ -void __iomem *__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot) +static void __iomem *hash__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot)
Is this the correct name ?
As far as I understand, this function will be used by nohash/64, looks strange to call hash__something() a function used by nohash platforms.
Quoted 10 lines
arch/powerpc/mm/pgtable_64.c +3
-0{unsigned long i;@@ -120,6 +120,54 @@ void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_if (pgprot_val (prot) &H_PAGE_4K_PFN )return NULL ; +for (i =0 ; i < size; i +=PAGE_SIZE ) +if (map_kernel_page ((unsigned long )ea + i, pa + i, prot)) +return NULL ;
arch/powerpc/mm/pgtable_64.c void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_ · line 135
Quoted 10 lines
arch/powerpc/mm/pgtable_64.c +10
-0+ +return (void __iomem *)ea; +} + +static int radix__ioremap_page_range (unsigned long addr,unsigned long end, +phys_addr_t phys_addr,pgprot_t prot) +{ +while (addr != end) { +if (unlikely (ioremap_huge_disabled)) + goto use_small_page;
I don't like too much a goto in the middle of an if/else set inside a loop.
Couldn't we have two while() loops, one for the !ioremap_huge_disabled() and one for the ioremap_huge_disabled() case ? It would duplicate some code but that's only 3 small lines.
Or, when ioremap_huge_disabled(), couldn't it just fallback to the hash__ioremap_at() function ?
Quoted 39 lines
arch/powerpc/mm/pgtable_64.c +35
-0+ +if (!(addr & ~PUD_MASK ) && !(phys_addr & ~PUD_MASK ) && + end - addr >=PUD_SIZE ) { +if (radix__map_kernel_page (addr, phys_addr, prot,PUD_SIZE )) +return -ENOMEM ; + addr +=PUD_SIZE ; + phys_addr +=PUD_SIZE ; + + }else if (!(addr & ~PMD_MASK ) && !(phys_addr & ~PMD_MASK ) && + end - addr >=PMD_SIZE ) { +if (radix__map_kernel_page (addr, phys_addr, prot,PMD_SIZE )) +return -ENOMEM ; + addr +=PMD_SIZE ; + phys_addr +=PMD_SIZE ; + + }else { +use_small_page: +if (radix__map_kernel_page (addr, phys_addr, prot,PAGE_SIZE )) +return -ENOMEM ; + addr +=PAGE_SIZE ; + phys_addr +=PAGE_SIZE ; + } + } +return 0 ; +} + +static void __iomem *radix__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot) +{ +if (radix__ioremap_page_range ((unsigned long )ea, (unsigned long )ea + size, pa, prot)) +return NULL ; +return ea; +} + +void __iomem *__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot) +{if ((ea + size) >= (void *)IOREMAP_END ) {pr_warn ("Outside the supported range\n" );return NULL ;
arch/powerpc/mm/pgtable_64.c void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_ · line 180
Quoted 9 lines
arch/powerpc/mm/pgtable_64.c +1
-5@@ -129,11 +177,9 @@ void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_WARN_ON (((unsigned long )ea) & ~PAGE_MASK );WARN_ON (size & ~PAGE_MASK ); -for (i =0 ; i < size; i +=PAGE_SIZE ) -if (map_kernel_page ((unsigned long )ea + i, pa + i, prot)) -return NULL ; - -return (void __iomem *)ea; +if (radix_enabled ())
What about if (radix_enabled() && !ioremap_huge_disabled()) instead ?
arch/powerpc/mm/pgtable_64.c void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_ · line 182
Quoted 2 lines
arch/powerpc/mm/pgtable_64.c +2
-0+return radix__ioremap_at (pa, ea, size, prot); +return hash__ioremap_at (pa, ea, size, prot);
Can't we just leave the no radix stuff here instead of making that hash__ioremap_at() function ?
Christophe
Quoted 29 lines
arch/powerpc/mm/pgtable_64.c +0
-0}/** include/linux/io.h +1
-0@@ -44,6 +44,7 @@ static inline int ioremap_page_range(unsigned long addr, unsigned long end,#endif #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP +extern int ioremap_huge_disabled;void __init ioremap_huge_init (void );int arch_ioremap_pud_supported (void );int arch_ioremap_pmd_supported (void );lib/ioremap.c +1
-1@@ -18,7 +18,7 @@static int __read_mostly ioremap_p4d_capable;static int __read_mostly ioremap_pud_capable;static int __read_mostly ioremap_pmd_capable; -static int __read_mostly ioremap_huge_disabled; +int __read_mostly ioremap_huge_disabled;static int __init set_nohugeiomap (char *str) {
Nicholas Piggin
reviewerRe: [PATCH 2/2] powerpc/64s/radix: ioremap use huge page mappings
Christophe Leroy's on June 7, 2019 4:56 pm:
Quoted 10 lines
Le 07/06/2019 à 08:19, Nicholas Piggin a écrit :
Quoted 4 lines
powerpc/64s does not use
ioremap_page_range, so it does not get huge vmap iomap mappings automatically. The radix kernel mapping function already allows larger page mappings that work with huge vmap, so wire that up to allow huge pages to be used for ioremap mappings.Argh ... I was on the way to merge
pgtable_64.candpgtable_32.c. This will complicate the task ... Anyway this looks a good improvment.
I can put the radix code into book3s64, at least then you have less gunk in the common code.
Quoted 1 line
Any reason to limit that to Radix ?
Just that the others don't have a page size easily exposed for their map_kernel_page.
It would be nice to make this a bit more generic so other sub archs can enable the larger mappings just by implementing map_kernel_page.
Quoted 28 lines
Quoted 15 lines
Signed-off-by Nicholas Piggin <[email protected]>--- arch/powerpc/include/asm/book3s/64/pgtable.h | 8 +++ arch/powerpc/mm/pgtable_64.c | 58 ++++++++++++++++++-- include/linux/io.h | 1 + lib/ioremap.c | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-)arch/powerpc/include/asm/book3s/64/pgtable.h +0
-0arch/powerpc/include/asm/book3s/64/pgtable.h extern unsigned long __vmalloc_end; · line 283
Quoted 10 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +7
-0@@ -274,6 +274,14 @@ extern unsigned long __vmalloc_end;#define VMALLOC_START __vmalloc_start#define VMALLOC_END __vmalloc_end +static inline unsigned int ioremap_max_order (void ) +{ +if (radix_enabled ()) +return PUD_SHIFT ; +return 7 +PAGE_SHIFT ;/* default from linux/vmalloc.h */ +} +#define IOREMAP_MAX_ORDER ({ ioremap_max_order();})Following form doesn't work ?
Quoted 1 line
#define
IOREMAP_MAX_ORDERioremap_max_order()
I suppose it would. I'm not sure why I did that.
Quoted 17 lines
Quoted 9 lines
arch/powerpc/include/asm/book3s/64/pgtable.h +1
-0+extern unsigned long __kernel_virt_start;extern unsigned long __kernel_virt_size;extern unsigned long __kernel_io_start;arch/powerpc/mm/pgtable_64.c +0
-0arch/powerpc/mm/pgtable_64.c unsigned long ioremap_bot = IOREMAP_BASE; · line 115
Quoted 5 lines
arch/powerpc/mm/pgtable_64.c +1
-1@@ -112,7 +112,7 @@ unsigned long ioremap_bot = IOREMAP_BASE;* __ioremap_at - Low level function to establish the page tables * for an IO mapping */ -void __iomem *__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot) +static void __iomem *hash__ioremap_at (phys_addr_t pa,void *ea,unsigned long size,pgprot_t prot)Is this the correct name ?
Quoted 2 lines
As far as I understand, this function will be used by nohash/64, looks strange to call
hash__something()a function used by nohash platforms.
Yeah you're right, I'll fix that.
Quoted 23 lines
Quoted 10 lines
arch/powerpc/mm/pgtable_64.c +3
-0{unsigned long i;@@ -120,6 +120,54 @@ void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_if (pgprot_val (prot) &H_PAGE_4K_PFN )return NULL ; +for (i =0 ; i < size; i +=PAGE_SIZE ) +if (map_kernel_page ((unsigned long )ea + i, pa + i, prot)) +return NULL ;arch/powerpc/mm/pgtable_64.c void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_ · line 135
Quoted 10 lines
arch/powerpc/mm/pgtable_64.c +10
-0+ +return (void __iomem *)ea; +} + +static int radix__ioremap_page_range (unsigned long addr,unsigned long end, +phys_addr_t phys_addr,pgprot_t prot) +{ +while (addr != end) { +if (unlikely (ioremap_huge_disabled)) + goto use_small_page;I don't like too much a goto in the middle of an if/else set inside a loop.
Quoted 6 lines
Couldn't we have two
while()loops, one for the !ioremap_huge_disabled() and one for theioremap_huge_disabled()case ? It would duplicate some code but that's only 3 small lines.Or, when
ioremap_huge_disabled(), couldn't it just fallback to thehash__ioremap_at()function ?
Yeah okay. I'll see how the code looks after I move it into radix_pgtable.c, it might be best to keep all radix code there together even for the disabled case.
Quoted 5 lines
arch/powerpc/mm/pgtable_64.c void __iomem *__ioremap_at(phys_addr_t pa, void *ea, unsigned long size, pgprot_ · line 182
Quoted 2 lines
arch/powerpc/mm/pgtable_64.c +2
-0+return radix__ioremap_at (pa, ea, size, prot); +return hash__ioremap_at (pa, ea, size, prot);Can't we just leave the no radix stuff here instead of making that
hash__ioremap_at()function ?
Yeah that'll probably work better.
Thanks,
Nick