powerpc/percpu: Update percpu bootmem allocator
PATCH v2 2/4 powerpc/64/mm: implement page mapping percpu first chunk allocator
In reply to Aneesh Kumar K.VImplement page mapping percpu first chunk allocator as a fallback to the embedding allocator. With 4K hash translation we limit our page table range to 64TB and commit: 0034d395f89d ("powerpc/mm/hash64: Map all the kernel regions in the same 0xc range") moved all kernel mapping to that 64TB range. In-order to support sparse memory layout we need to increase our linear mapping space and reduce other mappings.
With such a layout percpu embedded first chunk allocator will fail because of small vmalloc range. Add a fallback to page mapping percpu first chunk allocator for such failures.
The below dmesg output can be observed in such case.
percpu: max_distance=0x1ffffef00000 too large for vmalloc space 0x10000000000
PERCPU: auto allocator failed (-22), falling back to page size
percpu: 40 4K pages/cpu s148816 r0 d15024
--- arch/powerpc/Kconfig | 5 ++- arch/powerpc/kernel/setup_64.c | 62 ++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-)
arch/powerpc/Kconfig +4 -1
@@ -59,7 +59,10 @@ config HAVE_SETUP_PER_CPU_AREA def_bool PPC64 config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool PPC64 + def_bool y if PPC64 + +config NEED_PER_CPU_PAGE_FIRST_CHUNK + def_bool y if PPC64 config NR_IRQS int "Number of virtual interrupt numbers"
arch/powerpc/kernel/setup_64.c +59 -3
86 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v2 1/4 powerpc/percpu: Update percpu bootmem allocator
This update the ppc64 version to be closer to x86/sparc.
--- arch/powerpc/kernel/setup_64.c | 45 ++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-)
arch/powerpc/kernel/setup_64.c +37 -8
@@ -753,17 +753,46 @@ void __init emergency_stack_init(void) }#ifdef CONFIG_SMP -#define PCPU_DYN_SIZE () - -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align) +/** +* pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu +* @cpu: cpu to allocate for +* @size: size allocation in bytes +* @align: alignment +* +* Allocate @size bytes aligned at @align for cpu @cpu. This wrapper +* does the right thing for NUMA regardless of the current +* configuration. +* +* RETURNS: +* Pointer to the allocated area on success, NULL on failure. +*/ +static void *__init pcpu_alloc_bootmem (unsigned int cpu,size_t size, +size_t align) { -return memblock_alloc_try_nid (size, align,__pa (MAX_DMA_ADDRESS ), -MEMBLOCK_ALLOC_ACCESSIBLE , -early_cpu_to_node (cpu)); +const unsigned long goal =__pa (MAX_DMA_ADDRESS ); +#ifdef CONFIG_NEED_MULTIPLE_NODES +int node =early_cpu_to_node (cpu); +void *ptr; +if (!node_online (node) || !NODE_DATA (node)) { + ptr =memblock_alloc_from (size, align, goal); +pr_info ("cpu %d has no node %d or node-local memory\n" , + cpu, node); +pr_debug ("per cpu data for cpu%d %lu bytes at %016lx\n" , + cpu, size,__pa (ptr)); + }else { + ptr =memblock_alloc_try_nid (size, align, goal, +MEMBLOCK_ALLOC_ACCESSIBLE , node); +pr_debug ("per cpu data for cpu%d %lu bytes on node%d at " +"%016lx\n" , cpu, size, node,__pa (ptr)); + } +return ptr; +#else +return memblock_alloc_from (size, align, goal); +#endif } -static void __init pcpu_fc_free (void *ptr,size_t size) +static void __init pcpu_free_bootmem (void *ptr,size_t size) {memblock_free (__pa (ptr), size); }
@@ -798,7 +827,7 @@ void __init setup_per_cpu_areas(void) atom_size =1 <<20 ; rc =pcpu_embed_first_chunk (0 , dyn_size, atom_size, pcpu_cpu_distance, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_alloc_bootmem, pcpu_free_bootmem);if (rc <0 )panic ("cannot initialize percpu area (err=%d)" , rc);
Signature · 1 line
2.26.2
PATCH v2 4/4 powerpc/mm/book3s: Split radix and hash MAX_PHYSMEM limit
MAX_PHYSMEM #define is used along with sparsemem to determine the SECTION_SHIFT value. Powerpc also uses the same value to limit the max memory enabled on the system. With 4K PAGE_SIZE and hash translation mode, we want to limit the max memory enabled to 64TB due to page table size restrictions. However, with radix translation, we don't have these restrictions. Hence split the radix and hash MA_PHYSMEM limit and use different limit for each of them.
--- arch/powerpc/include/asm/book3s/64/hash-4k.h | 5 +++++ arch/powerpc/include/asm/book3s/64/hash-64k.h | 13 +++++++++++++ arch/powerpc/include/asm/book3s/64/mmu-hash.h | 4 ++-- arch/powerpc/include/asm/book3s/64/mmu.h | 15 --------------- arch/powerpc/include/asm/book3s/64/pgtable.h | 7 +++++++ arch/powerpc/include/asm/book3s/64/radix.h | 16 ++++++++++++++++ arch/powerpc/kernel/prom.c | 5 +++++ arch/powerpc/mm/book3s64/slb.c | 4 ++-- 8 files changed, 50 insertions(+), 19 deletions(-)
arch/powerpc/include/asm/book3s/64/hash-4k.h +5 -0
@@ -22,6 +22,11 @@#define REGION_SHIFT (40)#define H_KERN_MAP_SIZE (ASM_CONST(1) << REGION_SHIFT) +/* +* Limits the linear mapping range +*/ +#define H_MAX_PHYSMEM_BITS 46 +/* * Define the address range of the kernel non-linear virtual area (61TB) */
arch/powerpc/include/asm/book3s/64/hash-64k.h +13 -0
@@ -7,6 +7,19 @@#define H_PUD_INDEX_SIZE 10 // size: 8B << 10 = 8KB, maps 2^10 x 16GB = 16TB#define H_PGD_INDEX_SIZE 8 // size: 8B << 8 = 2KB, maps 2^8 x 16TB = 4PB +/* +* If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS +* if we increase SECTIONS_WIDTH we will not store node details in page->flags and +* page_to_nid does a page->section->node lookup +* Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce +* memory requirements with large number of sections. +* 51 bits is the max physical real address on POWER9 +*/ +#if defined(CONFIG_SPARSEMEM_VMEMMAP ) && defined(CONFIG_SPARSEMEM_EXTREME ) +#define H_MAX_PHYSMEM_BITS 51 +#else +#define H_MAX_PHYSMEM_BITS 46 +#endif /* * Each context is 512TB size. SLB miss for first context/default context
arch/powerpc/include/asm/book3s/64/mmu-hash.h +2 -2
@@ -577,8 +577,8 @@ extern void slb_set_size(u16 size);* For vmalloc and memmap, we use just one context with 512TB. With 64 byte * struct page size, we need ony 32 TB in memmap for 2PB (51 bits (MAX_PHYSMEM_BITS)). */ -#if (MAX_PHYSMEM_BITS >MAX_EA_BITS_PER_CONTEXT ) -#define MAX_KERNEL_CTX_CNT (1UL << (MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT)) +#if (H_MAX_PHYSMEM_BITS >MAX_EA_BITS_PER_CONTEXT ) +#define MAX_KERNEL_CTX_CNT (1UL << (H_MAX_PHYSMEM_BITS - MAX_EA_BITS_PER_CONTEXT))#else #define MAX_KERNEL_CTX_CNT 1#endif
arch/powerpc/include/asm/book3s/64/mmu.h +0 -15
@@ -27,21 +27,6 @@ struct mmu_psize_def {extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT ];#endif /* __ASSEMBLY__ */ -/* -* If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS -* if we increase SECTIONS_WIDTH we will not store node details in page->flags and -* page_to_nid does a page->section->node lookup -* Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce -* memory requirements with large number of sections. -* 51 bits is the max physical real address on POWER9 -*/ -#if defined(CONFIG_SPARSEMEM_VMEMMAP ) && defined(CONFIG_SPARSEMEM_EXTREME ) && \ - defined(CONFIG_PPC_64K_PAGES ) -#define MAX_PHYSMEM_BITS 51 -#else -#define MAX_PHYSMEM_BITS 46 -#endif -/* 64-bit classic hash table MMU */ #include <asm/book3s/64/mmu-hash.h>
arch/powerpc/include/asm/book3s/64/pgtable.h +7 -0
@@ -295,6 +295,13 @@ extern unsigned long pci_io_base;#include <asm/book3s/64/hash.h> #include <asm/book3s/64/radix.h> +#if H_MAX_PHYSMEM_BITS >R_MAX_PHYSMEM_BITS +#define MAX_PHYSMEM_BITS H_MAX_PHYSMEM_BITS +#else +#define MAX_PHYSMEM_BITS R_MAX_PHYSMEM_BITS +#endif + +#ifdef CONFIG_PPC_64K_PAGES #include <asm/book3s/64/pgtable-64k.h> #else
arch/powerpc/include/asm/book3s/64/radix.h +16 -0
@@ -91,6 +91,22 @@* +------------------------------+ Kernel linear (0xc.....) */ + +/* +* If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS +* if we increase SECTIONS_WIDTH we will not store node details in page->flags and +* page_to_nid does a page->section->node lookup +* Hence only increase for VMEMMAP. Further depending on SPARSEMEM_EXTREME reduce +* memory requirements with large number of sections. +* 51 bits is the max physical real address on POWER9 +*/ + +#if defined(CONFIG_SPARSEMEM_VMEMMAP ) && defined(CONFIG_SPARSEMEM_EXTREME ) +#define R_MAX_PHYSMEM_BITS 51 +#else +#define R_MAX_PHYSMEM_BITS 46 +#endif +#define RADIX_KERN_VIRT_START ASM_CONST(0xc008000000000000)/* * 49 = MAX_EA_BITS_PER_CONTEXT (hash specific). To make sure we pick
arch/powerpc/kernel/prom.c +5 -0
@@ -771,6 +771,11 @@ void __init early_init_devtree(void *params) limit =ALIGN (memory_limit ?:memblock_phys_mem_size (),PAGE_SIZE );memblock_enforce_memory_limit (limit); +#if defined(CONFIG_PPC_BOOK3S_64 ) && defined(CONFIG_PPC_4K_PAGES ) +if (!early_radix_enabled ()) +memblock_cap_memory_range (0 ,1UL << (H_MAX_PHYSMEM_BITS )); +#endif +memblock_allow_resize ();memblock_dump_all ();
arch/powerpc/mm/book3s64/slb.c +2 -2
@@ -765,8 +765,8 @@ static long slb_allocate_kernel(unsigned long ea, unsigned long id)if (id ==LINEAR_MAP_REGION_ID ) { -/* We only support upto MAX_PHYSMEM_BITS */ -if ((ea &EA_MASK ) > (1UL <<MAX_PHYSMEM_BITS )) +/* We only support upto H_MAX_PHYSMEM_BITS */ +if ((ea &EA_MASK ) > (1UL <<H_MAX_PHYSMEM_BITS ))return -EFAULT ; flags =SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp;
Signature · 1 line
2.26.2
PATCH v2 3/4 powerpc/book3s64/hash/4k: Support large linear mapping range with 4K
In reply to Aneesh Kumar K.VWith commit: 0034d395f89d ("powerpc/mm/hash64: Map all the kernel regions in the same 0xc range"), we now split the 64TB address range into 4 contexts each of 16TB. That implies we can do only 16TB linear mapping.
On some systems, eg. Power9, memory attached to nodes > 0 will appear above 16TB in the linear mapping. This resulted in kernel crash when we boot such systems in hash translation mode with 4K PAGE_SIZE.
This patch updates the kernel mapping such that we now start supporting upto 61TB of memory with 4K. The kernel mapping now looks like below 4K PAGE_SIZE and hash translation.
vmalloc start = 0xc0003d0000000000
IO start = 0xc0003e0000000000
vmemmap start = 0xc0003f0000000000Our MAX_PHYSMEM_BITS for 4K is still 64TB even though we can only map 61TB. We prevent bolt mapping anything outside 61TB range by checking against H_VMALLOC_START.
--- arch/powerpc/include/asm/book3s/64/hash-4k.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
arch/powerpc/include/asm/book3s/64/hash-4k.h +6 -7
@@ -13,20 +13,19 @@*/ #define MAX_EA_BITS_PER_CONTEXT 46 -#define REGION_SHIFT (MAX_EA_BITS_PER_CONTEXT - 2)/* -* Our page table limit us to 64TB. Hence for the kernel mapping, -* each MAP area is limited to 16 TB. -* The four map areas are: linear mapping, vmap, IO and vmemmap +* Our page table limit us to 64TB. For 64TB physical memory, we only need 64GB +* of vmemmap space. To better support sparse memory layout, we use 61TB +* linear map range, 1TB of vmalloc, 1TB of I/O and 1TB of vmememmap. */ +#define REGION_SHIFT (40)#define H_KERN_MAP_SIZE (ASM_CONST(1) << REGION_SHIFT)/* -* Define the address range of the kernel non-linear virtual area -* 16TB +* Define the address range of the kernel non-linear virtual area (61TB) */ -#define H_KERN_VIRT_START ASM_CONST(0xc000100000000000) +#define H_KERN_VIRT_START ASM_CONST(0xc0003d0000000000)#ifndef __ASSEMBLY__#define H_PTE_TABLE_SIZE (sizeof(pte_t) << H_PTE_INDEX_SIZE)
Signature · 1 line
2.26.2
Re: [PATCH v2 2/4] powerpc/64/mm: implement page mapping percpu first chunk allocator
In reply to Aneesh Kumar K.VQuoted 11 lines · Aneesh Kumar K.V
Implement page mapping percpu first chunk allocator as a fallback to the embedding allocator. With 4K hash translation we limit our page table range to 64TB and commit:
0034d395f89d("powerpc/mm/hash64: Map all the kernel regions in the same0xcrange") moved all kernel mapping to that 64TB range. In-order to support sparse memory layout we need to increase our linear mapping space and reduce other mappings.With such a layout percpu embedded first chunk allocator will fail because of small vmalloc range. Add a fallback to page mapping percpu first chunk allocator for such failures.
Quoted 6 lines
The below dmesg output can be observed in such case.
percpu: max_distance=0x1ffffef00000 too large for vmalloc space
0x10000000000
PERCPU: auto allocator failed (-22), falling back to page size
percpu: 40 4K pages/cpu s148816 r0 d15024
This patch requires powersave=off kernel command line to boot. We are working to make sure we don't access per cpu variables in real mode. Additionally, you can also try the below workaround patch
modified arch/powerpc/kernel/mce.c
@@ -711,7 +711,7 @@ long hmi_exception_realmode(struct pt_regs *regs)
{
int ret;
- __this_cpu_inc(irq_stat.hmi_exceptions);
+// __this_cpu_inc(irq_stat.hmi_exceptions);
ret = hmi_handle_debugtrig(regs);
if (ret >= 0)
-aneesh
Michael Ellerman
AppliedQuoted 1 line · Aneesh Kumar K.V
This update the ppc64 version to be closer to x86/sparc.
Applied to powerpc/next.
[1/4] powerpc/percpu: Update percpu bootmem allocator
https://git.kernel.org/powerpc/c/2a32abac8860aa1c3a1fc99973ce67179575b36c [2/4] powerpc/64/mm: implement page mapping percpu first chunk allocator https://git.kernel.org/powerpc/c/eb553f16973ade990d05946af9ae191394712c8a [3/4] powerpc/book3s64/hash/4k: Support large linear mapping range with 4K https://git.kernel.org/powerpc/c/7746406baa3bc9e23fdd7b7da2f04d86e25ab837 [4/4] powerpc/mm/book3s: Split radix and hash MAX_PHYSMEM limit https://git.kernel.org/powerpc/c/b32d5d7e920a364287f6206af2d20179978a617d
cheers