mm: percpu: Cleanup percpu first chunk function
Subsystems (10)
cover mm: percpu: Cleanup percpu first chunk function
When support page mapping percpu first chunk allocator on arm64, we found there are lots of duplicated codes in percpu embed/page first chunk allocator. This patchset is aimed to cleanup them and should no function change.
The currently supported status about 'embed' and 'page' in Archs shows below,
embed: NEED_PER_CPU_PAGE_FIRST_CHUNK page: NEED_PER_CPU_EMBED_FIRST_CHUNK
embed page
------------------------
arm64 Y Y
mips Y N
powerpc Y Y
riscv Y N
sparc Y Y
x86 Y Y
------------------------
There are two interfaces about percpu first chunk allocator,
extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
- pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn); + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);
extern int __init pcpu_page_first_chunk(size_t reserved_size,
- pcpu_fc_alloc_fn_t alloc_fn, - pcpu_fc_free_fn_t free_fn, - pcpu_fc_populate_pte_fn_t populate_pte_fn); + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);
The pcpu_fc_alloc_fn_t/pcpu_fc_free_fn_t is killed, we provide generic pcpu_fc_alloc() and pcpu_fc_free() function, which are called in the pcpu_embed/page_first_chunk().
1) For pcpu_embed_first_chunk(), pcpu_fc_cpu_to_node_fn_t is needed to be provided when archs supported NUMA.
2) For pcpu_page_first_chunk(), the pcpu_fc_populate_pte_fn_t is killed too, a generic pcpu_populate_pte() which marked '__weak' is provided, if you need a different function to populate pte on the arch(like x86), please provide its own implementation.
I have been built test on arm64/ia64/mips/powerpc/sparcx86/riscv based on v5.16-rc4 on my machine, also all patches are in[1], which checked by lkp too,
[kevin78:percpu-cleanup] BUILD SUCCESS c14a59a0aad7db88ebddbeb5e914ddcccb406e1c.
[1] https://github.com/kevin78/linux.git percpu-cleanup
Changes since RFC
- Address Dennis's comments
RFC:
https://lore.kernel.org/linux-mm/[email protected]/T/
Kefeng Wang (4):
mm: percpu: Generalize percpu related config
mm: percpu: Add pcpu_fc_cpu_to_node_fn_t typedef
mm: percpu: Add generic pcpu_fc_alloc/free funciton
mm: percpu: Add generic pcpu_populate_pte() function
arch/arm64/Kconfig | 20 +---- arch/ia64/Kconfig | 9 +- arch/mips/Kconfig | 10 +-- arch/mips/mm/init.c | 14 +-- arch/powerpc/Kconfig | 17 +--- arch/powerpc/kernel/setup_64.c | 97 ++------------------- arch/riscv/Kconfig | 10 +-- arch/sparc/Kconfig | 12 +-- arch/sparc/kernel/smp_64.c | 103 +--------------------- arch/x86/Kconfig | 17 +--- arch/x86/kernel/setup_percpu.c | 66 ++------------ drivers/base/arch_numa.c | 68 +-------------- include/linux/percpu.h | 13 +-- mm/Kconfig | 12 +++ mm/percpu.c | 154 +++++++++++++++++++++++++-------- 15 files changed, 180 insertions(+), 442 deletions(-)
Signature · 1 line
2.18.0.huawei.25
PATCH 1/4 mm: percpu: Generalize percpu related config
The HAVE_SETUP_PER_CPU_AREA/NEED_PER_CPU_EMBED_FIRST_CHUNK/ NEED_PER_CPU_PAGE_FIRST_CHUNK/USE_PERCPU_NUMA_NODE_ID configs, which have duplicate definitions on platforms that subscribe it.
Move them into mm, drop these redundant definitions and instead just select it on applicable platforms.
--- arch/arm64/Kconfig | 20 ++++---------------- arch/ia64/Kconfig | 9 ++------- arch/mips/Kconfig | 10 ++-------- arch/powerpc/Kconfig | 17 ++++------------- arch/riscv/Kconfig | 10 ++-------- arch/sparc/Kconfig | 12 +++--------- arch/x86/Kconfig | 17 ++++------------- mm/Kconfig | 12 ++++++++++++ 8 files changed, 33 insertions(+), 74 deletions(-)
arch/arm64/Kconfig +4 -16
@@ -1135,6 +1135,10 @@ config NUMA select GENERIC_ARCH_NUMA select ACPI_NUMA if ACPI select OF_NUMA + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support.
@@ -1151,22 +1155,6 @@ config NODES_SHIFT Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - -config HAVE_SETUP_PER_CPU_AREA - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y - depends on NUMA - source "kernel/Kconfig.hz" config ARCH_SPARSEMEM_ENABLE
arch/ia64/Kconfig +2 -7
@@ -32,6 +32,7 @@ config IA64 select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE if (!ITANIUM) select HAVE_FUNCTION_TRACER + select HAVE_SETUP_PER_CPU_AREA select TTY select HAVE_ARCH_TRACEHOOK select HAVE_VIRT_CPU_ACCOUNTING
@@ -88,9 +89,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config HAVE_SETUP_PER_CPU_AREA - def_bool y - config DMI bool default y
@@ -292,6 +290,7 @@ config NUMA bool "NUMA support" depends on !FLATMEM select SMP + select USE_PERCPU_NUMA_NODE_ID help Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option is for configuring high-end multiprocessor
@@ -311,10 +310,6 @@ config HAVE_ARCH_NODEDATA_EXTENSION def_bool y depends on NUMA -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - config HAVE_MEMORYLESS_NODES def_bool NUMA
arch/mips/Kconfig +2 -8
@@ -2666,6 +2666,8 @@ config NUMA bool "NUMA Support" depends on SYS_SUPPORTS_NUMA select SMP + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK help Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option improves performance on systems with more
@@ -2676,14 +2678,6 @@ config NUMA config SYS_SUPPORTS_NUMA bool -config HAVE_SETUP_PER_CPU_AREA - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - config RELOCATABLE bool "Relocatable kernel" depends on SYS_SUPPORTS_RELOCATABLE
arch/powerpc/Kconfig +4 -13
@@ -55,15 +55,6 @@ config ARCH_MMAP_RND_COMPAT_BITS_MIN default 9 if PPC_16K_PAGES # 9 = 23 (8MB) - 14 (16K) default 11 # 11 = 23 (8MB) - 12 (4K) -config HAVE_SETUP_PER_CPU_AREA - def_bool PPC64 - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y if PPC64 - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y if PPC64 - config NR_IRQS int "Number of virtual interrupt numbers" range 32 1048576
@@ -240,6 +231,7 @@ config PPC select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ + select HAVE_SETUP_PER_CPU_AREA if PPC64 select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
@@ -254,6 +246,8 @@ config PPC select MMU_GATHER_RCU_TABLE_FREE select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PPC64 || NOT_COHERENT_CACHE + select NEED_PER_CPU_EMBED_FIRST_CHUNK if PPC64 + select NEED_PER_CPU_PAGE_FIRST_CHUNK if PPC64 select NEED_SG_DMA_LENGTH select OF select OF_DMA_DEFAULT_COHERENT if !NOT_COHERENT_CACHE
@@ -659,6 +653,7 @@ config NUMA bool "NUMA Memory Allocation and Scheduler Support" depends on PPC64 && SMP default y if PPC_PSERIES || PPC_POWERNV + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support.
@@ -672,10 +667,6 @@ config NODES_SHIFT default "4" depends on NUMA -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - config HAVE_MEMORYLESS_NODES def_bool y depends on NUMA
arch/riscv/Kconfig +2 -8
@@ -334,6 +334,8 @@ config NUMA select GENERIC_ARCH_NUMA select OF_NUMA select ARCH_SUPPORTS_NUMA_BALANCING + select USE_PERCPU_NUMA_NODE_ID + select NEED_PER_CPU_EMBED_FIRST_CHUNK help Enable NUMA (Non-Uniform Memory Access) support.
@@ -349,14 +351,6 @@ config NODES_SHIFT Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - depends on NUMA - config RISCV_ISA_C bool "Emit compressed instructions when building Linux" default y
arch/sparc/Kconfig +3 -9
@@ -97,6 +97,9 @@ config SPARC64 select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE select HAVE_SOFTIRQ_ON_OWN_STACK + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK config ARCH_PROC_KCORE_TEXT def_bool y
@@ -123,15 +126,6 @@ config AUDIT_ARCH bool default y -config HAVE_SETUP_PER_CPU_AREA - def_bool y if SPARC64 - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y if SPARC64 - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y if SPARC64 - config MMU bool default y
arch/x86/Kconfig +4 -13
@@ -239,6 +239,7 @@ config X86 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_SETUP_PER_CPU_AREA select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64
@@ -252,6 +253,8 @@ config X86 select HAVE_GENERIC_VDSO select HOTPLUG_SMT if SMP select IRQ_FORCED_THREADING + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK select NEED_SG_DMA_LENGTH select PCI_DOMAINS if PCI select PCI_LOCKLESS_CONFIG if PCI
@@ -331,15 +334,6 @@ config ARCH_HAS_CPU_RELAX config ARCH_HAS_FILTER_PGPROT def_bool y -config HAVE_SETUP_PER_CPU_AREA - def_bool y - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y - config ARCH_HIBERNATION_POSSIBLE def_bool y
@@ -1557,6 +1551,7 @@ config NUMA depends on SMP depends on X86_64 || (X86_32 && HIGHMEM64G && X86_BIGSMP) default y if X86_BIGSMP + select USE_PERCPU_NUMA_NODE_ID help Enable NUMA (Non-Uniform Memory Access) support.
@@ -2430,10 +2425,6 @@ config ARCH_HAS_ADD_PAGES config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE def_bool y -config USE_PERCPU_NUMA_NODE_ID - def_bool y - depends on NUMA - menu "Power management and ACPI options" config ARCH_HIBERNATION_HEADER
mm/Kconfig +12 -0
@@ -432,6 +432,18 @@ config NEED_PER_CPU_KM bool default y +config NEED_PER_CPU_EMBED_FIRST_CHUNK + bool + +config NEED_PER_CPU_PAGE_FIRST_CHUNK + bool + +config USE_PERCPU_NUMA_NODE_ID + bool + +config HAVE_SETUP_PER_CPU_AREA + bool + config CLEANCACHE bool "Enable cleancache driver to cache clean pages if tmem is present" help
Signature · 1 line
2.18.0.huawei.25
PATCH 2/4 mm: percpu: Add pcpu_fc_cpu_to_node_fn_t typedef
In reply to Kefeng WangAdd pcpu_fc_cpu_to_node_fn_t and pass it into pcpu_fc_alloc_fn_t, pcpu first chunk allocation will call it to alloc memblock on the corresponding node by it, this is prepare for the next patch.
--- arch/mips/mm/init.c | 12 +++++++++--- arch/powerpc/kernel/setup_64.c | 15 +++++++++++---- arch/sparc/kernel/smp_64.c | 13 ++++++++++--- arch/x86/kernel/setup_percpu.c | 18 +++++++++++++----- drivers/base/arch_numa.c | 8 +++++--- include/linux/percpu.h | 7 +++++-- mm/percpu.c | 14 +++++++++----- 7 files changed, 62 insertions(+), 25 deletions(-)
arch/mips/mm/init.c +9 -3
@@ -519,12 +519,17 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)return node_distance (cpu_to_node (from),cpu_to_node (to)); } -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size, -size_t align) +static int __init pcpu_cpu_to_node (int cpu) +{ +return cpu_to_node (cpu); +} + +static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) {return memblock_alloc_try_nid (size, align,__pa (MAX_DMA_ADDRESS ),MEMBLOCK_ALLOC_ACCESSIBLE , -cpu_to_node (cpu)); +cpu_to_nd_fn (cpu)); }static void __init pcpu_fc_free (void *ptr,size_t size)
@@ -545,6 +550,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE ,PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE , pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free);if (rc <0 )panic ("Failed to initialize percpu areas." );
arch/powerpc/kernel/setup_64.c +11 -4
@@ -784,12 +784,12 @@ void __init emergency_stack_init(void)* RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void *__init pcpu_alloc_bootmem (unsigned int cpu,size_t size, -size_t align) +static void *__init pcpu_alloc_bootmem (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) {const unsigned long goal =__pa (MAX_DMA_ADDRESS );#ifdef CONFIG_NUMA -int node =early_cpu_to_node (cpu); +int node =cpu_to_nd_fun (cpu);void *ptr;if (!node_online (node) || !NODE_DATA (node)) {
@@ -823,6 +823,11 @@ static int pcpu_cpu_distance(unsigned int from, unsigned int to)return REMOTE_DISTANCE ; } +static __init int pcpu_cpu_to_node (int cpu) +{ +return early_cpu_to_node (cpu); +} +unsigned long __per_cpu_offset[NR_CPUS ]__read_mostly ;EXPORT_SYMBOL (__per_cpu_offset);
@@ -891,6 +896,7 @@ void __init setup_per_cpu_areas(void)if (pcpu_chosen_fc !=PCPU_FC_PAGE ) { rc =pcpu_embed_first_chunk (0 , dyn_size, atom_size, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem);if (rc) pr_warn("PERCPU: %s allocator failed (%d), "
@@ -899,7 +905,8 @@ void __init setup_per_cpu_areas(void) }if (rc <0 ) - rc =pcpu_page_first_chunk (0 , pcpu_alloc_bootmem, pcpu_free_bootmem, + rc =pcpu_page_first_chunk (0 , pcpu_cpu_to_node, + pcpu_alloc_bootmem, pcpu_free_bootmem, pcpu_populate_pte);if (rc <0 )panic ("cannot initialize percpu area (err=%d)" , rc);
arch/sparc/kernel/smp_64.c +10 -3
@@ -1539,12 +1539,12 @@ void smp_send_stop(void)* RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void *__init pcpu_alloc_bootmem (unsigned int cpu,size_t size, -size_t align) +static void *__init pcpu_alloc_bootmem (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) {const unsigned long goal =__pa (MAX_DMA_ADDRESS );#ifdef CONFIG_NUMA -int node =cpu_to_node (cpu); +int node =cpu_to_nd_fn (cpu);void *ptr;if (!node_online (node) || !NODE_DATA (node)) {
@@ -1578,6 +1578,11 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)return REMOTE_DISTANCE ; } +static int __init pcpu_cpu_to_node (int cpu) +{ +return cpu_to_node (cpu); +} +static void __init pcpu_populate_pte (unsigned long addr) {pgd_t *pgd =pgd_offset_k (addr);
@@ -1641,6 +1646,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE ,PERCPU_DYNAMIC_RESERVE ,4 <<20 , pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem);if (rc)
@@ -1650,6 +1656,7 @@ void __init setup_per_cpu_areas(void) }if (rc <0 ) rc =pcpu_page_first_chunk (PERCPU_MODULE_RESERVE , + pcpu_cpu_to_node, pcpu_alloc_bootmem, pcpu_free_bootmem, pcpu_populate_pte);
arch/x86/kernel/setup_percpu.c +13 -5
@@ -97,12 +97,12 @@ static bool __init pcpu_need_numa(void)* RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void *__init pcpu_alloc_bootmem (unsigned int cpu,unsigned long size, -unsigned long align) +static void *__init pcpu_alloc_bootmem (unsigned int cpu,unsigned long size,unsigned long align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) {const unsigned long goal =__pa (MAX_DMA_ADDRESS );#ifdef CONFIG_NUMA -int node =early_cpu_to_node (cpu); +int node =cpu_to_nd_fn (cpu);void *ptr;if (!node_online (node) || !NODE_DATA (node)) {
@@ -128,9 +128,10 @@ static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,/* * Helpers for first chunk memory allocation */ -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align) +static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { -return pcpu_alloc_bootmem (cpu, size, align); +return pcpu_alloc_bootmem (cpu, size, align, cpu_to_nd_fn); }static void __init pcpu_fc_free (void *ptr,size_t size)
@@ -150,6 +151,11 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)#endif } +static int __init pcpu_cpu_to_node (int cpu) +{ +return early_cpu_to_node (cpu); +} +static void __init pcpup_populate_pte (unsigned long addr) {populate_extra_pte (addr);
@@ -205,6 +211,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_FIRST_CHUNK_RESERVE , dyn_size, atom_size, pcpu_cpu_distance, + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free);if (rc <0 ) pr_warn("%s allocator failed (%d), falling back to page size\n",
@@ -212,6 +219,7 @@ void __init setup_per_cpu_areas(void) }if (rc <0 ) rc =pcpu_page_first_chunk (PERCPU_FIRST_CHUNK_RESERVE , + pcpu_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free, pcpup_populate_pte);if (rc <0 )
drivers/base/arch_numa.c +5 -3
@@ -155,10 +155,10 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)return node_distance (early_cpu_to_node (from),early_cpu_to_node (to)); } -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size, -size_t align) +static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) { -int nid =early_cpu_to_node (cpu); +int nid =cpu_to_nd_fn (cpu);return memblock_alloc_try_nid (size, align,__pa (MAX_DMA_ADDRESS ),MEMBLOCK_ALLOC_ACCESSIBLE , nid);
@@ -229,6 +229,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE ,PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE , pcpu_cpu_distance, + early_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free);#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc <0 )
@@ -240,6 +241,7 @@ void __init setup_per_cpu_areas(void)#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc <0 ) rc =pcpu_page_first_chunk (PERCPU_MODULE_RESERVE , + early_cpu_to_node, pcpu_fc_alloc, pcpu_fc_free, pcpu_populate_pte);
include/linux/percpu.h +5 -2
@@ -94,8 +94,9 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR];extern enum pcpu_fc pcpu_chosen_fc; -typedef void * (*pcpu_fc_alloc_fn_t )(unsigned int cpu,size_t size, -size_t align); +typedef int (pcpu_fc_cpu_to_node_fn_t )(int cpu); +typedef void * (*pcpu_fc_alloc_fn_t )(unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);typedef void (*pcpu_fc_free_fn_t )(void *ptr,size_t size);typedef void (*pcpu_fc_populate_pte_fn_t )(unsigned long addr);typedef int (pcpu_fc_cpu_distance_fn_t )(unsigned int from,unsigned int to);
@@ -111,12 +112,14 @@ extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,extern int __init pcpu_embed_first_chunk (size_t reserved_size,size_t dyn_size,size_t atom_size,pcpu_fc_cpu_distance_fn_t cpu_distance_fn, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn,pcpu_fc_alloc_fn_t alloc_fn,pcpu_fc_free_fn_t free_fn);#endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK extern int __init pcpu_page_first_chunk (size_t reserved_size, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn,pcpu_fc_alloc_fn_t alloc_fn,pcpu_fc_free_fn_t free_fn,pcpu_fc_populate_pte_fn_t populate_pte_fn);
mm/percpu.c +9 -5
@@ -3001,6 +3001,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(* @dyn_size: minimum free size for dynamic allocation in bytes * @atom_size: allocation atom size * @cpu_distance_fn: callback to determine distance between cpus, optional +* @cpu_to_nd_fn: callback to convert cpu to it's node, optional * @alloc_fn: function to allocate percpu page * @free_fn: function to free percpu page *
@@ -3030,6 +3031,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(int __init pcpu_embed_first_chunk (size_t reserved_size,size_t dyn_size,size_t atom_size,pcpu_fc_cpu_distance_fn_t cpu_distance_fn, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn,pcpu_fc_alloc_fn_t alloc_fn,pcpu_fc_free_fn_t free_fn) {
@@ -3066,7 +3068,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,BUG_ON (cpu ==NR_CPUS );/* allocate space for the whole group */ - ptr =alloc_fn (cpu, gi->nr_units * ai->unit_size, atom_size); + ptr =alloc_fn (cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn);if (!ptr) { rc = -ENOMEM ; goto out_free_areas;
@@ -3143,6 +3145,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,/** * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages * @reserved_size: the size of reserved percpu area in bytes +* @cpu_to_nd_fn: callback to convert cpu to it's node, optional * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE * @free_fn: function to free percpu page, always called with PAGE_SIZE * @populate_pte_fn: function to populate pte
@@ -3157,6 +3160,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,* 0 on success, -errno on failure. */ int __init pcpu_page_first_chunk (size_t reserved_size, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn,pcpu_fc_alloc_fn_t alloc_fn,pcpu_fc_free_fn_t free_fn,pcpu_fc_populate_pte_fn_t populate_pte_fn)
@@ -3201,7 +3205,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size,for (i =0 ; i < unit_pages; i++) {void *ptr; - ptr =alloc_fn (cpu,PAGE_SIZE ,PAGE_SIZE ); + ptr =alloc_fn (cpu,PAGE_SIZE ,PAGE_SIZE , cpu_to_nd_fn);if (!ptr) {pr_warn ("failed to allocate %s page for cpu%u\n" , psize_str, cpu);
@@ -3278,8 +3282,8 @@ int __init pcpu_page_first_chunk(size_t reserved_size,unsigned long __per_cpu_offset[NR_CPUS ]__read_mostly ;EXPORT_SYMBOL (__per_cpu_offset); -static void *__init pcpu_dfl_fc_alloc (unsigned int cpu,size_t size, -size_t align) +static void *__init pcpu_dfl_fc_alloc (unsigned int cpu,size_t size,size_t align, +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) {return memblock_alloc_from (size, align,__pa (MAX_DMA_ADDRESS )); }
@@ -3300,7 +3304,7 @@ void __init setup_per_cpu_areas(void)* what the legacy allocator did. */ rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE , -PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE ,NULL , +PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE ,NULL ,NULL , pcpu_dfl_fc_alloc, pcpu_dfl_fc_free);if (rc <0 )panic ("Failed to initialize percpu areas." );
Signature · 1 line
2.18.0.huawei.25
PATCH 3/4 mm: percpu: Add generic pcpu_fc_alloc/free funciton
In reply to Kefeng WangWith previous patch, we could add a generic pcpu first chunk allocate and free function to cleanup the duplicated definations. on each architecture.
--- arch/mips/mm/init.c | 16 +------ arch/powerpc/kernel/setup_64.c | 51 +--------------------- arch/sparc/kernel/smp_64.c | 50 +--------------------- arch/x86/kernel/setup_percpu.c | 59 +------------------------ drivers/base/arch_numa.c | 19 +-------- include/linux/percpu.h | 9 +--- mm/percpu.c | 78 ++++++++++++++++++++-------------- 7 files changed, 54 insertions(+), 228 deletions(-)
arch/mips/mm/init.c +1 -15
@@ -524,19 +524,6 @@ static int __init pcpu_cpu_to_node(int cpu)return cpu_to_node (cpu); } -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align, -pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ -return memblock_alloc_try_nid (size, align,__pa (MAX_DMA_ADDRESS ), -MEMBLOCK_ALLOC_ACCESSIBLE , -cpu_to_nd_fn (cpu)); -} - -static void __init pcpu_fc_free (void *ptr,size_t size) -{ -memblock_free (ptr, size); -} -void __init setup_per_cpu_areas (void ) {unsigned long delta;
@@ -550,8 +537,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE ,PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE , pcpu_cpu_distance, - pcpu_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free); + pcpu_cpu_to_node);if (rc <0 )panic ("Failed to initialize percpu areas." );
arch/powerpc/kernel/setup_64.c +2 -49
69 lines not on this page · raw message
arch/sparc/kernel/smp_64.c +1 -49
68 lines not on this page · raw message
arch/x86/kernel/setup_percpu.c +1 -58
77 lines not on this page · raw message
drivers/base/arch_numa.c +1 -18
@@ -155,20 +155,6 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)return node_distance (early_cpu_to_node (from),early_cpu_to_node (to)); } -static void *__init pcpu_fc_alloc (unsigned int cpu,size_t size,size_t align, -pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) -{ -int nid =cpu_to_nd_fn (cpu); - -return memblock_alloc_try_nid (size, align, -__pa (MAX_DMA_ADDRESS ),MEMBLOCK_ALLOC_ACCESSIBLE , nid); -} - -static void __init pcpu_fc_free (void *ptr,size_t size) -{ -memblock_free (ptr, size); -} -#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK static void __init pcpu_populate_pte (unsigned long addr) {
@@ -229,8 +215,7 @@ void __init setup_per_cpu_areas(void) rc =pcpu_embed_first_chunk (PERCPU_MODULE_RESERVE ,PERCPU_DYNAMIC_RESERVE ,PAGE_SIZE , pcpu_cpu_distance, - early_cpu_to_node, - pcpu_fc_alloc, pcpu_fc_free); + early_cpu_to_node);#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK if (rc <0 ) pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n",
@@ -242,8 +227,6 @@ void __init setup_per_cpu_areas(void)if (rc <0 ) rc =pcpu_page_first_chunk (PERCPU_MODULE_RESERVE , early_cpu_to_node, - pcpu_fc_alloc, - pcpu_fc_free, pcpu_populate_pte);#endif if (rc <0 )
include/linux/percpu.h +1 -8
@@ -95,9 +95,6 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR];extern enum pcpu_fc pcpu_chosen_fc;typedef int (pcpu_fc_cpu_to_node_fn_t )(int cpu); -typedef void * (*pcpu_fc_alloc_fn_t )(unsigned int cpu,size_t size,size_t align, -pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); -typedef void (*pcpu_fc_free_fn_t )(void *ptr,size_t size);typedef void (*pcpu_fc_populate_pte_fn_t )(unsigned long addr);typedef int (pcpu_fc_cpu_distance_fn_t )(unsigned int from,unsigned int to);
@@ -112,16 +109,12 @@ extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,extern int __init pcpu_embed_first_chunk (size_t reserved_size,size_t dyn_size,size_t atom_size,pcpu_fc_cpu_distance_fn_t cpu_distance_fn, -pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, -pcpu_fc_alloc_fn_t alloc_fn, -pcpu_fc_free_fn_t free_fn); +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);#endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK extern int __init pcpu_page_first_chunk (size_t reserved_size,pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, -pcpu_fc_alloc_fn_t alloc_fn, -pcpu_fc_free_fn_t free_fn,pcpu_fc_populate_pte_fn_t populate_pte_fn);#endif
mm/percpu.c +47 -31
165 lines not on this page · raw message
Signature · 1 line
2.18.0.huawei.25
PATCH 4/4 mm: percpu: Add generic pcpu_populate_pte() function
In reply to Kefeng WangWith NEED_PER_CPU_PAGE_FIRST_CHUNK enabled, we need a function to populate pte, this patch adds a generic pcpu populate pte function, pcpu_populate_pte(), which is marked __weak and used on most architectures, but it is overridden on x86, which has its own implemention.
--- arch/powerpc/kernel/setup_64.c | 47 +-------------------- arch/sparc/kernel/smp_64.c | 56 +------------------------ arch/x86/kernel/setup_percpu.c | 5 +-- drivers/base/arch_numa.c | 51 +---------------------- include/linux/percpu.h | 5 +-- mm/percpu.c | 76 +++++++++++++++++++++++++++++++--- 6 files changed, 78 insertions(+), 162 deletions(-)
arch/powerpc/kernel/setup_64.c +1 -46
@@ -787,51 +787,6 @@ static __init int pcpu_cpu_to_node(int cpu)unsigned long __per_cpu_offset[NR_CPUS ]__read_mostly ;EXPORT_SYMBOL (__per_cpu_offset); -static void __init pcpu_populate_pte (unsigned long addr) -{ -pgd_t *pgd =pgd_offset_k (addr); -p4d_t *p4d; -pud_t *pud; -pmd_t *pmd; - - p4d =p4d_offset (pgd, addr); -if (p4d_none (*p4d)) { -pud_t *new; - - new =memblock_alloc (PUD_TABLE_SIZE ,PUD_TABLE_SIZE ); -if (!new) - goto err_alloc; -p4d_populate (&init_mm, p4d, new); - } - - pud =pud_offset (p4d, addr); -if (pud_none (*pud)) { -pmd_t *new; - - new =memblock_alloc (PMD_TABLE_SIZE ,PMD_TABLE_SIZE ); -if (!new) - goto err_alloc; -pud_populate (&init_mm, pud, new); - } - - pmd =pmd_offset (pud, addr); -if (!pmd_present (*pmd)) { -pte_t *new; - - new =memblock_alloc (PTE_TABLE_SIZE ,PTE_TABLE_SIZE ); -if (!new) - goto err_alloc; -pmd_populate_kernel (&init_mm, pmd, new); - } - -return ; - -err_alloc: -panic ("%s: Failed to allocate %lu bytes align=%lx from=%lx\n" , - __func__,PAGE_SIZE ,PAGE_SIZE ,PAGE_SIZE ); -} - -void __init setup_per_cpu_areas (void ) {const size_t dyn_size =PERCPU_MODULE_RESERVE +PERCPU_DYNAMIC_RESERVE ;
@@ -860,7 +815,7 @@ void __init setup_per_cpu_areas(void) }if (rc <0 ) - rc =pcpu_page_first_chunk (0 , pcpu_cpu_to_node, pcpu_populate_pte); + rc =pcpu_page_first_chunk (0 , pcpu_cpu_to_node);if (rc <0 )panic ("cannot initialize percpu area (err=%d)" , rc);
arch/sparc/kernel/smp_64.c +1 -55
68 lines not on this page · raw message
arch/x86/kernel/setup_percpu.c +2 -3
@@ -101,7 +101,7 @@ static int __init pcpu_cpu_to_node(int cpu)return early_cpu_to_node (cpu); } -static void __init pcpup_populate_pte (unsigned long addr) +void __init pcpu_populate_pte (unsigned long addr) {populate_extra_pte (addr); }
@@ -163,8 +163,7 @@ void __init setup_per_cpu_areas(void) }if (rc <0 ) rc =pcpu_page_first_chunk (PERCPU_FIRST_CHUNK_RESERVE , - pcpu_cpu_to_node, - pcpup_populate_pte); + pcpu_cpu_to_node);if (rc <0 )panic ("cannot initialize percpu area (err=%d)" , rc);
drivers/base/arch_numa.c +1 -50
69 lines not on this page · raw message
include/linux/percpu.h +2 -3
@@ -95,7 +95,6 @@ extern const char * const pcpu_fc_names[PCPU_FC_NR];extern enum pcpu_fc pcpu_chosen_fc;typedef int (pcpu_fc_cpu_to_node_fn_t )(int cpu); -typedef void (*pcpu_fc_populate_pte_fn_t )(unsigned long addr);typedef int (pcpu_fc_cpu_distance_fn_t )(unsigned int from,unsigned int to); extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
@@ -113,9 +112,9 @@ extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,#endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK +void __init pcpu_populate_pte (unsigned long addr);extern int __init pcpu_page_first_chunk (size_t reserved_size, -pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, -pcpu_fc_populate_pte_fn_t populate_pte_fn); +pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn);#endif extern void __percpu *__alloc_reserved_percpu (size_t size,size_t align)__alloc_size(1) ;
mm/percpu.c +71 -5
98 lines not on this page · raw message
Signature · 1 line
2.18.0.huawei.25