ia64: convert to dynamic percpu allocator

Unlike other archs, ia64 reserves space for percpu areas during early memory initialization. The space for cpu0 is reserved separate in the linker script. Other cpus occupy a contiguous region indexed by cpu number on contiguous memory model or are grouped by node on discontiguous memory model.

As allocation and initialization are done by the arch code, all that setup_per_cpu_areas() needs to do is communicating the determined layout to the percpu allocator. This patch implements setup_per_cpu_areas() for both contig and discontig memory models and drops HAVE_LEGACY_PER_CPU_AREA.

NOT_SIGNED_OFF_YET
Cc: Tony Luck <[email protected]>
Cc: Fenghua Yu <[email protected]>
---
I don't have access to an ia64 machine so I could only test with ski. With sim_defconfig, it boots but I didn't have any userland. discontig configuration builds fine but I couldn't test it.

Can you guys please verify this patch?

This patch is available in the following git tree.

git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git review-ia64

Thanks.

 arch/ia64/Kconfig        |    3 -
 arch/ia64/kernel/setup.c |   12 ------
 arch/ia64/mm/contig.c    |   67 +++++++++++++++++++++++++++++++++
 arch/ia64/mm/discontig.c |   94 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 161 insertions(+), 15 deletions(-)

Index: work/arch/ia64/Kconfig =================================--- work.orig/arch/ia64/Kconfig +++ work/arch/ia64/Kconfig
@@ -89,9 +89,6 @@ config GENERIC_TIME_VSYSCALL
bool
default y

-config HAVE_LEGACY_PER_CPU_AREA
- def_bool y
-
config HAVE_SETUP_PER_CPU_AREA
def_bool y

Index: work/arch/ia64/mm/contig.c =================================--- work.orig/arch/ia64/mm/contig.c +++ work/arch/ia64/mm/contig.c
@@ -186,6 +186,73 @@ alloc_per_cpu_data(void)
cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS-1,
PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
}
+
+/**
+ * setup_per_cpu_areas - setup percpu areas
+ *
+ * Arch code has already allocated and initialized percpu areas. All
+ * this function has to do is to teach the determined layout to the
+ * dynamic percpu allocator, which happens to be more complex than
+ * creating whole new ones using helpers.
+ */
+void __init
+setup_per_cpu_areas(void)
+{
+ struct pcpu_alloc_info *ai;
+ struct pcpu_group_info *gi;
+ unsigned int *cpu_map;
+ unsigned int cpu;
+ void *base, *grp0_base, *grp1_base = NULL;
+ int rc;
+
+ ai = pcpu_alloc_alloc_info(2, nr_cpu_ids);
+ if (!ai)
+ panic("failed to allocate pcpu_alloc_info");
+ cpu_map = ai->groups[0].cpu_map;
+
+ /* cpus are identity mapped to units */
+ for_each_possible_cpu(cpu)
+ cpu_map[cpu] = cpu;
+
+ /* determine bases */
+ base = grp0_base = __per_cpu_start + __per_cpu_offset[0];
+ if (num_possible_cpus() > 1) {
+ grp1_base = __per_cpu_start + __per_cpu_offset[1];
+ base = min(grp0_base, grp1_base);
+ }
+
+ /* set basic parameters */
+ ai->static_size = __per_cpu_end - __per_cpu_start;
+ ai->reserved_size = PERCPU_MODULE_RESERVE; /* tj - necessary? */
+ ai->dyn_size = PERCPU_DYNAMIC_RESERVE;
+ ai->unit_size = PERCPU_PAGE_SIZE;
+ ai->atom_size = PAGE_SIZE;
+ ai->alloc_size = PERCPU_PAGE_SIZE;
+
+ /*
+ * BSP is always present and occupies separate statically
+ * reserved area.
+ */
+ ai->nr_groups = 0;
+ gi = &ai->groups[ai->nr_groups++];
+ gi->nr_units = 1;
+ gi->base_offset = grp0_base - base;
+ gi->cpu_map = &cpu_map[0];
+
+ if (grp1_base) {
+ /* all APs occupy single contiguos region indexed by cpu id */
+ gi = &ai->groups[ai->nr_groups++];
+ gi->nr_units = nr_cpu_ids - 1;
+ gi->base_offset = grp1_base - base;
+ gi->cpu_map = &cpu_map[1];
+ }
+
+ rc = pcpu_setup_first_chunk(ai, base);
+ if (rc)
+ panic("failed to setup percpu area (err=%d)", rc);
+
+ pcpu_free_alloc_info(ai);
+}
#else
#define alloc_per_cpu_data() do { } while (0)
#endif /* CONFIG_SMP */
Index: work/arch/ia64/mm/discontig.c =================================--- work.orig/arch/ia64/mm/discontig.c +++ work/arch/ia64/mm/discontig.c
@@ -159,6 +159,100 @@ static void *per_cpu_node_setup(void *cp
return cpu_data;
}

+#ifdef CONFIG_SMP
+/**
+ * setup_per_cpu_areas - setup percpu areas
+ *
+ * Arch code has already allocated and initialized percpu areas. All
+ * this function has to do is to teach the determined layout to the
+ * dynamic percpu allocator, which happens to be more complex than
+ * creating whole new ones using helpers.
+ */
+void __init setup_per_cpu_areas(void)
+{
+ struct pcpu_alloc_info *ai;
+ struct pcpu_group_info *gi;
+ unsigned int *cpu_map;
+ void *base;
+ unsigned long base_offset;
+ unsigned int cpu;
+ int node, prev_node, unit, nr_units, rc;
+
+ ai = pcpu_alloc_alloc_info(MAX_NUMNODES + 1, nr_cpu_ids);
+ if (!ai)
+ panic("failed to allocate pcpu_alloc_info");
+ cpu_map = ai->groups[0].cpu_map;
+
+ /* determine base */
+ base = (void *)ULONG_MAX;
+ for_each_possible_cpu(cpu)
+ base = min(base,
+ (void *)(__per_cpu_offset[cpu] + __per_cpu_start));
+ base_offset = (void *)__per_cpu_start - base;
+
+ /*
+ * Build cpu_map. cpu0 is always mapped to unit0. The rest
+ * are grouped by node.
+ */
+ unit = 0;
+ cpu_map[unit++] = 0;
+ for_each_node(node) {
+ for_each_possible_cpu(cpu) {
+ if (cpu = 0)
+ continue;
+ if (node = node_cpuid[cpu].nid)
+ cpu_map[unit++] = cpu;
+ }
+ }
+ nr_units = unit;
+
+ /* set basic parameters */
+ ai->static_size = __per_cpu_end - __per_cpu_start;
+ ai->reserved_size = PERCPU_MODULE_RESERVE; /* tj - necessary? */
+ ai->dyn_size = PERCPU_DYNAMIC_RESERVE;
+ ai->unit_size = PERCPU_PAGE_SIZE;
+ ai->atom_size = PAGE_SIZE;
+ ai->alloc_size = PERCPU_PAGE_SIZE;
+
+ /*
+ * BSP is always present and occupies separate statically
+ * reserved area.
+ */
+ ai->nr_groups = 0;
+ gi = &ai->groups[ai->nr_groups++];
+ gi->nr_units = 1;
+ gi->base_offset = __per_cpu_offset[0] + base_offset;
+ gi->cpu_map = &cpu_map[0];
+
+ /*
+ * APs should be put into groups according to node. Walk
+ * cpu_map and create new groups at node boundaries.
+ */
+ prev_node = -1;
+ for (unit = 1; unit < nr_units; unit++) {
+ unsigned int cpu = cpu_map[unit];
+
+ node = node_cpuid[cpu].nid;
+ if (node = prev_node) {
+ gi->nr_units++;
+ continue;
+ }
+ prev_node = node;
+
+ gi = &ai->groups[ai->nr_groups++];
+ gi->nr_units = 1;
+ gi->base_offset = __per_cpu_offset[cpu] + base_offset;
+ gi->cpu_map = &cpu_map[unit];
+ }
+
+ rc = pcpu_setup_first_chunk(ai, base);
+ if (rc)
+ panic("failed to setup percpu area (err=%d)", rc);
+
+ pcpu_free_alloc_info(ai);
+}
+#endif
+
/**
* fill_pernode - initialize pernode data.
* @node: the node id.
Index: work/arch/ia64/kernel/setup.c =================================--- work.orig/arch/ia64/kernel/setup.c +++ work/arch/ia64/kernel/setup.c
@@ -856,18 +856,6 @@ identify_cpu (struct cpuinfo_ia64 *c)
}

/*
- * In UP configuration, setup_per_cpu_areas() is defined in
- * include/linux/percpu.h
- */
-#ifdef CONFIG_SMP
-void __init
-setup_per_cpu_areas (void)
-{
- /* start_kernel() requires this... */
-}
-#endif
-
-/*
* Do the following calculations:
*
* 1. the max. cache line size.

Is this patch against the latest linux-next? It's applied cleanly on linux-next. But compilation reports undefined function pcpu_alloc_alloc_info() in both contig.c and discontig.c.

arch/ia64/mm/contig.c: In function 'setup_per_cpu_areas':
arch/ia64/mm/contig.c:208: error: implicit declaration of function 'pcpu_alloc_al loc_info'
arch/ia64/mm/contig.c:208: warning: assignment makes pointer from integer without
a cast
arch/ia64/mm/contig.c:211: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:225: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:226: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:227: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:228: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:229: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:230: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:236: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:237: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:237: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:238: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:239: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:240: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:244: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:244: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:245: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:246: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:247: error: dereferencing pointer to incomplete type arch/ia64/mm/contig.c:250: warning: passing argument 1 of 'pcpu_setup_first_chunk ' makes integer from pointer without a cast arch/ia64/mm/contig.c:250: warning: passing argument 2 of 'pcpu_setup_first_chunk ' makes integer from pointer without a cast arch/ia64/mm/contig.c:250: error: too few arguments to function 'pcpu_setup_first _chunk'
arch/ia64/mm/contig.c:254: error: implicit declaration of function 'pcpu_free_all oc_info'

Hello,

Oh, it's on top of percpu#for-next + 20 patches to implement sparse embedding[1]. I think it would be the easiest to fetch the above git tree.

Thanks.

tejun

[1] http://thread.gmane.org/gmane.linux.kernel.cross-arch/4124

After cloning from git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git, kernel built with CONFIG_DISCONTIGMEM=y boots ok.

After pulling from review-ia64, kernel built with CONFIG_DISCONTIGMEM=y can not boot on ia64. After loading kernel and initrd, system hangs and doesn't show anything on serial port.

Thanks.

-Fenghua

Hello,

Heh, I didn't really expect it to work that easily either. The code builds but is completely untested (I can't test them). Can you try to track it down? Serial console should be up and running at that point, no?

Thanks.

tejun

IA64 kernel boots hit this in mm/percpu.c
BUG_ON(ai->unit_size < size_sum);
ai->unit_size is PERCPU_PAGE_SIZE which is 64K on IA64. size_sum is relatively smaller than 64K.

Will you define PERCPU_DYNAMIC_RESERVE and PERCPU_MODULE_RESERVE as PAGE_SIZE or the bigger one between the current definition and PAGE_SIZE?

Even with the above PAGE_SIZE changes, the kernel still reports warning from 952: WARN_ON(chunk->immutable) and then panic.

Calibrating delay loop... 1593.34 BogoMIPS (lpj186688)

------------[ cut here ]------------
WARNING: at mm/percpu.c:952 pcpu_alloc+0x610/0xfe0()
Modules linked in:

Call Trace:
 [<a0000001000162e0>] show_stack+0x40/0xa0
                                sp 00000100c7fc30 bsp 00000100c711f0
 [<a000000100016370>] dump_stack+0x30/0x60
                                sp 00000100c7fe00 bsp 00000100c711d8
 [<a00000010009b340>] warn_slowpath_common+0xc0/0x100
                                sp 00000100c7fe00 bsp 00000100c711a0
 [<a00000010009b3b0>] warn_slowpath_null+0x30/0x60
                                sp 00000100c7fe00 bsp 00000100c71178
 [<a00000010018d3f0>] pcpu_alloc+0x610/0xfe0
                                sp 00000100c7fe00 bsp 00000100c710e0
 [<a00000010018de50>] __alloc_percpu+0x30/0x60
                                sp 00000100c7fe10 bsp 00000100c710b8
 [<a00000010044a180>] __percpu_counter_init+0x60/0x120
                                sp 00000100c7fe10 bsp 00000100c71090
 [<a000000100ac1110>] mmap_init+0x30/0x60
                                sp 00000100c7fe20 bsp 00000100c71078
 [<a000000100ab6420>] proc_caches_init+0x1a0/0x1c0
                                sp 00000100c7fe20 bsp 00000100c71060
 [<a000000100aa1520>] start_kernel+0x7e0/0x8c0
                                sp 00000100c7fe20 bsp 00000100c70fe0
 [<a00000010085be60>] _start+0x760/0x780
                                sp 00000100c7fe30 bsp 00000100c70f40
---[ end trace 4eaa2a86a8e2da22 ]---
------------[ cut here ]------------
WARNING: at mm/vmalloc.c:106 vmap_page_range_noflush+0x3d0/0x460()
Modules linked in:

Call Trace:
 [<a0000001000162e0>] show_stack+0x40/0xa0
                                sp 00000100c7fc40 bsp 00000100c71300
 [<a000000100016370>] dump_stack+0x30/0x60
                                sp 00000100c7fe10 bsp 00000100c712e8
 [<a00000010009b340>] warn_slowpath_common+0xc0/0x100
                                sp 00000100c7fe10 bsp 00000100c712b0
 [<a00000010009b3b0>] warn_slowpath_null+0x30/0x60
                                sp 00000100c7fe10 bsp 00000100c71288
 [<a00000010015f470>] vmap_page_range_noflush+0x3d0/0x460
                                sp 00000100c7fe10 bsp 00000100c71210
 [<a00000010015f590>] vmap_page_range+0x30/0x60
                                sp 00000100c7fe10 bsp 00000100c711d0
 [<a00000010015f610>] map_vm_area+0x50/0xa0
                                sp 00000100c7fe10 bsp 00000100c711a0
 [<a00000010015f950>] __vmalloc_area_node+0x2f0/0x3a0
                                sp 00000100c7fe10 bsp 00000100c71158
 [<a00000010015fae0>] __vmalloc_node+0xe0/0x120
                                sp 00000100c7fe20 bsp 00000100c71118
 [<a00000010015fe60>] __vmalloc+0x40/0x60
                                sp 00000100c7fe20 bsp 00000100c710e8
 [<a000000100ac0350>] alloc_large_system_hash+0x2f0/0x4e0
                                sp 00000100c7fe20 bsp 00000100c71088
 [<a000000100ac58b0>] vfs_caches_init+0x150/0x280
                                sp 00000100c7fe20 bsp 00000100c71060
 [<a000000100aa1540>] start_kernel+0x800/0x8c0
                                sp 00000100c7fe20 bsp 00000100c70fe0
 [<a00000010085be60>] _start+0x760/0x780
                                sp 00000100c7fe30 bsp 00000100c70f40
---[ end trace 4eaa2a86a8e2da23 ]---
Dentry cache hash table entries: 1048576 (order: 7, 8388608 bytes)
Inode-cache hash table entries: 1048576 (order: 7, 8388608 bytes)
Mount-cache hash table entries: 4096
Unable to handle kernel paging request at virtual address 00d0000000000000
swapper[0]: Oops 8813272891392 [1]
Modules linked in:

Pid: 0, CPU 0, comm: swapper
psr : 00001010084a2018 ifs : 800000000000038c ip : [<a0000001001840e0>] Tain
ted: G W (2.6.31-rc5)
ip is at kmem_cache_alloc+0xc0/0x1a0
unat: 0000000000000000 pfs : 000000000000040b rsc : 0000000000000003
rnat: a000000101362dc8 bsps: a000000101363f6c pr : 656960155aa6a659
ldrs: 0000000000000000 ccv : 0000000000000000 fpsr: 0009804c8a70433f csd : 0930ffff00078000 ssd : 0930ffff00078000
b0 : a0000001001c6290 b6 : a0000001002fefe0 b7 : a000000100195b00 f6 : 0fff592492492483a4000 f7 : 0ffe7cd08000000000000 f8 : 1000f8000000000000000 f9 : 10008e000000000000000 f10 : 1000592492492483a4000 f11 : 1003e0000000000000049 r1 : a000000101547240 r2 : a000000101246438 r3 : a0000001011cacf0 r8 : 0000000000000000 r9 : 0000000000000000 r10 : a000000100c70ee4 r11 : 00000000003fffff r12 : a000000100c7fe20 r13 : a000000100c70000 r14 : 00d0000000000000 r15 : a00000010124644c r16 : a000000100db4f58 r17 : a000000101347b70 r18 : 656960155aa6a659 r19 : a000000100a004c0 r20 : 0000000000000000 r21 : e00000018012c098 r22 : 0000000000000006 r23 : 0000000000000008 r24 : 0000000000000370 r25 : e000000180130010 r26 : 0000000000000020 r27 : a000000100cb0200 r28 : a000000101347740 r29 : 0000000080000000 r30 : 00000000000000c1 r31 : a000000101246340

Call Trace:
 [<a0000001000162e0>] show_stack+0x40/0xa0
                                sp 00000100c7f9f0 bsp 00000100c712a0
 [<a000000100016bf0>] show_regs+0x850/0x8a0
                                sp 00000100c7fbc0 bsp 00000100c71248
 [<a00000010003a170>] die+0x1b0/0x2c0
                                sp 00000100c7fbc0 bsp 00000100c71200
 [<a000000100064570>] ia64_do_page_fault+0x8d0/0xa40
                                sp 00000100c7fbc0 bsp 00000100c711a8
 [<a00000010000c780>] ia64_native_leave_kernel+0x0/0x270
                                sp 00000100c7fc50 bsp 00000100c711a8
 [<a0000001001840e0>] kmem_cache_alloc+0xc0/0x1a0
                                sp 00000100c7fe20 bsp 00000100c71148
 [<a0000001001c6290>] alloc_vfsmnt+0x30/0x2e0
                                sp 00000100c7fe20 bsp 00000100c71108
 [<a000000100197450>] vfs_kern_mount+0x30/0x1a0
                                sp 00000100c7fe20 bsp 00000100c710c8
 [<a000000100197600>] kern_mount_data+0x40/0x60
                                sp 00000100c7fe20 bsp 00000100c710a0
 [<a000000100ac6970>] bdev_cache_init+0xb0/0x140
                                sp 00000100c7fe20 bsp 00000100c71088
 [<a000000100ac59a0>] vfs_caches_init+0x240/0x280
                                sp 00000100c7fe20 bsp 00000100c71060
 [<a000000100aa1540>] start_kernel+0x800/0x8c0
                                sp 00000100c7fe20 bsp 00000100c70fe0
 [<a00000010085be60>] _start+0x760/0x780
                                sp 00000100c7fe30 bsp 00000100c70f40
Disabling lock debugging due to kernel taint
Kernel panic - not syncing: Attempted to kill the idle task!

Thanks.

-Fenghua

Hmmm...

* Is ai->reserved_size necessary for ia64? This is necessary if there's linking range restriction when loading modules. ai->reserved_size guarantees that all static module percpu variables are allocated in the first chunk which will be in the linear address range and very close to __per_cpu_start. If ai->reserved_size is not set, these areas are likely to end up high in the vmalloc area.

For example, x86_64 assumes 32bit relocations should be enough to link module symbols and thus needs to set reserved_size but x86_32 can link to the whole 32bit space and thus can leave reserved_size at zero.

* After determining the above, we can set ai->dyn_size to be

ai->dyn_size = min(ai->unit_size - ai->static_size - ai->reserved_size,

		     PERCPU_DYNAMIC_RESERVE);

Would the above work?

Hah... strange. Can you please attach full boot log? This is dicontig configuration, right?

Thanks.

tejun

Any news?

Thanks.

tejun