Skip to contentSkip to conversation

per-cpu support inside modules (minimal)

PATCH In review Rusty Russell 3 participants 4 messages May 22, 2003 – May 23, 2003

Subsystems (3)
Generic include/asm header files Module support Per-CPU memory allocator
lore
Share & tools
lore

Rusty Russell

author
#0 lore raw

PATCH 1/1 per-cpu support inside modules (minimal)

OK, this does the *minimum* required to support DEFINE_PER_CPU inside modules. If we decide to change kmalloc_percpu later, great, we can turf this out.

Basically, overallocates the amount of per-cpu data at boot to at least PERCPU_ENOUGH_ROOM if CONFIG_MODULES=y (arch-specific by default 16k: I have only 5700 bytes of percpu data in my kernel here, so makes sense), and a special allocator in module.c dishes it out.

Comments welcome!
Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.

Name: per-cpu support inside modules
Author: Rusty Russell
Status: Tested on 2.5.69-bk14

D: This adds percpu support for modules.

include/asm-generic/percpu.h +8 -6
diff -urNp --exclude TAGS -X /home/rusty/current-dontdiff --minimal linux-2.5.69-bk14/include/asm-generic/percpu.h working-2.5.69-bk14-module-percpu/include/asm-generic/percpu.h
@@ -4,26 +4,28 @@
 
 #define __GENERIC_PER_CPU
 #ifdef CONFIG_SMP
-
 extern unsigned long __per_cpu_offset[NR_CPUS];
 
 /* Separate out the type, so (int[3], foo) works. */
-#ifndef MODULE
 #define DEFINE_PER_CPU(type, name) \
     __attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu
-#endif
 
 /* var is in discarded region: offset to particular copy we want */
 #define per_cpu(var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu]))
 #define __get_cpu_var(var) per_cpu(var, smp_processor_id())
 
+static inline void percpu_modcopy(void *pcpudst, const void *src,
+				  unsigned long size)
+{
+	unsigned int i;
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i))
+			memcpy(pcpudst + __per_cpu_offset[i], src, size);
+}
 #else /* ! SMP */
 
-/* Can't define per-cpu variables in modules.  Sorry --RR */
-#ifndef MODULE
 #define DEFINE_PER_CPU(type, name) \
     __typeof__(type) name##__per_cpu
-#endif
 
 #define per_cpu(var, cpu)			((void)cpu, var##__per_cpu)
 #define __get_cpu_var(var)			var##__per_cpu
include/asm-ia64/percpu.h +9 -2
diff -urNp --exclude TAGS -X /home/rusty/current-dontdiff --minimal linux-2.5.69-bk14/include/asm-ia64/percpu.h working-2.5.69-bk14-module-percpu/include/asm-ia64/percpu.h
@@ -19,15 +19,22 @@
 
 extern unsigned long __per_cpu_offset[NR_CPUS];
 
-#ifndef MODULE
 #define DEFINE_PER_CPU(type, name) \
     __attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu
-#endif
 #define DECLARE_PER_CPU(type, name) extern __typeof__(type) name##__per_cpu
 
 #define __get_cpu_var(var)	(var##__per_cpu)
 #ifdef CONFIG_SMP
 # define per_cpu(var, cpu)	(*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu]))
+
+static inline void percpu_modcopy(void *pcpudst, const void *src,
+				  unsigned long size)
+{
+	unsigned int i;
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i))
+			memcpy(pcpudst + __per_cpu_offset[i], src, size);
+}
 #else
 # define per_cpu(var, cpu)	((void)cpu, __get_cpu_var(var))
 #endif
include/linux/module.h +3 -0
diff -urNp --exclude TAGS -X /home/rusty/current-dontdiff --minimal linux-2.5.69-bk14/include/linux/module.h working-2.5.69-bk14-module-percpu/include/linux/module.h
@@ -247,6 +247,9 @@ struct module
 	char *strtab;
 #endif
 
+	/* Per-cpu data. */
+	void *percpu;
+
 	/* The command line arguments (may be mangled).  People like
 	   keeping pointers to this stuff */
 	char *args;
include/linux/percpu.h +8 -1
diff -urNp --exclude TAGS -X /home/rusty/current-dontdiff --minimal linux-2.5.69-bk14/include/linux/percpu.h working-2.5.69-bk14-module-percpu/include/linux/percpu.h
@@ -1,9 +1,16 @@
 #ifndef __LINUX_PERCPU_H
 #define __LINUX_PERCPU_H
 #include <linux/spinlock.h> /* For preempt_disable() */
-#include <linux/slab.h> /* For kmalloc_percpu() */
+#include <linux/slab.h> /* For kmalloc() */
+#include <linux/smp.h>
+#include <linux/string.h>
 #include <asm/percpu.h>
 
+/* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
+#ifndef PERCPU_ENOUGH_ROOM
+#define PERCPU_ENOUGH_ROOM 16384
+#endif
+
 /* Must be an lvalue. */
 #define get_cpu_var(var) (*({ preempt_disable(); &__get_cpu_var(var); }))
 #define put_cpu_var(var) preempt_enable()
init/main.c +5 -3
diff -urNp --exclude TAGS -X /home/rusty/current-dontdiff --minimal linux-2.5.69-bk14/init/main.c working-2.5.69-bk14-module-percpu/init/main.c
@@ -318,14 +318,16 @@ static void __init setup_per_cpu_areas(v
 
 	/* Copy section for each CPU (we discard the original) */
 	size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
-	if (!size)
-		return;
+#ifdef CONFIG_MODULES
+	if (size < PERCPU_ENOUGH_ROOM)
+		size = PERCPU_ENOUGH_ROOM;
+#endif
 
 	ptr = alloc_bootmem(size * NR_CPUS);
 
 	for (i = 0; i < NR_CPUS; i++, ptr += size) {
 		__per_cpu_offset[i] = ptr - __per_cpu_start;
-		memcpy(ptr, __per_cpu_start, size);
+		memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
 	}
 }
 #endif /* !__GENERIC_PER_CPU */
kernel/module.c +188 -8

271 lines not on this page · raw message

#1 lore raw

I'll give this a run on the ppc64 tomorrow.

Quoted 7 lines · Rusty Russell

+static inline unsigned int abs(int val)
+{
+ if (val < 0)
+ return -val;
+ return val;
+}

Several architectures define their own abs() in asm/system.h which return signed int.

author
#2 lore raw

In message <[email protected]> you write:

Quoted 9 lines
Quoted 6 lines

+static inline unsigned int abs(int val)
+{
+ if (val < 0)
+ return -val;
+ return val;
+}

Several architectures define their own abs() in asm/system.h which return signed int.

Hmm, it's actually declated that way in linux/kernel.h, too:

	extern int abs(int);

But AFAICT it's never actually defined.

Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.

#3 lore raw
In reply to Rusty Russell

Rusty> OK, this does the *minimum* required to support Rusty> DEFINE_PER_CPU inside modules. If we decide to change Rusty> kmalloc_percpu later, great, we can turf this out.

Rusty> Basically, overallocates the amount of per-cpu data at boot Rusty> to at least PERCPU_ENOUGH_ROOM if CONFIG_MODULES=y Rusty> (arch-specific by default 16k: I have only 5700 bytes of Rusty> percpu data in my kernel here, so makes sense), and a special Rusty> allocator in module.c dishes it out.

Rusty> Comments welcome!

Looks good to me. For ia64, I guess I'll want to add a

	#define PERCPU_ENOUGH_ROOM	PERCPU_PAGE_SIZE

somewhere, but that doesn't sound so hard. ;-)

Thanks!

	--david