Skip to contentSkip to conversation

x86_64: SMP alternatives

PATCH New Gerd Hoffmann 1 participant 1 message Apr 12, 2006 Atomic infrastructure

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86_64: SMP alternatives

Hi,

$subject says pretty much all, this is the x86_64 version of the smp alternatives patch. Detailed description and patch are attached below.

cheers,

Gerd

This is the x86_64 version of the smp alternative patch.

Changes are largely identical to the i386 version:

* alternative #define are moved to the new alternative.h file.
* one new elf section with pointers to the lock prefixes which can be nop'ed out for non-smp.
* two new elf sections simliar to the "classic" alternatives to replace SMP code with simpler UP code.
* fixup headers to use alternative.h instead of defining their own LOCK / LOCK_PREFIX macros.

The patch reuses the i386 version of the alternatives code to avoid code duplication. The code in alternatives.c was shuffled around a bit to reduce the number of #ifdefs needed. It also got some tweaks needed for x86_64 (vsyscall page handling) and new features (noreplacement option which was x86_64 only up to now). Debug printk's are changed from compile-time to runtime.

Loosely based on a early version from Bastian Blank <[email protected]>

arch/x86_64/kernel/Makefile +2 -1
Index: vanilla-2.6.17-rc1/arch/x86_64/kernel/Makefile
===================================================================
@@ -8,7 +8,7 @@ obj-y	:= process.o signal.o entry.o trap
 		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \
 		x8664_ksyms.o i387.o syscall.o vsyscall.o \
 		setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \
-		dmi_scan.o pci-dma.o pci-nommu.o
+		dmi_scan.o pci-dma.o pci-nommu.o alternative.o
 
 obj-$(CONFIG_X86_MCE)         += mce.o
 obj-$(CONFIG_X86_MCE_INTEL)	+= mce_intel.o
@@ -50,4 +50,5 @@ quirks-y			+= ../../i386/kernel/quirks.o
 i8237-y				+= ../../i386/kernel/i8237.o
 msr-$(subst m,y,$(CONFIG_X86_MSR))  += ../../i386/kernel/msr.o
 dmi_scan-y			+= ../../i386/kernel/dmi_scan.o
+alternative-y			+= ../../i386/kernel/alternative.o
 
arch/x86_64/kernel/module.c +25 -13
Index: vanilla-2.6.17-rc1/arch/x86_64/kernel/module.c
===================================================================
@@ -145,26 +145,38 @@ int apply_relocate(Elf_Shdr *sechdrs,
 	return -ENOSYS;
 } 
 
-extern void apply_alternatives(void *start, void *end); 
-
 int module_finalize(const Elf_Ehdr *hdr,
-		    const Elf_Shdr *sechdrs,
-		    struct module *me)
+                    const Elf_Shdr *sechdrs,
+                    struct module *me)
 {
-	const Elf_Shdr *s;
+	const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL;
 	char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
-	/* look for .altinstructions to patch */ 
-	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 
-		void *seg; 		
-		if (strcmp(".altinstructions", secstrings + s->sh_name))
-			continue;
-		seg = (void *)s->sh_addr; 
-		apply_alternatives(seg, seg + s->sh_size); 
-	} 	
+	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
+		if (!strcmp(".text", secstrings + s->sh_name))
+			text = s;
+		if (!strcmp(".altinstructions", secstrings + s->sh_name))
+			alt = s;
+		if (!strcmp(".smp_locks", secstrings + s->sh_name))
+			locks= s;
+	}
+
+	if (alt) {
+		/* patch .altinstructions */
+		void *aseg = (void *)alt->sh_addr;
+		apply_alternatives(aseg, aseg + alt->sh_size);
+	}
+	if (locks && text) {
+		void *lseg = (void *)locks->sh_addr;
+		void *tseg = (void *)text->sh_addr;
+		alternatives_smp_module_add(me, me->name,
+					    lseg, lseg + locks->sh_size,
+					    tseg, tseg + text->sh_size);
+	}
 	return 0;
 }
 
 void module_arch_cleanup(struct module *mod)
 {
+	alternatives_smp_module_del(mod);
 }
arch/x86_64/kernel/setup.c +1 -75

88 lines not on this page · raw message

arch/x86_64/kernel/smpboot.c +4 -0
Index: vanilla-2.6.17-rc1/arch/x86_64/kernel/smpboot.c
===================================================================
@@ -797,6 +797,8 @@ static int __cpuinit do_boot_cpu(int cpu
 	}
 
 
+	alternatives_smp_switch(1);
+
 	c_idle.idle = get_idle_for_cpu(cpu);
 
 	if (c_idle.idle) {
@@ -1259,6 +1261,8 @@ void __cpu_die(unsigned int cpu)
 		/* They ack this in play_dead by setting CPU_DEAD */
 		if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
 			printk ("CPU %d is now offline\n", cpu);
+			if (1 == num_online_cpus())
+				alternatives_smp_switch(0);
 			return;
 		}
 		msleep(100);
arch/x86_64/kernel/vmlinux.lds.S +20 -0
Index: vanilla-2.6.17-rc1/arch/x86_64/kernel/vmlinux.lds.S
===================================================================
@@ -131,6 +131,26 @@ SECTIONS
 	*(.data.page_aligned)
   }
 
+  /* might get freed after init */
+  . = ALIGN(4096);
+  __smp_alt_begin = .;
+  __smp_alt_instructions = .;
+  .smp_altinstructions : AT(ADDR(.smp_altinstructions) - LOAD_OFFSET) {
+	*(.smp_altinstructions)
+  }
+  __smp_alt_instructions_end = .;
+  . = ALIGN(8);
+  __smp_locks = .;
+  .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
+	*(.smp_locks)
+  }
+  __smp_locks_end = .;
+  .smp_altinstr_replacement : AT(ADDR(.smp_altinstr_replacement) - LOAD_OFFSET) {
+	*(.smp_altinstr_replacement)
+  }
+  . = ALIGN(4096);
+  __smp_alt_end = .;
+
   . = ALIGN(4096);		/* Init code and data */
   __init_begin = .;
   .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
arch/x86_64/mm/init.c +14 -13
Index: vanilla-2.6.17-rc1/arch/x86_64/mm/init.c
===================================================================
@@ -613,20 +613,29 @@ void __init mem_init(void)
 #endif
 }
 
-void free_initmem(void)
+void free_init_pages(char *what, unsigned long begin, unsigned long end)
 {
 	unsigned long addr;
 
-	addr = (unsigned long)(&__init_begin);
-	for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
+	if (begin >= end)
+		return;
+
+	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
+	for (addr = begin; addr < end; addr += PAGE_SIZE) {
 		ClearPageReserved(virt_to_page(addr));
 		init_page_count(virt_to_page(addr));
 		memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE); 
 		free_page(addr);
 		totalram_pages++;
 	}
+}
+
+void free_initmem(void)
+{
 	memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
-	printk ("Freeing unused kernel memory: %luk freed\n", (__init_end - __init_begin) >> 10);
+	free_init_pages("unused kernel memory",
+			(unsigned long)(&__init_begin),
+			(unsigned long)(&__init_end));
 }
 
 #ifdef CONFIG_DEBUG_RODATA
@@ -655,15 +664,7 @@ void mark_rodata_ro(void)
 #ifdef CONFIG_BLK_DEV_INITRD
 void free_initrd_mem(unsigned long start, unsigned long end)
 {
-	if (start >= end)
-		return;
-	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
-	for (; start < end; start += PAGE_SIZE) {
-		ClearPageReserved(virt_to_page(start));
-		init_page_count(virt_to_page(start));
-		free_page(start);
-		totalram_pages++;
-	}
+	free_init_pages("initrd memory", start, end);
 }
 #endif
 
include/asm-x86_64/alternative.h new file +146 -0

146 lines not on this page · raw message

include/asm-x86_64/atomic.h +21 -29

179 lines not on this page · raw message

include/asm-x86_64/bitops.h +1 -6
Index: vanilla-2.6.17-rc1/include/asm-x86_64/bitops.h
===================================================================
@@ -6,12 +6,7 @@
  */
 
 #include <linux/config.h>
-
-#ifdef CONFIG_SMP
-#define LOCK_PREFIX "lock ; "
-#else
-#define LOCK_PREFIX ""
-#endif
+#include <asm/alternative.h>
 
 #define ADDR (*(volatile long *) addr)
 
include/asm-x86_64/cpufeature.h +1 -0
Index: vanilla-2.6.17-rc1/include/asm-x86_64/cpufeature.h
===================================================================
@@ -64,6 +64,7 @@
 #define X86_FEATURE_REP_GOOD	(3*32+ 4) /* rep microcode works well on this CPU */
 #define X86_FEATURE_CONSTANT_TSC (3*32+5) /* TSC runs at constant rate */
 #define X86_FEATURE_SYNC_RDTSC  (3*32+6)  /* RDTSC syncs CPU core */
+#define X86_FEATURE_UP		(3*32+ 7) /* SMP kernel running on UP */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3	(4*32+ 0) /* Streaming SIMD Extensions-3 */
include/asm-x86_64/rwlock.h +4 -4
Index: vanilla-2.6.17-rc1/include/asm-x86_64/rwlock.h
===================================================================
@@ -24,7 +24,7 @@
 #define RW_LOCK_BIAS_STR	"0x01000000"
 
 #define __build_read_lock_ptr(rw, helper)   \
-	asm volatile(LOCK "subl $1,(%0)\n\t" \
+	asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \
 		     "js 2f\n" \
 		     "1:\n" \
 		    LOCK_SECTION_START("") \
@@ -34,7 +34,7 @@
 		     ::"a" (rw) : "memory")
 
 #define __build_read_lock_const(rw, helper)   \
-	asm volatile(LOCK "subl $1,%0\n\t" \
+	asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \
 		     "js 2f\n" \
 		     "1:\n" \
 		    LOCK_SECTION_START("") \
@@ -54,7 +54,7 @@
 					} while (0)
 
 #define __build_write_lock_ptr(rw, helper) \
-	asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \
+	asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \
 		     "jnz 2f\n" \
 		     "1:\n" \
 		     LOCK_SECTION_START("") \
@@ -64,7 +64,7 @@
 		     ::"a" (rw) : "memory")
 
 #define __build_write_lock_const(rw, helper) \
-	asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \
+	asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \
 		     "jnz 2f\n" \
 		     "1:\n" \
 		    LOCK_SECTION_START("") \
include/asm-x86_64/spinlock.h +7 -3
Index: vanilla-2.6.17-rc1/include/asm-x86_64/spinlock.h
===================================================================
@@ -32,15 +32,19 @@
 	"jmp 1b\n" \
 	LOCK_SECTION_END
 
+#define __raw_spin_lock_string_up \
+	"\n\tdecl %0"
+
 #define __raw_spin_unlock_string \
 	"movl $1,%0" \
 		:"=m" (lock->slock) : : "memory"
 
 static inline void __raw_spin_lock(raw_spinlock_t *lock)
 {
-	__asm__ __volatile__(
-		__raw_spin_lock_string
-		:"=m" (lock->slock) : : "memory");
+	alternative_smp(
+		__raw_spin_lock_string,
+		__raw_spin_lock_string_up,
+		"=m" (lock->slock) : : "memory");
 }
 
 #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
include/asm-x86_64/system.h +3 -83

112 lines not on this page · raw message

arch/i386/kernel/alternative.c +85 -33

203 lines not on this page · raw message

include/asm-i386/alternative.h +2 -0
Index: vanilla-2.6.17-rc1/include/asm-i386/alternative.h
===================================================================
@@ -3,6 +3,8 @@
 
 #ifdef __KERNEL__
 
+#include <linux/types.h>
+
 struct alt_instr {
 	u8 *instr; 		/* original instruction */
 	u8 *replacement;
include/asm-x86_64/mutex.h +2 -2
Index: vanilla-2.6.17-rc1/include/asm-x86_64/mutex.h
===================================================================
@@ -24,7 +24,7 @@ do {									\
 	typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);		\
 									\
 	__asm__ __volatile__(						\
-		LOCK	"   decl (%%rdi)	\n"			\
+		LOCK_PREFIX "   decl (%%rdi)	\n"			\
 			"   js 2f		\n"			\
 			"1:			\n"			\
 									\
@@ -74,7 +74,7 @@ do {									\
 	typecheck_fn(fastcall void (*)(atomic_t *), fail_fn);		\
 									\
 	__asm__ __volatile__(						\
-		LOCK	"   incl (%%rdi)	\n"			\
+		LOCK_PREFIX "   incl (%%rdi)	\n"			\
 			"   jle 2f		\n"			\
 			"1:			\n"			\
 									\
include/asm-x86_64/semaphore.h +4 -4
Index: vanilla-2.6.17-rc1/include/asm-x86_64/semaphore.h
===================================================================
@@ -106,7 +106,7 @@ static inline void down(struct semaphore
 
 	__asm__ __volatile__(
 		"# atomic down operation\n\t"
-		LOCK "decl %0\n\t"     /* --sem->count */
+		LOCK_PREFIX "decl %0\n\t"     /* --sem->count */
 		"js 2f\n"
 		"1:\n"
 		LOCK_SECTION_START("")
@@ -130,7 +130,7 @@ static inline int down_interruptible(str
 
 	__asm__ __volatile__(
 		"# atomic interruptible down operation\n\t"
-		LOCK "decl %1\n\t"     /* --sem->count */
+		LOCK_PREFIX "decl %1\n\t"     /* --sem->count */
 		"js 2f\n\t"
 		"xorl %0,%0\n"
 		"1:\n"
@@ -154,7 +154,7 @@ static inline int down_trylock(struct se
 
 	__asm__ __volatile__(
 		"# atomic interruptible down operation\n\t"
-		LOCK "decl %1\n\t"     /* --sem->count */
+		LOCK_PREFIX "decl %1\n\t"     /* --sem->count */
 		"js 2f\n\t"
 		"xorl %0,%0\n"
 		"1:\n"
@@ -178,7 +178,7 @@ static inline void up(struct semaphore *
 {
 	__asm__ __volatile__(
 		"# atomic up operation\n\t"
-		LOCK "incl %0\n\t"     /* ++sem->count */
+		LOCK_PREFIX "incl %0\n\t"     /* ++sem->count */
 		"jle 2f\n"
 		"1:\n"
 		LOCK_SECTION_START("")