x86_64: SMP alternatives
PATCH 1/1 x86_64: SMP alternatives
Hi,
$subject says pretty much all, this is the x86_64 version of the smp alternatives patch. Detailed description and patch are attached below.
cheers,
Gerd
This is the x86_64 version of the smp alternative patch.
Changes are largely identical to the i386 version:
* alternative #define are moved to the new alternative.h file.
* one new elf section with pointers to the lock prefixes which can be nop'ed out for non-smp.
* two new elf sections simliar to the "classic" alternatives to replace SMP code with simpler UP code.
* fixup headers to use alternative.h instead of defining their own LOCK / LOCK_PREFIX macros.
The patch reuses the i386 version of the alternatives code to avoid code duplication. The code in alternatives.c was shuffled around a bit to reduce the number of #ifdefs needed. It also got some tweaks needed for x86_64 (vsyscall page handling) and new features (noreplacement option which was x86_64 only up to now). Debug printk's are changed from compile-time to runtime.
Loosely based on a early version from Bastian Blank <[email protected]>
arch/x86_64/kernel/Makefile +2 -1
@@ -8,7 +8,7 @@ obj-y := process.o signal.o entry.o trap ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \ x8664_ksyms.o i387.o syscall.o vsyscall.o \ setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \ - dmi_scan.o pci-dma.o pci-nommu.o + dmi_scan.o pci-dma.o pci-nommu.o alternative.o obj-$(CONFIG_X86_MCE) += mce.o obj-$(CONFIG_X86_MCE_INTEL) += mce_intel.o
@@ -50,4 +50,5 @@ quirks-y += ../../i386/kernel/quirks.o i8237-y += ../../i386/kernel/i8237.o msr-$(subst m,y,$(CONFIG_X86_MSR)) += ../../i386/kernel/msr.o dmi_scan-y += ../../i386/kernel/dmi_scan.o +alternative-y += ../../i386/kernel/alternative.o
arch/x86_64/kernel/module.c +25 -13
@@ -145,26 +145,38 @@ int apply_relocate(Elf_Shdr *sechdrs,return -ENOSYS ; } -extern void apply_alternatives (void *start,void *end); -int module_finalize (const Elf_Ehdr *hdr, -const Elf_Shdr *sechdrs, -struct module *me) +const Elf_Shdr *sechdrs, +struct module *me) { -const Elf_Shdr *s; +const Elf_Shdr *s, *text =NULL , *alt =NULL , *locks =NULL ;char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; -/* look for .altinstructions to patch */ -for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { -void *seg; -if (strcmp (".altinstructions" , secstrings + s->sh_name)) -continue ; - seg = (void *)s->sh_addr; -apply_alternatives (seg, seg + s->sh_size); - } +for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { +if (!strcmp (".text" , secstrings + s->sh_name)) + text = s; +if (!strcmp (".altinstructions" , secstrings + s->sh_name)) + alt = s; +if (!strcmp (".smp_locks" , secstrings + s->sh_name)) + locks= s; + } + +if (alt) { +/* patch .altinstructions */ +void *aseg = (void *)alt->sh_addr; +apply_alternatives (aseg, aseg + alt->sh_size); + } +if (locks && text) { +void *lseg = (void *)locks->sh_addr; +void *tseg = (void *)text->sh_addr; +alternatives_smp_module_add (me, me->name, + lseg, lseg + locks->sh_size, + tseg, tseg + text->sh_size); + }return 0 ; }void module_arch_cleanup (struct module *mod) { +alternatives_smp_module_del (mod); }
arch/x86_64/kernel/setup.c +1 -75
88 lines not on this page · raw message
arch/x86_64/kernel/smpboot.c +4 -0
@@ -797,6 +797,8 @@ static int __cpuinit do_boot_cpu(int cpu } +alternatives_smp_switch (1 ); + c_idle.idle =get_idle_for_cpu (cpu);if (c_idle.idle) {
@@ -1259,6 +1261,8 @@ void __cpu_die(unsigned int cpu)/* They ack this in play_dead by setting CPU_DEAD */ if (per_cpu (cpu_state, cpu) ==CPU_DEAD ) {printk ("CPU %d is now offline\n" , cpu); +if (1 ==num_online_cpus ()) +alternatives_smp_switch (0 );return ; }msleep (100 );
arch/x86_64/kernel/vmlinux.lds.S +20 -0
@@ -131,6 +131,26 @@ SECTIONS *(.data.page_aligned) } + /* might get freed after init */ + . = ALIGN(4096); + __smp_alt_begin = .; + __smp_alt_instructions = .; + .smp_altinstructions : AT(ADDR(.smp_altinstructions) - LOAD_OFFSET) { + *(.smp_altinstructions) + } + __smp_alt_instructions_end = .; + . = ALIGN(8); + __smp_locks = .; + .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) { + *(.smp_locks) + } + __smp_locks_end = .; + .smp_altinstr_replacement : AT(ADDR(.smp_altinstr_replacement) - LOAD_OFFSET) { + *(.smp_altinstr_replacement) + } + . = ALIGN(4096); + __smp_alt_end = .; + . = ALIGN(4096); /* Init code and data */ __init_begin = .; .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
arch/x86_64/mm/init.c +14 -13
@@ -613,20 +613,29 @@ void __init mem_init(void)#endif } -void free_initmem (void ) +void free_init_pages (char *what,unsigned long begin,unsigned long end) {unsigned long addr; - addr = (unsigned long )(&__init_begin); -for (; addr < (unsigned long )(&__init_end); addr +=PAGE_SIZE ) { +if (begin >= end) +return ; + +printk (KERN_INFO "Freeing %s: %ldk freed\n" , what, (end - begin) >>10 ); +for (addr = begin; addr < end; addr +=PAGE_SIZE ) {ClearPageReserved (virt_to_page (addr));init_page_count (virt_to_page (addr));memset ((void *)(addr & ~(PAGE_SIZE -1 )),0xcc ,PAGE_SIZE );free_page (addr); totalram_pages++; } +} + +void free_initmem (void ) +{memset (__initdata_begin,0xba , __initdata_end - __initdata_begin); -printk ("Freeing unused kernel memory: %luk freed\n" , (__init_end - __init_begin) >>10 ); +free_init_pages ("unused kernel memory" , + (unsigned long )(&__init_begin), + (unsigned long )(&__init_end)); }#ifdef CONFIG_DEBUG_RODATA
@@ -655,15 +664,7 @@ void mark_rodata_ro(void)#ifdef CONFIG_BLK_DEV_INITRD void free_initrd_mem (unsigned long start,unsigned long end) { -if (start >= end) -return ; -printk ("Freeing initrd memory: %ldk freed\n" , (end - start) >>10 ); -for (; start < end; start +=PAGE_SIZE ) { -ClearPageReserved (virt_to_page (start)); -init_page_count (virt_to_page (start)); -free_page (start); - totalram_pages++; - } +free_init_pages ("initrd memory" , start, end); }#endif
include/asm-x86_64/alternative.h new file +146 -0
146 lines not on this page · raw message
include/asm-x86_64/atomic.h +21 -29
179 lines not on this page · raw message
include/asm-x86_64/bitops.h +1 -6
@@ -6,12 +6,7 @@*/ #include <linux/config.h> - -#ifdef CONFIG_SMP -#define LOCK_PREFIX "lock ; " -#else -#define LOCK_PREFIX "" -#endif +#include <asm/alternative.h> #define ADDR (*(volatile long *) addr)
include/asm-x86_64/cpufeature.h +1 -0
@@ -64,6 +64,7 @@#define X86_FEATURE_REP_GOOD (3*32+ 4)/* rep microcode works well on this CPU */ #define X86_FEATURE_CONSTANT_TSC (3*32+5)/* TSC runs at constant rate */ #define X86_FEATURE_SYNC_RDTSC (3*32+6)/* RDTSC syncs CPU core */ +#define X86_FEATURE_UP (3*32+ 7)/* SMP kernel running on UP */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0)/* Streaming SIMD Extensions-3 */
include/asm-x86_64/rwlock.h +4 -4
@@ -24,7 +24,7 @@#define RW_LOCK_BIAS_STR "0x01000000"#define __build_read_lock_ptr (rw, helper) \ - asm volatile(LOCK "subl $1,(%0)\n\t" \ + asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \ "js 2f\n" \ "1:\n" \ LOCK_SECTION_START("") \
@@ -34,7 +34,7 @@ ::"a" (rw) : "memory")#define __build_read_lock_const (rw, helper) \ - asm volatile(LOCK "subl $1,%0\n\t" \ + asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \ "js 2f\n" \ "1:\n" \ LOCK_SECTION_START("") \
@@ -54,7 +54,7 @@ } while (0)#define __build_write_lock_ptr (rw, helper) \ - asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ + asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ "jnz 2f\n" \ "1:\n" \ LOCK_SECTION_START("") \
@@ -64,7 +64,7 @@ ::"a" (rw) : "memory")#define __build_write_lock_const (rw, helper) \ - asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ + asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ "jnz 2f\n" \ "1:\n" \ LOCK_SECTION_START("") \
include/asm-x86_64/spinlock.h +7 -3
@@ -32,15 +32,19 @@"jmp 1b\n" \ LOCK_SECTION_END +#define __raw_spin_lock_string_up \ + "\n\tdecl %0" +#define __raw_spin_unlock_string \ "movl $1,%0" \ :"=m" (lock->slock) : : "memory"static inline void __raw_spin_lock (raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_lock_string - :"=m" (lock->slock) : : "memory"); +alternative_smp ( + __raw_spin_lock_string, + __raw_spin_lock_string_up, + "=m" (lock->slock) : : "memory"); }#define __raw_spin_lock_flags (lock, flags) __raw_spin_lock(lock)
include/asm-x86_64/system.h +3 -83
112 lines not on this page · raw message
arch/i386/kernel/alternative.c +85 -33
203 lines not on this page · raw message
include/asm-i386/alternative.h +2 -0
@@ -3,6 +3,8 @@ #ifdef __KERNEL__ +#include <linux/types.h> + struct alt_instr { u8 *instr; /* original instruction */ u8 *replacement;
include/asm-x86_64/mutex.h +2 -2
@@ -24,7 +24,7 @@ do { \ typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ \ __asm__ __volatile__( \ -LOCK " decl (%%rdi) \n" \ +LOCK_PREFIX " decl (%%rdi) \n" \" js 2f \n" \ "1: \n" \ \
@@ -74,7 +74,7 @@ do { \ typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ \ __asm__ __volatile__( \ -LOCK " incl (%%rdi) \n" \ +LOCK_PREFIX " incl (%%rdi) \n" \" jle 2f \n" \ "1: \n" \ \
include/asm-x86_64/semaphore.h +4 -4
@@ -106,7 +106,7 @@ static inline void down(struct semaphore __asm__ __volatile__("# atomic down operation\n\t" -LOCK "decl %0\n\t" /* --sem->count */ +LOCK_PREFIX "decl %0\n\t" /* --sem->count */ "js 2f\n" "1:\n" LOCK_SECTION_START("")
@@ -130,7 +130,7 @@ static inline int down_interruptible(str __asm__ __volatile__("# atomic interruptible down operation\n\t" -LOCK "decl %1\n\t" /* --sem->count */ +LOCK_PREFIX "decl %1\n\t" /* --sem->count */ "js 2f\n\t" "xorl %0,%0\n" "1:\n"
@@ -154,7 +154,7 @@ static inline int down_trylock(struct se __asm__ __volatile__("# atomic interruptible down operation\n\t" -LOCK "decl %1\n\t" /* --sem->count */ +LOCK_PREFIX "decl %1\n\t" /* --sem->count */ "js 2f\n\t" "xorl %0,%0\n" "1:\n"
@@ -178,7 +178,7 @@ static inline void up(struct semaphore * { __asm__ __volatile__("# atomic up operation\n\t" -LOCK "incl %0\n\t" /* ++sem->count */ +LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ "jle 2f\n" "1:\n" LOCK_SECTION_START("")